From 8ce1155a1eb490625a7e949a10c4283b4b773d30 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 13 Apr 2023 16:50:24 +0200 Subject: core: replace bundle checksum with BLAKE3 BLAKE3 has a verified streaming mode (Bao), which allows variable chunk sizes without altering the root hash. This means that a BLAKE3 hash can serve as a long-term stable content address in location-independent storage (as demonstrated by iroh). Signed-off-by: Kim Altintop --- Cargo.lock | 34 ++++++++++++++++++++++++++++++++++ Cargo.toml | 3 +++ src/bundle.rs | 46 ++++++++++++++++++++++++++++++++++++---------- src/bundle/fetch.rs | 10 +++------- src/bundle/header.rs | 6 ++---- src/bundle/list.rs | 6 ++---- src/cmd/patch/prepare.rs | 6 ++---- src/git.rs | 6 ++---- src/http.rs | 6 ++---- src/io.rs | 9 +++------ src/metadata.rs | 6 ++---- src/metadata/drop.rs | 6 ++---- src/metadata/identity.rs | 2 +- src/patches.rs | 14 ++++++-------- src/patches/bundle.rs | 18 ++++++++---------- src/patches/record.rs | 8 ++------ 16 files changed, 110 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19c6e65..e19a8d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -86,6 +86,18 @@ dependencies = [ "backtrace", ] +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + [[package]] name = "ascii" version = "1.1.0" @@ -148,6 +160,20 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "blake3" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "digest 0.10.6", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -318,6 +344,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" +[[package]] +name = "constant_time_eq" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" + [[package]] name = "core-foundation" version = "0.9.3" @@ -785,10 +817,12 @@ version = "0.1.0" dependencies = [ "anyhow", "base64", + "blake3", "clap", "clap_complete", "clap_mangen", "console", + "digest 0.10.6", "directories", "either", "erased-serde", diff --git a/Cargo.toml b/Cargo.toml index 4de6fdb..229e85b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,12 +16,15 @@ sha1dc = ["sha1collisiondetection"] anyhow.features = ["backtrace"] anyhow.version = "1" base64.version = "0.13" +blake3.version = "1.3.3" +blake3.features = ["traits-preview"] clap.features = ["derive", "env", "string", "wrap_help"] clap.version = "4.0" clap_complete.version = "4.0" clap_mangen.version = "0.2" console.default-features = false console.version = "0.15" +digest.version = "0.10" directories.version = "4.0" either.version = "1.8" erased-serde.version = "0.3" diff --git a/src/bundle.rs b/src/bundle.rs index 25eafd0..6f333ca 100644 --- a/src/bundle.rs +++ b/src/bundle.rs @@ -1,13 +1,16 @@ // Copyright © 2022 Kim Altintop // SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception -use std::io; +use std::{ + fmt::{ + self, + Debug, + Display, + }, + io, +}; use log::info; -use sha2::{ - Digest, - Sha256, -}; use url::Url; use crate::io::{ @@ -42,12 +45,35 @@ pub use list::{ pub const FILE_EXTENSION: &str = "bundle"; pub const DOT_FILE_EXTENSION: &str = ".bundle"; +#[derive(Clone, Copy, Eq, Hash, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Checksum(#[serde(with = "crate::serde::display")] blake3::Hash); + +impl Debug for Checksum { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let hex = self.0.to_hex(); + let hex: &str = hex.as_str(); + + f.debug_tuple("Checksum").field(&hex).finish() + } +} + +impl Display for Checksum { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.0, f) + } +} + +impl From<&blake3::Hasher> for Checksum { + fn from(hasher: &blake3::Hasher) -> Self { + Self(hasher.finalize()) + } +} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Info { pub len: u64, pub hash: Hash, - #[serde(with = "hex::serde")] - pub checksum: [u8; 32], + pub checksum: Checksum, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub uris: Vec, } @@ -56,7 +82,7 @@ pub struct Info { pub struct Expect<'a> { pub len: u64, pub hash: &'a Hash, - pub checksum: Option<&'a [u8]>, + pub checksum: Option<&'a Checksum>, } impl<'a> From<&'a Info> for Expect<'a> { @@ -80,7 +106,7 @@ pub fn create(mut out: W, repo: &git2::Repository, header: &Header) -> crate: where W: io::Write, { - let mut hasher = HashWriter::new(Sha256::new(), &mut out); + let mut hasher = HashWriter::new(blake3::Hasher::new(), &mut out); let mut writer = LenWriter::new(&mut hasher); let mut pack = { let mut pack = repo.packbuilder()?; @@ -101,7 +127,7 @@ where let len = writer.bytes_written(); let hash = header.hash(); - let checksum = hasher.hash().into(); + let checksum = Checksum::from(hasher.hasher()); info!("Created patch bundle {hash}"); diff --git a/src/bundle/fetch.rs b/src/bundle/fetch.rs index 4e58000..ff10e27 100644 --- a/src/bundle/fetch.rs +++ b/src/bundle/fetch.rs @@ -22,10 +22,6 @@ use either::Either::{ Left, Right, }; -use sha2::{ - Digest, - Sha256, -}; use tempfile::NamedTempFile; use url::Url; @@ -96,13 +92,13 @@ impl Fetcher { LockedFile::atomic(&path, true, LockedFile::DEFAULT_PERMISSIONS)? }; - let mut out = HashWriter::new(Sha256::new(), &mut lck); + let mut out = HashWriter::new(blake3::Hasher::new(), &mut lck); out.write_all(&buf)?; let len = buf.len() as u64 + io::copy(&mut body.take(expect.len), &mut out)?; - let checksum = out.hash().into(); + let checksum = bundle::Checksum::from(out.hasher()); if let Some(chk) = expect.checksum { - ensure!(chk == checksum, "checksum mismatch"); + ensure!(chk == &checksum, "checksum mismatch"); } lck.seek(SeekFrom::Start(0))?; let header = Header::from_reader(&mut lck)?; diff --git a/src/bundle/header.rs b/src/bundle/header.rs index 6f3dfe3..d296af1 100644 --- a/src/bundle/header.rs +++ b/src/bundle/header.rs @@ -12,15 +12,13 @@ use std::{ str::FromStr, }; +use digest::Digest; use hex::{ FromHex, FromHexError, }; use refs::Refname; -use sha2::{ - Digest, - Sha256, -}; +use sha2::Sha256; use super::error; use crate::{ diff --git a/src/bundle/list.rs b/src/bundle/list.rs index 21753fa..ea6296f 100644 --- a/src/bundle/list.rs +++ b/src/bundle/list.rs @@ -19,11 +19,9 @@ use std::{ }; use anyhow::anyhow; +use digest::Digest; use once_cell::sync::Lazy; -use sha2::{ - Digest, - Sha256, -}; +use sha2::Sha256; use url::Url; use crate::git::{ diff --git a/src/cmd/patch/prepare.rs b/src/cmd/patch/prepare.rs index 06d5ec9..8f8b871 100644 --- a/src/cmd/patch/prepare.rs +++ b/src/cmd/patch/prepare.rs @@ -11,11 +11,9 @@ use anyhow::{ bail, ensure, }; +use digest::Digest; use either::Either::Left; -use sha2::{ - Digest, - Sha256, -}; +use sha2::Sha256; use crate::{ bundle, diff --git a/src/git.rs b/src/git.rs index f837711..a627048 100644 --- a/src/git.rs +++ b/src/git.rs @@ -11,11 +11,9 @@ use anyhow::{ ensure, Context, }; +use digest::Digest; use once_cell::sync::Lazy; -use sha2::{ - Digest, - Sha256, -}; +use sha2::Sha256; mod commit; pub use commit::{ diff --git a/src/http.rs b/src/http.rs index d52ef8f..f8cfaba 100644 --- a/src/http.rs +++ b/src/http.rs @@ -15,15 +15,13 @@ use std::{ }, }; +use digest::Digest; use log::{ debug, error, }; use once_cell::sync::Lazy; -use sha2::{ - Digest, - Sha256, -}; +use sha2::Sha256; use threadpool::ThreadPool; use tiny_http::{ Header, diff --git a/src/io.rs b/src/io.rs index 86f91c6..324d12c 100644 --- a/src/io.rs +++ b/src/io.rs @@ -1,10 +1,7 @@ // Copyright © 2022 Kim Altintop // SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception -use sha2::{ - digest::generic_array::GenericArray, - Digest, -}; +use digest::Digest; /// Created by [`Lines::until_blank`], stops iteration at the first blank line. pub struct UntilBlank { @@ -94,8 +91,8 @@ impl HashWriter where D: Digest, { - pub fn hash(self) -> GenericArray { - self.hasher.finalize() + pub fn hasher(&self) -> &D { + &self.hasher } } diff --git a/src/metadata.rs b/src/metadata.rs index 2da268f..378f462 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -15,11 +15,9 @@ use std::{ ops::DerefMut, }; +use digest::Digest; use serde::ser::SerializeSeq; -use sha2::{ - Digest, - Sha512, -}; +use sha2::Sha512; use time::{ Duration, OffsetDateTime, diff --git a/src/metadata/drop.rs b/src/metadata/drop.rs index ae9aefd..601a157 100644 --- a/src/metadata/drop.rs +++ b/src/metadata/drop.rs @@ -13,11 +13,9 @@ use std::{ ops::Deref, }; +use digest::Digest; use log::warn; -use sha2::{ - Digest, - Sha512, -}; +use sha2::Sha512; use signature::Verifier; use super::{ diff --git a/src/metadata/identity.rs b/src/metadata/identity.rs index 8ded144..0b45731 100644 --- a/src/metadata/identity.rs +++ b/src/metadata/identity.rs @@ -20,10 +20,10 @@ use anyhow::{ anyhow, ensure, }; +use digest::Digest; use hex::FromHex; use log::warn; use sha2::{ - Digest, Sha256, Sha512, }; diff --git a/src/patches.rs b/src/patches.rs index 8623e4e..5efcf90 100644 --- a/src/patches.rs +++ b/src/patches.rs @@ -15,16 +15,14 @@ use anyhow::{ bail, }; -use hex::FromHex; -use once_cell::sync::Lazy; -use sha2::{ - digest::{ - generic_array::GenericArray, - typenum::U32, - }, +use digest::{ + generic_array::GenericArray, + typenum::U32, Digest, - Sha256, }; +use hex::FromHex; +use once_cell::sync::Lazy; +use sha2::Sha256; use crate::{ git::Refname, diff --git a/src/patches/bundle.rs b/src/patches/bundle.rs index 296b24a..52397a1 100644 --- a/src/patches/bundle.rs +++ b/src/patches/bundle.rs @@ -21,11 +21,9 @@ use anyhow::{ ensure, Context, }; +use digest::Digest; use multipart::client::lazy::Multipart; -use sha2::{ - Digest, - Sha256, -}; +use sha2::Sha256; use tempfile::NamedTempFile; use url::Url; @@ -105,14 +103,14 @@ impl Bundle { let encryption = pack.encryption()?; drop(pack); let mut file = File::open(&path)?; - let mut sha2 = Sha256::new(); + let mut hasher = blake3::Hasher::new(); - let len = io::copy(&mut file, &mut sha2)?; + let len = io::copy(&mut file, &mut hasher)?; let hash = header.hash(); ensure!(expect.hash == &hash, "header hash mismatch"); - let checksum = sha2.finalize().into(); + let checksum = bundle::Checksum::from(&hasher); if let Some(expect) = expect.checksum { - ensure!(expect == checksum, "claimed and actual hash differ"); + ensure!(expect == &checksum, "claimed and actual hash differ"); } let info = bundle::Info { @@ -138,10 +136,10 @@ impl Bundle { { std::fs::create_dir_all(&to)?; let mut tmp = NamedTempFile::new_in(&to)?; - let mut out = HashWriter::new(Sha256::new(), &mut tmp); + let mut out = HashWriter::new(blake3::Hasher::new(), &mut tmp); let len = io::copy(&mut from, &mut out)?; - let checksum = out.hash().into(); + let checksum = bundle::Checksum::from(out.hasher()); let (header, mut pack) = split(tmp.path())?; let hash = header.hash(); diff --git a/src/patches/record.rs b/src/patches/record.rs index 6a95973..9bf7856 100644 --- a/src/patches/record.rs +++ b/src/patches/record.rs @@ -25,16 +25,12 @@ use anyhow::{ ensure, Context, }; - +use digest::Digest; use hex::{ FromHex, ToHex, }; - -use sha2::{ - Digest, - Sha256, -}; +use sha2::Sha256; use signature::{ Signature as _, Verifier, -- cgit v1.2.3