summaryrefslogtreecommitdiff
path: root/src/cmd/drop
diff options
context:
space:
mode:
Diffstat (limited to 'src/cmd/drop')
-rw-r--r--src/cmd/drop/bundles.rs32
-rw-r--r--src/cmd/drop/bundles/prune.rs113
-rw-r--r--src/cmd/drop/bundles/sync.rs276
-rw-r--r--src/cmd/drop/edit.rs368
-rw-r--r--src/cmd/drop/init.rs194
-rw-r--r--src/cmd/drop/serve.rs140
-rw-r--r--src/cmd/drop/show.rs208
-rw-r--r--src/cmd/drop/snapshot.rs20
-rw-r--r--src/cmd/drop/unbundle.rs93
9 files changed, 1444 insertions, 0 deletions
diff --git a/src/cmd/drop/bundles.rs b/src/cmd/drop/bundles.rs
new file mode 100644
index 0000000..7c3e726
--- /dev/null
+++ b/src/cmd/drop/bundles.rs
@@ -0,0 +1,32 @@
+// Copyright © 2022 Kim Altintop <kim@eagain.io>
+// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception
+
+use crate::cmd;
+
+mod prune;
+pub use prune::{
+ prune,
+ Prune,
+};
+
+mod sync;
+pub use sync::{
+ sync,
+ Sync,
+};
+
+#[derive(Debug, clap::Subcommand)]
+#[allow(clippy::large_enum_variant)]
+pub enum Bundles {
+ Sync(Sync),
+ Prune(Prune),
+}
+
+impl Bundles {
+ pub fn run(self) -> cmd::Result<cmd::Output> {
+ match self {
+ Self::Sync(args) => sync(args).map(cmd::IntoOutput::into_output),
+ Self::Prune(args) => prune(args).map(cmd::IntoOutput::into_output),
+ }
+ }
+}
diff --git a/src/cmd/drop/bundles/prune.rs b/src/cmd/drop/bundles/prune.rs
new file mode 100644
index 0000000..6bd984d
--- /dev/null
+++ b/src/cmd/drop/bundles/prune.rs
@@ -0,0 +1,113 @@
+// Copyright © 2022 Kim Altintop <kim@eagain.io>
+// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception
+
+use std::{
+ collections::BTreeSet,
+ fs,
+ path::PathBuf,
+ str::FromStr,
+};
+
+use clap::ValueHint;
+
+use crate::{
+ bundle,
+ cfg,
+ cmd::{
+ self,
+ ui::{
+ info,
+ warn,
+ },
+ },
+ git,
+ patches::iter::dropped,
+};
+
+// TODO:
+//
+// - option to prune bundles made obsolete by snapshots
+
+#[derive(Debug, clap::Args)]
+pub struct Prune {
+ /// Path to the drop repository
+ #[clap(from_global)]
+ git_dir: PathBuf,
+ /// The directory where to write the bundle to
+ ///
+ /// Unless this is an absolute path, it is treated as relative to $GIT_DIR.
+ #[clap(
+ long,
+ value_parser,
+ value_name = "DIR",
+ default_value_os_t = cfg::paths::bundles().to_owned(),
+ value_hint = ValueHint::DirPath,
+ )]
+ bundle_dir: PathBuf,
+ /// Name of a git ref holding the drop metadata history
+ ///
+ /// All locally tracked drops should be given, otherwise bundles might get
+ /// pruned which are still being referred to.
+ #[clap(long = "drop", value_parser, value_name = "REF")]
+ drop_refs: Vec<String>,
+ /// Pretend to unlink, but don't
+ #[clap(long, value_parser)]
+ dry_run: bool,
+ /// Also remove location files (.uris)
+ #[clap(long, value_parser)]
+ remove_locations: bool,
+}
+
+pub fn prune(args: Prune) -> cmd::Result<Vec<bundle::Hash>> {
+ let repo = git::repo::open_bare(&args.git_dir)?;
+ let bundle_dir = if args.bundle_dir.is_relative() {
+ repo.path().join(args.bundle_dir)
+ } else {
+ args.bundle_dir
+ };
+
+ let mut seen = BTreeSet::new();
+ for short in &args.drop_refs {
+ let drop_ref = repo.resolve_reference_from_short_name(short)?;
+ let ref_name = drop_ref.name().expect("drop references to be valid utf8");
+ info!("Collecting bundle hashes from {ref_name} ...");
+ for record in dropped::records(&repo, ref_name) {
+ let record = record?;
+ seen.insert(*record.bundle_hash());
+ }
+ }
+
+ info!("Traversing bundle dir {} ...", bundle_dir.display());
+ let mut pruned = Vec::new();
+ for entry in fs::read_dir(&bundle_dir)? {
+ let entry = entry?;
+ let path = entry.path();
+ match path.extension() {
+ Some(ext) if ext == bundle::FILE_EXTENSION => {
+ let name = path.file_stem();
+ match name
+ .and_then(|n| n.to_str())
+ .and_then(|s| bundle::Hash::from_str(s).ok())
+ {
+ Some(hash) => {
+ if !seen.contains(&hash) {
+ if !args.dry_run {
+ fs::remove_file(&path)?;
+ }
+ pruned.push(hash);
+ }
+ },
+ None => warn!("Ignoring {}: file name not a bundle hash", path.display()),
+ }
+ },
+ Some(ext) if ext == bundle::list::FILE_EXTENSION => {
+ if args.remove_locations {
+ fs::remove_file(&path)?;
+ }
+ },
+ _ => warn!("Ignoring {}: missing .bundle", path.display()),
+ }
+ }
+
+ Ok(pruned)
+}
diff --git a/src/cmd/drop/bundles/sync.rs b/src/cmd/drop/bundles/sync.rs
new file mode 100644
index 0000000..21fd58b
--- /dev/null
+++ b/src/cmd/drop/bundles/sync.rs
@@ -0,0 +1,276 @@
+// Copyright © 2022 Kim Altintop <kim@eagain.io>
+// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception
+
+use std::{
+ borrow::Cow,
+ mem,
+ num::NonZeroUsize,
+ path::PathBuf,
+ sync::{
+ Arc,
+ Mutex,
+ },
+ time::{
+ SystemTime,
+ UNIX_EPOCH,
+ },
+};
+
+use anyhow::anyhow;
+use clap::ValueHint;
+use either::Either::{
+ Left,
+ Right,
+};
+use threadpool::ThreadPool;
+use url::Url;
+
+use crate::{
+ bundle,
+ cfg,
+ cmd::{
+ self,
+ drop::Common,
+ ui::{
+ debug,
+ info,
+ warn,
+ },
+ },
+ git::{
+ self,
+ if_not_found_none,
+ },
+ patches::{
+ self,
+ iter::dropped,
+ record,
+ REF_IT_PATCHES,
+ },
+};
+
+/// Max number of locations to store from the remote for which we don't know if
+/// they'd succeed or not.
+pub const MAX_UNTRIED_LOCATIONS: usize = 3;
+
+#[derive(Debug, clap::Args)]
+pub struct Sync {
+ #[clap(flatten)]
+ common: Common,
+ /// The directory where to write the bundle to
+ ///
+ /// Unless this is an absolute path, it is treated as relative to $GIT_DIR.
+ #[clap(
+ long,
+ value_parser,
+ value_name = "DIR",
+ default_value_os_t = cfg::paths::bundles().to_owned(),
+ value_hint = ValueHint::DirPath,
+ )]
+ bundle_dir: PathBuf,
+ /// Name of the git ref holding the drop metadata history
+ #[clap(long = "drop", value_parser, value_name = "REF")]
+ drop_ref: Option<String>,
+ /// Base URL to fetch from
+ #[clap(long, value_parser, value_name = "URL", value_hint = ValueHint::Url)]
+ url: Url,
+ /// Fetch via IPFS
+ #[clap(
+ long,
+ value_parser,
+ value_name = "URL",
+ value_hint = ValueHint::Url,
+ env = "IPFS_GATEWAY",
+ default_value_t = Url::parse("https://ipfs.io").unwrap(),
+ )]
+ ipfs_gateway: Url,
+ /// Fetch even if the bundle already exists locally
+ #[clap(long, value_parser)]
+ overwrite: bool,
+ /// Ignore snapshots if encountered
+ #[clap(long, value_parser)]
+ no_snapshots: bool,
+ /// Maximum number of concurrent downloads. Default is the number of
+ /// available cores.
+ #[clap(short, long, value_parser, default_value_t = def_jobs())]
+ jobs: NonZeroUsize,
+}
+
+fn def_jobs() -> NonZeroUsize {
+ NonZeroUsize::new(num_cpus::get()).unwrap_or_else(|| NonZeroUsize::new(1).unwrap())
+}
+
+pub fn sync(args: Sync) -> cmd::Result<Vec<bundle::Info>> {
+ let repo = git::repo::open_bare(&args.common.git_dir)?;
+ let bundle_dir = if args.bundle_dir.is_relative() {
+ repo.path().join(args.bundle_dir)
+ } else {
+ args.bundle_dir
+ };
+ let drop_ref = match args.drop_ref {
+ Some(rev) => if_not_found_none(repo.resolve_reference_from_short_name(&rev))?
+ .ok_or_else(|| anyhow!("no ref matching {rev} found"))?
+ .name()
+ .ok_or_else(|| anyhow!("invalid drop"))?
+ .to_owned(),
+ None => REF_IT_PATCHES.to_owned(),
+ };
+ let base_url = args.url.join("bundles/")?;
+ let fetcher = Arc::new(Fetcher {
+ fetcher: bundle::Fetcher::default(),
+ bundle_dir,
+ base_url: base_url.clone(),
+ ipfs_gateway: args.ipfs_gateway,
+ });
+
+ let pool = ThreadPool::new(args.jobs.get());
+
+ let fetched = Arc::new(Mutex::new(Vec::new()));
+ let mut chasing_snaphots = false;
+ for record in dropped::records(&repo, &drop_ref) {
+ let record = record?;
+ let hexdig = record.bundle_hash().to_string();
+
+ if record.is_snapshot() {
+ if args.no_snapshots {
+ info!("Skipping snapshot bundle {hexdig}");
+ continue;
+ } else {
+ chasing_snaphots = true;
+ }
+ } else if chasing_snaphots && !record.is_mergepoint() {
+ info!("Skipping non-snapshot bundle {hexdig}");
+ continue;
+ }
+
+ if !args.overwrite && record.bundle_path(&fetcher.bundle_dir).exists() {
+ info!("Skipping existing bundle {hexdig}");
+ continue;
+ }
+
+ let record::BundleInfo {
+ info: bundle::Info { len, hash, .. },
+ prerequisites,
+ ..
+ } = record.bundle_info();
+ let url = base_url.join(&hexdig)?;
+
+ pool.execute({
+ let len = *len;
+ let hash = *hash;
+ let fetched = Arc::clone(&fetched);
+ let fetcher = Arc::clone(&fetcher);
+ move || match fetcher.try_fetch(url, len, &hash) {
+ Ok(hash) => fetched.lock().unwrap().push(hash),
+ Err(e) => warn!("Download failed: {e}"),
+ }
+ });
+
+ if record.is_snapshot() && prerequisites.is_empty() {
+ info!("Full snapshot encountered, stopping here");
+ break;
+ }
+ }
+
+ pool.join();
+ let fetched = {
+ let mut guard = fetched.lock().unwrap();
+ mem::take(&mut *guard)
+ };
+
+ Ok(fetched)
+}
+
+struct Fetcher {
+ fetcher: bundle::Fetcher,
+ bundle_dir: PathBuf,
+ base_url: Url,
+ ipfs_gateway: Url,
+}
+
+impl Fetcher {
+ fn try_fetch(&self, url: Url, len: u64, hash: &bundle::Hash) -> cmd::Result<bundle::Info> {
+ info!("Fetching {url} ...");
+
+ let expect = bundle::Expect {
+ len,
+ hash,
+ checksum: None,
+ };
+ let mut locations = Vec::new();
+ let (fetched, origin) = self
+ .fetcher
+ .fetch(&url, &self.bundle_dir, expect)
+ .and_then(|resp| match resp {
+ Right(fetched) => Ok((fetched, url)),
+ Left(lst) => {
+ info!("{url}: response was a bundle list, trying alternate locations");
+
+ let mut iter = lst.bundles.into_iter();
+ let mut found = None;
+
+ for bundle::Location { uri, .. } in &mut iter {
+ if let Some(url) = self.url_from_uri(uri) {
+ if let Ok(Right(info)) =
+ self.fetcher.fetch(&url, &self.bundle_dir, expect)
+ {
+ found = Some((info, url));
+ break;
+ }
+ }
+ }
+
+ // If there are bundle uris left, remember a few
+ let now = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .expect("backwards system clock")
+ .as_secs();
+ locations.extend(
+ iter
+ // Don't let the remote inflate the priority of
+ // unverified locations
+ .filter(|loc| loc.creation_token.map(|t| t < now).unwrap_or(true))
+ // Only known protocols, relative to base url
+ .filter_map(|loc| {
+ let url = loc.uri.abs(&self.base_url).ok()?;
+ matches!(url.scheme(), "http" | "https" | "ipfs").then(|| {
+ bundle::Location {
+ uri: url.into_owned().into(),
+ ..loc
+ }
+ })
+ })
+ .take(MAX_UNTRIED_LOCATIONS),
+ );
+
+ found.ok_or_else(|| anyhow!("{url}: no reachable location found"))
+ },
+ })?;
+
+ info!("Downloaded {hash} from {origin}");
+ let bundle = patches::Bundle::from_fetched(fetched)?;
+ bundle.write_bundle_list(locations)?;
+
+ Ok(bundle.into())
+ }
+
+ fn url_from_uri(&self, uri: bundle::Uri) -> Option<Url> {
+ uri.abs(&self.base_url)
+ .map_err(Into::into)
+ .and_then(|url: Cow<Url>| -> cmd::Result<Url> {
+ match url.scheme() {
+ "http" | "https" => Ok(url.into_owned()),
+ "ipfs" => {
+ let cid = url
+ .host_str()
+ .ok_or_else(|| anyhow!("{url}: host part not an IPFS CID"))?;
+ let url = self.ipfs_gateway.join(&format!("/ipfs/{cid}"))?;
+ Ok(url)
+ },
+ _ => Err(anyhow!("{url}: unsupported protocol")),
+ }
+ })
+ .map_err(|e| debug!("discarding {}: {}", uri.as_str(), e))
+ .ok()
+ }
+}
diff --git a/src/cmd/drop/edit.rs b/src/cmd/drop/edit.rs
new file mode 100644
index 0000000..9103819
--- /dev/null
+++ b/src/cmd/drop/edit.rs
@@ -0,0 +1,368 @@
+// Copyright © 2022 Kim Altintop <kim@eagain.io>
+// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception
+
+use std::{
+ iter,
+ path::PathBuf,
+};
+
+use anyhow::{
+ anyhow,
+ ensure,
+};
+
+use super::{
+ find_id,
+ Common,
+ Editable,
+};
+use crate::{
+ cfg,
+ cmd::{
+ self,
+ ui::{
+ self,
+ edit_commit_message,
+ edit_metadata,
+ info,
+ },
+ Aborted,
+ },
+ git::{
+ self,
+ refs,
+ Refname,
+ },
+ json,
+ keys::Signer,
+ metadata::{
+ self,
+ git::{
+ FromGit,
+ GitDrop,
+ META_FILE_ALTERNATES,
+ META_FILE_DROP,
+ META_FILE_MIRRORS,
+ },
+ IdentityId,
+ Metadata,
+ },
+ patches::{
+ self,
+ REF_HEADS_PATCHES,
+ REF_IT_PATCHES,
+ },
+};
+
+#[derive(Debug, clap::Args)]
+pub struct Edit {
+ #[clap(flatten)]
+ common: Common,
+ /// Commit message for this edit
+ ///
+ /// Like git, $EDITOR will be invoked if not specified.
+ #[clap(short, long, value_parser)]
+ message: Option<String>,
+
+ #[clap(subcommand)]
+ cmd: Option<Cmd>,
+}
+
+#[derive(Debug, clap::Subcommand)]
+enum Cmd {
+ /// Edit the mirrors file
+ Mirrors,
+ /// Edit the alternates file
+ Alternates,
+}
+
+#[derive(serde::Serialize)]
+pub struct Output {
+ repo: PathBuf,
+ #[serde(rename = "ref")]
+ refname: Refname,
+ #[serde(with = "crate::git::serde::oid")]
+ commit: git2::Oid,
+}
+
+pub fn edit(args: Edit) -> cmd::Result<Output> {
+ let Common { git_dir, id_path } = args.common;
+
+ let repo = git::repo::open(git_dir)?;
+ let drop_ref = if repo.is_bare() {
+ REF_HEADS_PATCHES
+ } else {
+ REF_IT_PATCHES
+ }
+ .parse()
+ .unwrap();
+
+ let id_path = id_path.open_git();
+ git::add_alternates(&repo, &id_path)?;
+ let cfg = repo.config()?.snapshot()?;
+ let signer = cfg::signer(&cfg, ui::askpass)?;
+ let signer_id = SignerIdentity::new(&signer, &repo, &cfg, &id_path)?;
+ let meta = metadata::Drop::from_tip(&repo, &drop_ref)?;
+
+ let s = EditState {
+ repo,
+ id_path,
+ signer,
+ signer_id,
+ drop_ref,
+ meta,
+ };
+
+ match args.cmd {
+ None => s.edit_drop(args.message),
+ Some(Cmd::Mirrors) => s.edit_mirrors(args.message),
+ Some(Cmd::Alternates) => s.edit_alternates(args.message),
+ }
+}
+
+struct EditState<S> {
+ repo: git2::Repository,
+ id_path: Vec<git2::Repository>,
+ signer: S,
+ signer_id: SignerIdentity,
+ drop_ref: Refname,
+ meta: GitDrop,
+}
+
+impl<S: Signer + 'static> EditState<S> {
+ fn edit_drop(mut self, message: Option<String>) -> cmd::Result<Output> {
+ let GitDrop {
+ hash: parent_hash,
+ signed: metadata::Signed { signed: parent, .. },
+ } = self.meta;
+
+ ensure!(
+ self.signer_id.can_edit_drop(&parent),
+ "signer identity not allowed to edit the drop metadata"
+ );
+
+ let mut meta: metadata::Drop = edit_metadata(Editable::from(parent.clone()))?.try_into()?;
+ if meta.canonicalise()? == parent.canonicalise()? {
+ info!("Document unchanged");
+ cmd::abort!();
+ }
+ meta.prev = Some(parent_hash);
+
+ let signed = Metadata::drop(&meta).sign(iter::once(&mut self.signer as &mut dyn Signer))?;
+
+ let mut tx = refs::Transaction::new(&self.repo)?;
+ let drop_ref = tx.lock_ref(self.drop_ref)?;
+
+ let parent = self
+ .repo
+ .find_reference(drop_ref.name())?
+ .peel_to_commit()?;
+ let parent_tree = parent.tree()?;
+ let mut root = self.repo.treebuilder(Some(&parent_tree))?;
+ patches::Record::remove_from(&mut root)?;
+
+ let mut ids = self
+ .repo
+ .treebuilder(get_tree(&self.repo, &root, "ids")?.as_ref())?;
+ let identities = meta
+ .roles
+ .ids()
+ .into_iter()
+ .map(|id| find_id(&self.repo, &self.id_path, &id).map(|signed| (id, signed)))
+ .collect::<Result<Vec<_>, _>>()?;
+ for (iid, id) in identities {
+ let iid = iid.to_string();
+ let mut tb = self
+ .repo
+ .treebuilder(get_tree(&self.repo, &ids, &iid)?.as_ref())?;
+ metadata::identity::fold_to_tree(&self.repo, &mut tb, id)?;
+ ids.insert(&iid, tb.write()?, git2::FileMode::Tree.into())?;
+ }
+ root.insert("ids", ids.write()?, git2::FileMode::Tree.into())?;
+
+ root.insert(
+ META_FILE_DROP,
+ json::to_blob(&self.repo, &signed)?,
+ git2::FileMode::Blob.into(),
+ )?;
+ let tree = self.repo.find_tree(root.write()?)?;
+
+ let msg = message.map(Ok).unwrap_or_else(|| {
+ edit_commit_message(&self.repo, drop_ref.name(), &parent_tree, &tree)
+ })?;
+ let commit = git::commit_signed(&mut self.signer, &self.repo, msg, &tree, &[&parent])?;
+ drop_ref.set_target(commit, "it: metadata edit");
+
+ tx.commit()?;
+
+ Ok(Output {
+ repo: self.repo.path().to_owned(),
+ refname: drop_ref.into(),
+ commit,
+ })
+ }
+
+ pub fn edit_mirrors(mut self, message: Option<String>) -> cmd::Result<Output> {
+ ensure!(
+ self.signer_id.can_edit_mirrors(&self.meta.signed.signed),
+ "signer identity not allowed to edit mirrors"
+ );
+
+ let prev = metadata::Mirrors::from_tip(&self.repo, &self.drop_ref)
+ .map(|m| m.signed.signed)
+ .or_else(|e| {
+ if e.is::<metadata::git::error::FileNotFound>() {
+ Ok(Default::default())
+ } else {
+ Err(e)
+ }
+ })?;
+ let prev_canonical = prev.canonicalise()?;
+ let meta = edit_metadata(prev)?;
+ if meta.canonicalise()? == prev_canonical {
+ info!("Document unchanged");
+ cmd::abort!();
+ }
+
+ let signed =
+ Metadata::mirrors(meta).sign(iter::once(&mut self.signer as &mut dyn Signer))?;
+
+ let mut tx = refs::Transaction::new(&self.repo)?;
+ let drop_ref = tx.lock_ref(self.drop_ref)?;
+
+ let parent = self
+ .repo
+ .find_reference(drop_ref.name())?
+ .peel_to_commit()?;
+ let parent_tree = parent.tree()?;
+ let mut root = self.repo.treebuilder(Some(&parent_tree))?;
+ patches::Record::remove_from(&mut root)?;
+ root.insert(
+ META_FILE_MIRRORS,
+ json::to_blob(&self.repo, &signed)?,
+ git2::FileMode::Blob.into(),
+ )?;
+ let tree = self.repo.find_tree(root.write()?)?;
+
+ let msg = message.map(Ok).unwrap_or_else(|| {
+ edit_commit_message(&self.repo, drop_ref.name(), &parent_tree, &tree)
+ })?;
+ let commit = git::commit_signed(&mut self.signer, &self.repo, msg, &tree, &[&parent])?;
+ drop_ref.set_target(commit, "it: mirrors edit");
+
+ tx.commit()?;
+
+ Ok(Output {
+ repo: self.repo.path().to_owned(),
+ refname: drop_ref.into(),
+ commit,
+ })
+ }
+
+ pub fn edit_alternates(mut self, message: Option<String>) -> cmd::Result<Output> {
+ ensure!(
+ self.signer_id.can_edit_mirrors(&self.meta.signed.signed),
+ "signer identity not allowed to edit alternates"
+ );
+
+ let prev = metadata::Alternates::from_tip(&self.repo, &self.drop_ref)
+ .map(|m| m.signed.signed)
+ .or_else(|e| {
+ if e.is::<metadata::git::error::FileNotFound>() {
+ Ok(Default::default())
+ } else {
+ Err(e)
+ }
+ })?;
+ let prev_canonical = prev.canonicalise()?;
+ let meta = edit_metadata(prev)?;
+ if meta.canonicalise()? == prev_canonical {
+ info!("Document unchanged");
+ cmd::abort!();
+ }
+
+ let signed =
+ Metadata::alternates(meta).sign(iter::once(&mut self.signer as &mut dyn Signer))?;
+
+ let mut tx = refs::Transaction::new(&self.repo)?;
+ let drop_ref = tx.lock_ref(self.drop_ref)?;
+
+ let parent = self
+ .repo
+ .find_reference(drop_ref.name())?
+ .peel_to_commit()?;
+ let parent_tree = parent.tree()?;
+ let mut root = self.repo.treebuilder(Some(&parent_tree))?;
+ patches::Record::remove_from(&mut root)?;
+ root.insert(
+ META_FILE_ALTERNATES,
+ json::to_blob(&self.repo, &signed)?,
+ git2::FileMode::Blob.into(),
+ )?;
+ let tree = self.repo.find_tree(root.write()?)?;
+
+ let msg = message.map(Ok).unwrap_or_else(|| {
+ edit_commit_message(&self.repo, drop_ref.name(), &parent_tree, &tree)
+ })?;
+ let commit = git::commit_signed(&mut self.signer, &self.repo, msg, &tree, &[&parent])?;
+ drop_ref.set_target(commit, "it: alternates edit");
+
+ tx.commit()?;
+
+ Ok(Output {
+ repo: self.repo.path().to_owned(),
+ refname: drop_ref.into(),
+ commit,
+ })
+ }
+}
+
+fn get_tree<'a>(
+ repo: &'a git2::Repository,
+ builder: &git2::TreeBuilder,
+ name: &str,
+) -> cmd::Result<Option<git2::Tree<'a>>> {
+ if let Some(entry) = builder.get(name)? {
+ return Ok(Some(
+ entry
+ .to_object(repo)?
+ .into_tree()
+ .map_err(|_| anyhow!("{name} is not a tree"))?,
+ ));
+ }
+
+ Ok(None)
+}
+
+struct SignerIdentity {
+ id: IdentityId,
+}
+
+impl SignerIdentity {
+ pub fn new<S: Signer>(
+ signer: &S,
+ repo: &git2::Repository,
+ cfg: &git2::Config,
+ id_path: &[git2::Repository],
+ ) -> cmd::Result<Self> {
+ let id =
+ cfg::git::identity(cfg)?.ok_or_else(|| anyhow!("signer identity not in gitconfig"))?;
+ let meta = find_id(repo, id_path, &id)?;
+ let keyid = metadata::KeyId::from(signer.ident());
+
+ ensure!(
+ meta.signed.keys.contains_key(&keyid),
+ "signing key {keyid} is not in identity {id}"
+ );
+
+ Ok(Self { id })
+ }
+
+ pub fn can_edit_drop(&self, parent: &metadata::Drop) -> bool {
+ parent.roles.root.ids.contains(&self.id)
+ }
+
+ pub fn can_edit_mirrors(&self, parent: &metadata::Drop) -> bool {
+ parent.roles.mirrors.ids.contains(&self.id)
+ }
+}
diff --git a/src/cmd/drop/init.rs b/src/cmd/drop/init.rs
new file mode 100644
index 0000000..b843255
--- /dev/null
+++ b/src/cmd/drop/init.rs
@@ -0,0 +1,194 @@
+// Copyright © 2022 Kim Altintop <kim@eagain.io>
+// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception
+
+use std::{
+ iter,
+ num::NonZeroUsize,
+ path::PathBuf,
+};
+
+use anyhow::{
+ anyhow,
+ ensure,
+};
+
+use super::{
+ find_id,
+ Common,
+ Editable,
+};
+use crate::{
+ cfg,
+ cmd::{
+ self,
+ args::Refname,
+ ui::{
+ self,
+ edit_metadata,
+ },
+ },
+ git::{
+ self,
+ if_not_found_none,
+ refs,
+ },
+ json,
+ metadata::{
+ self,
+ git::META_FILE_DROP,
+ Metadata,
+ },
+ patches::{
+ REF_HEADS_PATCHES,
+ REF_IT_PATCHES,
+ },
+};
+
+#[derive(Debug, clap::Args)]
+pub struct Init {
+ #[clap(flatten)]
+ common: Common,
+ /// A description for this drop instance, max. 128 characters
+ #[clap(long, value_parser, value_name = "STRING")]
+ description: metadata::drop::Description,
+ /// If the repository does not already exist, initialise it as non-bare
+ ///
+ /// A drop is usually initialised inside an already existing git repository,
+ /// or as a standalone drop repository. The latter is advisable for serving
+ /// over the network.
+ ///
+ /// When init is given a directory which does not already exist, it is
+ /// assumed that a standalone drop should be created, and thus the
+ /// repository is initialised as bare. This behaviour can be overridden
+ /// by --no-bare.
+ #[clap(long, value_parser)]
+ no_bare: bool,
+}
+
+#[derive(serde::Serialize)]
+pub struct Output {
+ repo: PathBuf,
+ #[serde(rename = "ref")]
+ refname: Refname,
+ #[serde(with = "crate::git::serde::oid")]
+ commit: git2::Oid,
+}
+
+pub fn init(args: Init) -> cmd::Result<Output> {
+ let Common { git_dir, id_path } = args.common;
+ let drop_ref: Refname = REF_IT_PATCHES.parse().unwrap();
+
+ let repo = git::repo::open_or_init(
+ git_dir,
+ git::repo::InitOpts {
+ bare: !args.no_bare,
+ description: "`it` drop",
+ initial_head: &drop_ref,
+ },
+ )?;
+
+ let mut tx = refs::Transaction::new(&repo)?;
+ let drop_ref = tx.lock_ref(drop_ref)?;
+ ensure!(
+ if_not_found_none(repo.refname_to_id(drop_ref.name()))?.is_none(),
+ "{} already exists",
+ drop_ref
+ );
+
+ let id_path = id_path.open_git();
+ git::add_alternates(&repo, &id_path)?;
+
+ let cfg = repo.config()?.snapshot()?;
+ let mut signer = cfg::signer(&cfg, ui::askpass)?;
+ let signer_id = {
+ let iid =
+ cfg::git::identity(&cfg)?.ok_or_else(|| anyhow!("signer identity not in gitconfig"))?;
+ let id = find_id(&repo, &id_path, &iid)?;
+ let keyid = metadata::KeyId::from(signer.ident());
+ ensure!(
+ id.signed.keys.contains_key(&keyid),
+ "signing key {keyid} is not in identity {iid}"
+ );
+
+ iid
+ };
+
+ let default = {
+ let default_role = metadata::drop::Role {
+ ids: [signer_id].into(),
+ threshold: NonZeroUsize::new(1).unwrap(),
+ };
+ let default_branch = cfg::git::default_branch(&cfg)?;
+
+ metadata::Drop {
+ spec_version: crate::SPEC_VERSION,
+ description: args.description,
+ prev: None,
+ custom: Default::default(),
+ roles: metadata::drop::Roles {
+ root: default_role.clone(),
+ snapshot: default_role.clone(),
+ mirrors: default_role.clone(),
+ branches: [(
+ default_branch,
+ metadata::drop::Annotated {
+ role: default_role,
+ description: metadata::drop::Description::try_from(
+ "the default branch".to_owned(),
+ )
+ .unwrap(),
+ },
+ )]
+ .into(),
+ },
+ }
+ };
+ let meta: metadata::Drop = edit_metadata(Editable::from(default))?.try_into()?;
+ ensure!(
+ meta.roles.root.ids.contains(&signer_id),
+ "signing identity {signer_id} is lacking the drop role required to sign the metadata"
+ );
+ let signed = Metadata::drop(&meta).sign(iter::once(&mut signer))?;
+
+ let mut root = repo.treebuilder(None)?;
+ let mut ids = repo.treebuilder(None)?;
+ let identities = meta
+ .roles
+ .ids()
+ .into_iter()
+ .map(|id| find_id(&repo, &id_path, &id).map(|signed| (id, signed)))
+ .collect::<Result<Vec<_>, _>>()?;
+ for (iid, id) in identities {
+ let iid = iid.to_string();
+ let mut tb = repo.treebuilder(None)?;
+ metadata::identity::fold_to_tree(&repo, &mut tb, id)?;
+ ids.insert(&iid, tb.write()?, git2::FileMode::Tree.into())?;
+ }
+ root.insert("ids", ids.write()?, git2::FileMode::Tree.into())?;
+ root.insert(
+ META_FILE_DROP,
+ json::to_blob(&repo, &signed)?,
+ git2::FileMode::Blob.into(),
+ )?;
+ let tree = repo.find_tree(root.write()?)?;
+ let msg = format!("Create drop '{}'", meta.description);
+ let commit = git::commit_signed(&mut signer, &repo, msg, &tree, &[])?;
+
+ if repo.is_bare() {
+ // Arrange refs to be `git-clone`-friendly
+ let heads_patches = tx.lock_ref(REF_HEADS_PATCHES.parse()?)?;
+ heads_patches.set_target(commit, "it: create");
+ drop_ref.set_symbolic_target(heads_patches.name().clone(), String::new());
+ repo.set_head(heads_patches.name())?;
+ } else {
+ drop_ref.set_target(commit, "it: create");
+ }
+
+ tx.commit()?;
+
+ Ok(Output {
+ repo: repo.path().to_owned(),
+ refname: drop_ref.into(),
+ commit,
+ })
+}
diff --git a/src/cmd/drop/serve.rs b/src/cmd/drop/serve.rs
new file mode 100644
index 0000000..7540d58
--- /dev/null
+++ b/src/cmd/drop/serve.rs
@@ -0,0 +1,140 @@
+// Copyright © 2022 Kim Altintop <kim@eagain.io>
+// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception
+
+use std::{
+ fs::File,
+ io::Read,
+ path::PathBuf,
+ str::FromStr,
+};
+
+use clap::ValueHint;
+use url::Url;
+
+use super::Common;
+use crate::{
+ cfg,
+ cmd::{
+ self,
+ args::Refname,
+ },
+ http,
+ patches::{
+ REF_IT_BUNDLES,
+ REF_IT_PATCHES,
+ REF_IT_SEEN,
+ },
+};
+
+#[derive(Debug, clap::Args)]
+pub struct Serve {
+ #[clap(flatten)]
+ common: Common,
+ /// The directory where to write the bundle to
+ ///
+ /// Unless this is an absolute path, it is treated as relative to $GIT_DIR.
+ #[clap(
+ long,
+ value_parser,
+ value_name = "DIR",
+ default_value_os_t = cfg::paths::bundles().to_owned(),
+ value_hint = ValueHint::DirPath,
+ )]
+ bundle_dir: PathBuf,
+ /// Ref prefix under which to store the refs contained in patch bundles
+ #[clap(
+ long,
+ value_parser,
+ value_name = "REF",
+ default_value_t = Refname::from_str(REF_IT_BUNDLES).unwrap()
+ )]
+ unbundle_prefix: Refname,
+ /// The refname anchoring the seen objects tree
+ #[clap(
+ long,
+ value_parser,
+ value_name = "REF",
+ default_value_t = Refname::from_str(REF_IT_SEEN).unwrap()
+ )]
+ seen_ref: Refname,
+ /// 'host:port' to listen on
+ #[clap(
+ long,
+ value_parser,
+ value_name = "HOST:PORT",
+ default_value = "127.0.0.1:8084"
+ )]
+ listen: String,
+ /// Number of threads to use for the server
+ ///
+ /// If not set, the number of available cores is used.
+ #[clap(long, value_parser, value_name = "INT")]
+ threads: Option<usize>,
+ /// PEM-encoded TLS certificate
+ ///
+ /// Requires 'tls-key'. If not set (the default), the server will not use
+ /// TLS.
+ #[clap(
+ long,
+ value_parser,
+ value_name = "FILE",
+ requires = "tls_key",
+ value_hint = ValueHint::FilePath
+ )]
+ tls_cert: Option<PathBuf>,
+ /// PEM-encoded TLS private key
+ ///
+ /// Requires 'tls-cert'. If not set (the default), the server will not use
+ /// TLS.
+ #[clap(
+ long,
+ value_parser,
+ value_name = "FILE",
+ requires = "tls_cert",
+ value_hint = ValueHint::FilePath
+ )]
+ tls_key: Option<PathBuf>,
+ /// IPFS API to publish received patch bundle to
+ #[clap(
+ long,
+ value_parser,
+ value_name = "URL",
+ value_hint = ValueHint::Url,
+ )]
+ ipfs_api: Option<Url>,
+}
+
+#[derive(serde::Serialize)]
+pub struct Output;
+
+pub fn serve(args: Serve) -> cmd::Result<Output> {
+ let tls = args
+ .tls_cert
+ .map(|cert_path| -> cmd::Result<http::SslConfig> {
+ let mut certificate = Vec::new();
+ let mut private_key = Vec::new();
+ File::open(cert_path)?.read_to_end(&mut certificate)?;
+ File::open(args.tls_key.expect("presence of 'tls-key' ensured by clap"))?
+ .read_to_end(&mut private_key)?;
+
+ Ok(http::SslConfig {
+ certificate,
+ private_key,
+ })
+ })
+ .transpose()?;
+
+ http::serve(
+ args.listen,
+ http::Options {
+ git_dir: args.common.git_dir,
+ bundle_dir: args.bundle_dir,
+ unbundle_prefix: args.unbundle_prefix.into(),
+ drop_ref: REF_IT_PATCHES.into(),
+ seen_ref: args.seen_ref.into(),
+ threads: args.threads,
+ tls,
+ ipfs_api: args.ipfs_api,
+ },
+ )
+}
diff --git a/src/cmd/drop/show.rs b/src/cmd/drop/show.rs
new file mode 100644
index 0000000..e3fdcfc
--- /dev/null
+++ b/src/cmd/drop/show.rs
@@ -0,0 +1,208 @@
+// Copyright © 2022 Kim Altintop <kim@eagain.io>
+// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception
+
+use std::{
+ collections::BTreeMap,
+ io,
+ path::PathBuf,
+};
+
+use anyhow::Context;
+
+use super::{
+ Common,
+ META_FILE_ALTERNATES,
+ META_FILE_MIRRORS,
+};
+use crate::{
+ cmd::{
+ self,
+ util::args::Refname,
+ FromGit as _,
+ GitAlternates,
+ GitDrop,
+ GitMirrors,
+ },
+ git,
+ metadata::{
+ self,
+ ContentHash,
+ IdentityId,
+ KeySet,
+ },
+ patches::REF_IT_PATCHES,
+};
+
+#[derive(Debug, clap::Args)]
+pub struct Show {
+ #[clap(flatten)]
+ common: Common,
+ /// Name of the git ref holding the drop metadata history
+ #[clap(
+ long = "drop",
+ value_parser,
+ value_name = "REF",
+ default_value_t = REF_IT_PATCHES.parse().unwrap(),
+ )]
+ drop_ref: Refname,
+}
+
+#[derive(serde::Serialize)]
+pub struct Output {
+ repo: PathBuf,
+ refname: Refname,
+ drop: Data<metadata::Drop>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ mirrors: Option<Data<metadata::Mirrors>>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ alternates: Option<Data<metadata::Alternates>>,
+}
+
+#[derive(serde::Serialize)]
+pub struct Data<T> {
+ hash: ContentHash,
+ status: Status,
+ json: T,
+}
+
+#[derive(serde::Serialize)]
+#[serde(rename_all = "UPPERCASE")]
+pub enum Status {
+ Verified,
+ #[serde(with = "crate::serde::display")]
+ Invalid(metadata::error::Verification),
+}
+
+impl From<Result<(), metadata::error::Verification>> for Status {
+ fn from(r: Result<(), metadata::error::Verification>) -> Self {
+ r.map(|()| Self::Verified).unwrap_or_else(Self::Invalid)
+ }
+}
+
+pub fn show(args: Show) -> cmd::Result<Output> {
+ let Common { git_dir, .. } = args.common;
+ let drop_ref = args.drop_ref;
+
+ let repo = git::repo::open(git_dir)?;
+
+ let GitDrop {
+ hash,
+ signed: metadata::Signed {
+ signed: drop,
+ signatures,
+ },
+ } = metadata::Drop::from_tip(&repo, &drop_ref)?;
+
+ let mut signer_cache = SignerCache::new(&repo, &drop_ref)?;
+ let status = drop
+ .verify(
+ &signatures,
+ cmd::find_parent(&repo),
+ find_signer(&mut signer_cache),
+ )
+ .into();
+
+ let mut mirrors = None;
+ let mut alternates = None;
+
+ let tree = repo.find_reference(&drop_ref)?.peel_to_commit()?.tree()?;
+ if let Some(entry) = tree.get_name(META_FILE_MIRRORS) {
+ let blob = entry.to_object(&repo)?.peel_to_blob()?;
+ let GitMirrors { hash, signed } = metadata::Mirrors::from_blob(&blob)?;
+ let status = drop
+ .verify_mirrors(&signed, find_signer(&mut signer_cache))
+ .into();
+
+ mirrors = Some(Data {
+ hash,
+ status,
+ json: signed.signed,
+ });
+ }
+
+ if let Some(entry) = tree.get_name(META_FILE_ALTERNATES) {
+ let blob = entry.to_object(&repo)?.peel_to_blob()?;
+ let GitAlternates { hash, signed } = metadata::Alternates::from_blob(&blob)?;
+ let status = drop
+ .verify_alternates(&signed, find_signer(&mut signer_cache))
+ .into();
+
+ alternates = Some(Data {
+ hash,
+ status,
+ json: signed.signed,
+ });
+ }
+
+ Ok(Output {
+ repo: repo.path().to_owned(),
+ refname: drop_ref,
+ drop: Data {
+ hash,
+ status,
+ json: drop,
+ },
+ mirrors,
+ alternates,
+ })
+}
+
+struct SignerCache<'a> {
+ repo: &'a git2::Repository,
+ root: git2::Tree<'a>,
+ keys: BTreeMap<IdentityId, KeySet<'static>>,
+}
+
+impl<'a> SignerCache<'a> {
+ pub(self) fn new(repo: &'a git2::Repository, refname: &Refname) -> git::Result<Self> {
+ let root = {
+ let id = repo
+ .find_reference(refname)?
+ .peel_to_tree()?
+ .get_name("ids")
+ .ok_or_else(|| {
+ git2::Error::new(
+ git2::ErrorCode::NotFound,
+ git2::ErrorClass::Tree,
+ "'ids' tree not found",
+ )
+ })?
+ .id();
+ repo.find_tree(id)?
+ };
+ let keys = BTreeMap::new();
+
+ Ok(Self { repo, root, keys })
+ }
+}
+
+fn find_signer<'a>(
+ cache: &'a mut SignerCache,
+) -> impl FnMut(&IdentityId) -> io::Result<KeySet<'static>> + 'a {
+ fn go(
+ repo: &git2::Repository,
+ root: &git2::Tree,
+ keys: &mut BTreeMap<IdentityId, KeySet<'static>>,
+ id: &IdentityId,
+ ) -> cmd::Result<KeySet<'static>> {
+ match keys.get(id) {
+ Some(keys) => Ok(keys.clone()),
+ None => {
+ let (id, verified) = metadata::identity::find_in_tree(repo, root, id)
+ .with_context(|| format!("identity {id} failed to verify"))?
+ .into_parts();
+ keys.insert(id, verified.keys.clone());
+ Ok(verified.keys)
+ },
+ }
+ }
+
+ |id| go(cache.repo, &cache.root, &mut cache.keys, id).map_err(as_io)
+}
+
+fn as_io<E>(e: E) -> io::Error
+where
+ E: Into<Box<dyn std::error::Error + Send + Sync>>,
+{
+ io::Error::new(io::ErrorKind::Other, e)
+}
diff --git a/src/cmd/drop/snapshot.rs b/src/cmd/drop/snapshot.rs
new file mode 100644
index 0000000..b1348d3
--- /dev/null
+++ b/src/cmd/drop/snapshot.rs
@@ -0,0 +1,20 @@
+// Copyright © 2022 Kim Altintop <kim@eagain.io>
+// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception
+
+use crate::{
+ cmd::{
+ self,
+ patch,
+ },
+ patches,
+};
+
+#[derive(Debug, clap::Args)]
+pub struct Snapshot {
+ #[clap(flatten)]
+ common: patch::Common,
+}
+
+pub fn snapshot(Snapshot { common }: Snapshot) -> cmd::Result<patches::Record> {
+ patch::create(patch::Kind::Snapshot { common })
+}
diff --git a/src/cmd/drop/unbundle.rs b/src/cmd/drop/unbundle.rs
new file mode 100644
index 0000000..a9c9f77
--- /dev/null
+++ b/src/cmd/drop/unbundle.rs
@@ -0,0 +1,93 @@
+// Copyright © 2022 Kim Altintop <kim@eagain.io>
+// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception
+
+use std::{
+ collections::BTreeMap,
+ path::PathBuf,
+};
+
+use anyhow::anyhow;
+use clap::ValueHint;
+
+use crate::{
+ cmd,
+ git::{
+ self,
+ if_not_found_none,
+ refs,
+ Refname,
+ },
+ patches::{
+ self,
+ iter::dropped,
+ Bundle,
+ REF_IT_BUNDLES,
+ REF_IT_PATCHES,
+ },
+ paths,
+};
+
+// TODO:
+//
+// - require drop metadata verification
+// - abort if existing ref would be set to a different target (or --force)
+// - honour snapshots
+//
+
+#[derive(Debug, clap::Args)]
+pub struct Unbundle {
+ #[clap(from_global)]
+ git_dir: PathBuf,
+ /// The directory where to write the bundle to
+ ///
+ /// Unless this is an absolute path, it is treated as relative to $GIT_DIR.
+ #[clap(
+ long,
+ value_parser,
+ value_name = "DIR",
+ default_value_os_t = paths::bundles().to_owned(),
+ value_hint = ValueHint::DirPath,
+ )]
+ bundle_dir: PathBuf,
+ /// The drop history to find the topic in
+ #[clap(value_parser)]
+ drop: Option<String>,
+}
+
+#[derive(serde::Serialize)]
+pub struct Output {
+ updated: BTreeMap<Refname, git::serde::oid::Oid>,
+}
+
+pub fn unbundle(args: Unbundle) -> cmd::Result<Output> {
+ let repo = git::repo::open(&args.git_dir)?;
+ let bundle_dir = if args.bundle_dir.is_relative() {
+ repo.path().join(args.bundle_dir)
+ } else {
+ args.bundle_dir
+ };
+ let drop = match args.drop {
+ Some(rev) => if_not_found_none(repo.resolve_reference_from_short_name(&rev))?
+ .ok_or_else(|| anyhow!("no ref matching {rev} found"))?
+ .name()
+ .ok_or_else(|| anyhow!("invalid drop"))?
+ .to_owned(),
+ None => REF_IT_PATCHES.to_owned(),
+ };
+
+ let odb = repo.odb()?;
+ let mut tx = refs::Transaction::new(&repo)?;
+ let mut up = BTreeMap::new();
+ for rec in dropped::records_rev(&repo, &drop) {
+ let rec = rec?;
+ let bundle = Bundle::from_stored(&bundle_dir, rec.bundle_info().as_expect())?;
+ bundle.packdata()?.index(&odb)?;
+ let updated = patches::unbundle(&odb, &mut tx, REF_IT_BUNDLES, &rec)?;
+ for (name, oid) in updated {
+ up.insert(name, oid.into());
+ }
+ }
+ tx.commit()?;
+
+ Ok(Output { updated: up })
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 13:48:49 +0000