diff --git a/Cargo.lock b/Cargo.lock index 08b476fb09..c870d0f059 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,6 +53,19 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +[[package]] +name = "async-compression" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "atty" version = "0.2.14" @@ -1308,9 +1321,9 @@ dependencies = [ [[package]] name = "ostree" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9358905170a49b82e5baecb879bb944c42736a4b02074a40286a7b20665af381" +checksum = "c9ea1a013ad73775e461259248dc92de413577edcfabaaecb6ad6adba44acf78" dependencies = [ "bitflags", "gio", @@ -1325,11 +1338,12 @@ dependencies = [ [[package]] name = "ostree-ext" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1e40c1a1b52ed5164b0fda610952438afff5162ff2dafa49e55c0a78698228d" +version = "0.4.0-alpha.0" +source = "git+https://github.com/cgwalters/ostree-rs-ext?branch=layered-import#709ec6fa920658c690a3f017738edf6ef6e52b92" dependencies = [ "anyhow", + "async-compression", + "bitflags", "bytes", "camino", "cjson", @@ -1338,7 +1352,9 @@ dependencies = [ "futures-util", "gvariant", "hex", + "hyper", "indicatif", + "lazy_static", "libc", "maplit", "nix", @@ -1347,6 +1363,8 @@ dependencies = [ "openssl", "ostree", "phf 0.9.0", + "pin-project", + "scopeguard", "serde", "serde_json", "structopt", @@ -1496,6 +1514,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.7" diff --git a/Cargo.toml b/Cargo.toml index be4016639a..c4f1f2967b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ nix = "0.22.1" openat = "0.1.21" openat-ext = "^0.2.2" os-release = "0.1.0" -ostree-ext = "0.3.0" +ostree-ext = "0.4.0-alpha.0" paste = "1.0" phf = { version = "0.10", features = ["macros"] } rand = "0.8.4" @@ -101,3 +101,7 @@ fedora-integration = [] sanitizers = [] default = [] + +[patch.crates-io] +ostree-ext = { git = "https://github.com/cgwalters/ostree-rs-ext", branch = "layered-import" } +#ostree-ext = { path = "../../ostreedev/ostree-rs-ext/lib" } \ No newline at end of file diff --git a/rust/src/lib.rs b/rust/src/lib.rs index bbf041ae49..8c2efc325d 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -137,19 +137,19 @@ pub mod ffi { /// cxx.rs currently requires types used as extern Rust types to be defined by the same crate /// that contains the bridge using them, so we redefine an `ContainerImport` struct here. pub(crate) struct ContainerImport { + pub changed: bool, pub ostree_commit: String, pub image_digest: String, } // sysroot_upgrade.rs extern "Rust" { - fn import_container( + fn pull_container( repo: Pin<&mut OstreeRepo>, cancellable: Pin<&mut GCancellable>, - imgref: String, + imgref: &str, + current_digest: &str, ) -> Result>; - - fn fetch_digest(imgref: String, cancellable: Pin<&mut GCancellable>) -> Result; } // core.rs diff --git a/rust/src/sysroot_upgrade.rs b/rust/src/sysroot_upgrade.rs index a46468d5c4..ad56cb9a2e 100644 --- a/rust/src/sysroot_upgrade.rs +++ b/rust/src/sysroot_upgrade.rs @@ -3,48 +3,102 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use crate::cxxrsutil::*; -use crate::ffi::ContainerImport; -use std::convert::TryInto; +use crate::ffi::{output_message, ContainerImport}; +use anyhow::Result; +use ostree::glib; +use ostree_container::store::LayeredImageImporter; +use ostree_container::store::PrepareResult; +use ostree_container::OstreeImageReference; +use ostree_ext::container as ostree_container; +use ostree_ext::ostree; +use std::convert::TryFrom; use std::pin::Pin; use tokio::runtime::Handle; +async fn pull_container_async( + repo: &ostree::Repo, + imgref: &OstreeImageReference, + current_digest: Option<&str>, +) -> Result { + let unchanged = || ContainerImport { + changed: false, + ostree_commit: "".to_string(), + image_digest: "".to_string(), + }; + output_message(&format!("Pulling manifest: {}", &imgref)); + match &imgref.sigverify { + ostree_container::SignatureSource::OstreeRemote(_) => { + let (_, digest) = ostree_container::fetch_manifest(&imgref).await?; + if Some(digest.as_str()) == current_digest { + return Ok(unchanged()); + } + output_message(&format!("Importing: {} (digest: {})", &imgref, &digest)); + let i = ostree_container::import(repo, imgref, None).await?; + Ok(ContainerImport { + changed: true, + ostree_commit: i.ostree_commit, + image_digest: i.image_digest, + }) + } + _ => { + let mut imp = LayeredImageImporter::new(repo, imgref).await?; + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(_) => return Ok(unchanged()), + PrepareResult::Ready(r) => r, + }; + let digest = prep.manifest_digest.clone(); + output_message(&format!("Importing: {} (digest: {})", &imgref, &digest)); + if prep.base_layer.commit.is_none() { + let size = glib::format_size(prep.base_layer.size()); + output_message(&format!( + "Downloading base layer: {} ({})", + prep.base_layer.digest(), + size + )); + } else { + output_message(&format!("Using base: {}", prep.base_layer.digest())); + } + // TODO add nice download progress + for layer in prep.layers.iter() { + if layer.commit.is_some() { + output_message(&format!("Using layer: {}", layer.digest())); + } else { + let size = glib::format_size(layer.size()); + output_message(&format!("Downloading layer: {} ({})", layer.digest(), size)); + } + } + let import = imp.import(prep).await?; + Ok(ContainerImport { + changed: true, + ostree_commit: import.commit, + image_digest: digest, + }) + } + } +} + /// Import ostree commit in container image using ostree-rs-ext's API. -pub(crate) fn import_container( +pub(crate) fn pull_container( mut repo: Pin<&mut crate::FFIOstreeRepo>, mut cancellable: Pin<&mut crate::FFIGCancellable>, - imgref: String, + imgref: &str, + current_digest: &str, ) -> CxxResult> { - let repo = repo.gobj_wrap(); - let cancellable = cancellable.gobj_wrap(); - let imgref = imgref.as_str().try_into()?; - - let imported = Handle::current().block_on(async { - crate::utils::run_with_cancellable( - async { ostree_ext::container::import(&repo, &imgref, None).await }, - &cancellable, - ) - .await - })?; - Ok(Box::new(ContainerImport { - ostree_commit: imported.ostree_commit, - image_digest: imported.image_digest, - })) -} - -/// Fetch the image digest for `imgref` using ostree-rs-ext's API. -pub(crate) fn fetch_digest( - imgref: String, - mut cancellable: Pin<&mut crate::FFIGCancellable>, -) -> CxxResult { - let imgref = imgref.as_str().try_into()?; + let repo = &repo.gobj_wrap(); let cancellable = cancellable.gobj_wrap(); + let current_digest = if current_digest.is_empty() { + None + } else { + Some(current_digest) + }; + let imgref = &OstreeImageReference::try_from(imgref)?; - let digest = Handle::current().block_on(async { + let r = Handle::current().block_on(async { crate::utils::run_with_cancellable( - async { ostree_ext::container::fetch_manifest_info(&imgref).await }, + async { pull_container_async(repo, imgref, current_digest).await }, &cancellable, ) .await })?; - Ok(digest.manifest_digest) + Ok(Box::new(r)) } diff --git a/src/daemon/rpmostree-sysroot-upgrader.cxx b/src/daemon/rpmostree-sysroot-upgrader.cxx index ac435f9297..434b0134be 100644 --- a/src/daemon/rpmostree-sysroot-upgrader.cxx +++ b/src/daemon/rpmostree-sysroot-upgrader.cxx @@ -450,26 +450,16 @@ rpmostree_sysroot_upgrader_pull_base (RpmOstreeSysrootUpgrader *self, if (override_commit) return glnx_throw (error, "Specifying commit overrides for container-image-reference type refspecs is not supported"); - if (strstr (refspec, "@")) - return glnx_throw (error, "Pinned to specific container image digest, cannot upgrade"); - - /* Check to see if the digest that `refspec` points to is the same before pulling a new container image. */ - const char *cur_digest = rpmostree_origin_get_container_image_reference_digest (self->original_origin); - if (cur_digest) + auto refspec_s = std::string(refspec); + const char *cur_digest_c = rpmostree_origin_get_container_image_reference_digest (self->original_origin); + auto cur_digest = std::string(cur_digest_c ?: ""); + auto import = rpmostreecxx::pull_container(*self->repo, *cancellable, refspec_s, cur_digest); + if (!import->changed) { - rpmostree_output_message ("Pulling manifest: %s", refspec); - auto new_digest = CXX_TRY_VAL(fetch_digest(std::string(refspec), *cancellable), error); - if (strcmp (new_digest.c_str(), cur_digest) == 0) - { - /* No new digest. */ - *out_changed = FALSE; - return TRUE; - } + /* No new digest. */ + *out_changed = FALSE; + return TRUE; } - - rpmostree_output_message ("Pulling: %s", refspec); - auto import = CXX_TRY_VAL(import_container(*self->repo, *cancellable, std::string(refspec)), error); - rpmostree_origin_set_container_image_reference_digest (self->original_origin, import->image_digest.c_str()); new_base_rev = strdup (import->ostree_commit.c_str()); break;