migrate source-tarball-unpacking to library, use in registry api

Cargo.toml

@@ -37,6 +37,7 @@ bytes = "1.11.0"
 chrono = { version = "0.4.11", default-features = false, features = ["clock", "serde"] }
 clap = { version = "4.0.22", features = ["derive"] }
 crates-index = { version = "3.14.1", default-features = false }
+flate2 = "1.1.5"
 futures-util = "0.3.5"
 http = "1.0.0"
 itertools = "0.15.0"
@@ -62,6 +63,7 @@ serde_with = "3.4.0"
 slug = "0.1.1"
 sqlx = { version = "0.9", features = ["chrono", "postgres", "runtime-tokio", "sqlite"] }
 strum = { version = "0.28.0", features = ["derive"] }
+tar = "0.4.46"
 tempfile = "3.1.0"
 test-case = "3.0.0"
 thiserror = "2.0.3"

crates/bin/docs_rs_import_release/Cargo.toml

@@ -9,7 +9,6 @@ description = "Import a successfully built release from docs.rs into a test depl
 
 [dependencies]
 anyhow = { workspace = true }
-async-tar = { version = "0.6.0", default-features = false, features = ["runtime-tokio", "xattr"] }
 clap = { workspace = true }
 docs_rs_cargo_metadata = { path = "../../lib/docs_rs_cargo_metadata" }
 docs_rs_context = { path = "../../lib/docs_rs_context" }

crates/bin/docs_rs_import_release/src/import.rs

@@ -1,6 +1,5 @@
 use crate::{
     common::{DOCS_RS, download, download_to_temp_file},
-    crates_io::download_and_extract_source,
     rustdoc::{download_static_files, find_static_paths, find_successful_build_targets},
     rustdoc_status::fetch_rustdoc_status,
 };
@@ -99,15 +98,17 @@ async fn import_test_release_inner(
     build_id: BuildId,
 ) -> Result<()> {
     info!("download & inspect source from crates.io...");
-    let source_dir = download_and_extract_source(registry_api, name, version).await?;
+    let source_dir = registry_api
+        .download_and_extract_source(name, version)
+        .await?;
 
     let cargo_metadata = spawn_blocking({
-        let source_dir = source_dir.source_path.clone();
+        let source_dir = source_dir.path().to_owned();
         move || CargoMetadata::load_from_host_path(&source_dir)
     })
     .await?;
     let docsrs_metadata = spawn_blocking({
-        let source_dir = source_dir.source_path.clone();
+        let source_dir = source_dir.path().to_owned();
         move || Ok(Metadata::from_crate_root(&source_dir)?)
     })
     .await?;

crates/bin/docs_rs_import_release/src/main.rs

@@ -1,5 +1,4 @@
 pub(crate) mod common;
-pub(crate) mod crates_io;
 mod import;
 mod rustdoc;
 pub(crate) mod rustdoc_status;

crates/lib/docs_rs_crate_archive/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "docs_rs_crate_archive"
+version = "0.1.0"
+license.workspace = true
+repository.workspace = true
+edition.workspace = true
+
+[features]
+testing = ["dep:docs_rs_types"]
+
+[dependencies]
+anyhow = { workspace = true }
+docs_rs_types = { path = "../docs_rs_types", optional = true }
+flate2 = { workspace = true }
+tar = { workspace = true }
+tempfile = { workspace = true }
+
+[dev-dependencies]
+docs_rs_types = { path = "../docs_rs_types" }
+
+[lints]
+workspace = true

crates/lib/docs_rs_crate_archive/src/lib.rs

@@ -0,0 +1,107 @@
+//! Read crate package archives.
+
+use anyhow::{Context as _, Result, bail};
+use flate2::read::GzDecoder;
+use std::{
+    fs,
+    io::Read,
+    path::{Path, PathBuf},
+};
+
+/// A crate archive extracted into a temporary source directory.
+///
+/// Keeping this value alive keeps the source directory alive.
+#[derive(Debug)]
+pub struct SourceDir {
+    _temporary: tempfile::TempDir,
+    source_dir: PathBuf,
+}
+
+impl SourceDir {
+    /// Return the root directory of the unpacked crate source.
+    pub fn path(&self) -> &Path {
+        &self.source_dir
+    }
+}
+
+impl AsRef<Path> for SourceDir {
+    fn as_ref(&self) -> &Path {
+        self.path()
+    }
+}
+
+/// Gzip-decompress and unpack a `.crate` archive.
+///
+/// The archive must contain exactly one top-level directory, which is returned as [`SourceDir`].
+pub fn unpack_crate_archive(archive: impl Read) -> Result<SourceDir> {
+    let temporary = tempfile::tempdir().context("creating temporary source directory")?;
+    tar::Archive::new(GzDecoder::new(archive))
+        .unpack(temporary.path())
+        .context("extracting crate archive")?;
+
+    let entries = fs::read_dir(temporary.path())
+        .context("reading extracted crate archive")?
+        .collect::<std::result::Result<Vec<_>, _>>()?;
+
+    let source_dir = match entries.as_slice() {
+        [entry] if entry.file_type()?.is_dir() => entry.path(),
+        _ => bail!(
+            "expected the crate archive to contain one root directory, found {} entries",
+            entries.len()
+        ),
+    };
+
+    Ok(SourceDir {
+        _temporary: temporary,
+        source_dir,
+    })
+}
+
+#[cfg(any(test, feature = "testing"))]
+/// Test utilities for creating crate package archives.
+pub mod testing {
+    use super::*;
+    use docs_rs_types::{KrateName, Version};
+    use flate2::write::GzEncoder;
+
+    /// Create a gzip-compressed crate archive from a crate source root.
+    ///
+    /// The archive contains `root` beneath a single `<name>-<version>` top-level directory.
+    pub fn create_source_tarball(
+        name: &KrateName,
+        version: &Version,
+        root: impl AsRef<Path>,
+    ) -> Result<Vec<u8>> {
+        let root = root.as_ref();
+        let encoder = GzEncoder::new(Vec::new(), flate2::Compression::default());
+        let mut archive = tar::Builder::new(encoder);
+        archive.append_dir_all(format!("{name}-{version}"), root)?;
+        Ok(archive.into_inner()?.finish()?)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Cursor;
+
+    #[test]
+    fn unpacks_the_single_source_root() -> Result<()> {
+        let root = tempfile::tempdir()?;
+        fs::write(
+            root.path().join("Cargo.toml"),
+            "[package]\nname = \"krate\"\n",
+        )?;
+        let name = "krate".parse()?;
+        let version = "1.0.0".parse()?;
+        let archive = testing::create_source_tarball(&name, &version, &root)?;
+
+        let source = unpack_crate_archive(Cursor::new(archive))?;
+        assert_eq!(
+            fs::read_to_string(source.path().join("Cargo.toml"))?,
+            "[package]\nname = \"krate\"\n"
+        );
+
+        Ok(())
+    }
+}

crates/lib/docs_rs_registry_api/Cargo.toml

@@ -6,17 +6,19 @@ repository = "https://github.com/rust-lang/docs.rs"
 edition = "2024"
 
 [features]
-testing = ["dep:mockito", "dep:tempfile", "dep:tokio"]
+testing = ["dep:mockito"]
 
 [dependencies]
 anyhow = { workspace = true }
 bon = { workspace = true }
 chrono = { workspace = true }
 crates-index = { workspace = true, default-features = false, features = ["sparse"] }
 docs_rs_config = { path = "../docs_rs_config" }
+docs_rs_crate_archive = { path = "../docs_rs_crate_archive" }
 docs_rs_env_vars = { path = "../docs_rs_env_vars" }
 docs_rs_types = { path = "../docs_rs_types" }
 docs_rs_utils = { path = "../docs_rs_utils" }
+futures-util = { workspace = true }
 http = { workspace = true }
 mime = { workspace = true }
 mockito = { workspace = true, optional = true }
@@ -29,14 +31,15 @@ serde_urlencoded = "0.7.1"
 serde_with = { workspace = true }
 sqlx = { workspace = true }
 strum = { workspace = true }
-tempfile = { workspace = true, optional = true }
+tempfile = { workspace = true }
 thiserror = { workspace = true }
-tokio = { workspace = true, optional = true }
+tokio = { workspace = true }
 tracing = { workspace = true }
 url = { workspace = true }
 
 [dev-dependencies]
 docs_rs_config = { path = "../docs_rs_config", features = ["testing"] }
+docs_rs_crate_archive = { path = "../docs_rs_crate_archive", features = ["testing"] }
 docs_rs_types = { path = "../docs_rs_types", features = ["testing"] }
 mockito = { workspace = true }
 tempfile = { workspace = true }