diff --git a/Cargo.lock b/Cargo.lock index 1032c2aa4..a3951dcbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -414,21 +414,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "async-tar" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6affe71e5b6180fb5eaf9e8127a243694baf6ae1120c199227167302f56c14b" -dependencies = [ - "filetime", - "futures-core", - "libc", - "redox_syscall 0.7.5", - "tokio", - "tokio-stream", - "xattr", -] - [[package]] name = "async-trait" version = "0.1.92" @@ -2171,6 +2156,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "docs_rs_crate_archive" +version = "0.1.0" +dependencies = [ + "anyhow", + "docs_rs_types", + "flate2", + "tar", + "tempfile", +] + [[package]] name = "docs_rs_crates_io" version = "0.1.0" @@ -2268,7 +2264,6 @@ name = "docs_rs_import_release" version = "0.6.0" dependencies = [ "anyhow", - "async-tar", "clap", "docs_rs_cargo_metadata", "docs_rs_context", @@ -2352,9 +2347,11 @@ dependencies = [ "chrono", "crates-index", "docs_rs_config", + "docs_rs_crate_archive", "docs_rs_env_vars", "docs_rs_types", "docs_rs_utils", + "futures-util", "http 1.5.0", "mime", "mockito", @@ -5979,7 +5976,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link", ] @@ -6628,15 +6625,6 @@ dependencies = [ "bitflags 2.13.1", ] -[[package]] -name = "redox_syscall" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" -dependencies = [ - "bitflags 2.13.1", -] - [[package]] name = "ref-cast" version = "1.0.27" diff --git a/Cargo.toml b/Cargo.toml index 923cabc16..f207f0812 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ bytes = "1.11.0" chrono = { version = "0.4.11", default-features = false, features = ["clock", "serde"] } clap = { version = "4.0.22", features = ["derive"] } crates-index = { version = "3.14.1", default-features = false } +flate2 = "1.1.5" futures-util = "0.3.5" http = "1.0.0" itertools = "0.15.0" @@ -62,6 +63,7 @@ serde_with = "3.4.0" slug = "0.1.1" sqlx = { version = "0.9", features = ["chrono", "postgres", "runtime-tokio", "sqlite"] } strum = { version = "0.28.0", features = ["derive"] } +tar = "0.4.46" tempfile = "3.1.0" test-case = "3.0.0" thiserror = "2.0.3" diff --git a/crates/bin/docs_rs_import_release/Cargo.toml b/crates/bin/docs_rs_import_release/Cargo.toml index 540ee7b71..7bfa4010c 100644 --- a/crates/bin/docs_rs_import_release/Cargo.toml +++ b/crates/bin/docs_rs_import_release/Cargo.toml @@ -9,7 +9,6 @@ description = "Import a successfully built release from docs.rs into a test depl [dependencies] anyhow = { workspace = true } -async-tar = { version = "0.6.0", default-features = false, features = ["runtime-tokio", "xattr"] } clap = { workspace = true } docs_rs_cargo_metadata = { path = "../../lib/docs_rs_cargo_metadata" } docs_rs_context = { path = "../../lib/docs_rs_context" } diff --git a/crates/bin/docs_rs_import_release/src/crates_io.rs b/crates/bin/docs_rs_import_release/src/crates_io.rs deleted file mode 100644 index 242f19bdb..000000000 --- a/crates/bin/docs_rs_import_release/src/crates_io.rs +++ /dev/null @@ -1,54 +0,0 @@ -use crate::common::download_to_temp_file; -use anyhow::{Result, bail}; -use async_tar::Archive; -use docs_rs_registry_api::RegistryApi; -use docs_rs_storage::compression::wrap_reader_for_decompression; -use docs_rs_types::{CompressionAlgorithm, KrateName, Version}; -use docs_rs_utils::spawn_blocking; -use std::path::{Path, PathBuf}; -use tokio::io; -use tracing::debug; - -#[derive(Debug)] -pub(crate) struct SourceDir { - _temp_dir: tempfile::TempDir, - pub(crate) source_path: PathBuf, -} - -impl AsRef for SourceDir { - fn as_ref(&self) -> &Path { - &self.source_path - } -} - -pub(crate) async fn download_and_extract_source( - registry: &RegistryApi, - name: &KrateName, - version: &Version, -) -> Result { - debug!("downloading source"); - let crate_archive = download_to_temp_file(registry.download_url(name, version)?).await?; - - let temp_dir = spawn_blocking(|| Ok(tempfile::tempdir()?)).await?; - - debug!("unpacking source archive"); - { - let mut file = io::BufReader::new(crate_archive); - let mut decompressed = wrap_reader_for_decompression(&mut file, CompressionAlgorithm::Gzip); - let archive = Archive::new(&mut decompressed); - archive.unpack(&temp_dir).await?; - } - - let source_path = temp_dir.path().join(format!("{name}-{version}")); - if !source_path.is_dir() { - bail!( - "broken crate archive, missing source directory {:?}", - source_path - ); - }; - - Ok(SourceDir { - source_path, - _temp_dir: temp_dir, - }) -} diff --git a/crates/bin/docs_rs_import_release/src/import.rs b/crates/bin/docs_rs_import_release/src/import.rs index 920d766ef..5f0303c0d 100644 --- a/crates/bin/docs_rs_import_release/src/import.rs +++ b/crates/bin/docs_rs_import_release/src/import.rs @@ -1,6 +1,5 @@ use crate::{ common::{DOCS_RS, download, download_to_temp_file}, - crates_io::download_and_extract_source, rustdoc::{download_static_files, find_static_paths, find_successful_build_targets}, rustdoc_status::fetch_rustdoc_status, }; @@ -99,15 +98,17 @@ async fn import_test_release_inner( build_id: BuildId, ) -> Result<()> { info!("download & inspect source from crates.io..."); - let source_dir = download_and_extract_source(registry_api, name, version).await?; + let source_dir = registry_api + .download_and_extract_source(name, version) + .await?; let cargo_metadata = spawn_blocking({ - let source_dir = source_dir.source_path.clone(); + let source_dir = source_dir.path().to_owned(); move || CargoMetadata::load_from_host_path(&source_dir) }) .await?; let docsrs_metadata = spawn_blocking({ - let source_dir = source_dir.source_path.clone(); + let source_dir = source_dir.path().to_owned(); move || Ok(Metadata::from_crate_root(&source_dir)?) }) .await?; diff --git a/crates/bin/docs_rs_import_release/src/main.rs b/crates/bin/docs_rs_import_release/src/main.rs index 302a6d28a..aab6de566 100644 --- a/crates/bin/docs_rs_import_release/src/main.rs +++ b/crates/bin/docs_rs_import_release/src/main.rs @@ -1,5 +1,4 @@ pub(crate) mod common; -pub(crate) mod crates_io; mod import; mod rustdoc; pub(crate) mod rustdoc_status; diff --git a/crates/lib/docs_rs_crate_archive/Cargo.toml b/crates/lib/docs_rs_crate_archive/Cargo.toml new file mode 100644 index 000000000..104926260 --- /dev/null +++ b/crates/lib/docs_rs_crate_archive/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "docs_rs_crate_archive" +version = "0.1.0" +license.workspace = true +repository.workspace = true +edition.workspace = true + +[features] +testing = ["dep:docs_rs_types"] + +[dependencies] +anyhow = { workspace = true } +docs_rs_types = { path = "../docs_rs_types", optional = true } +flate2 = { workspace = true } +tar = { workspace = true } +tempfile = { workspace = true } + +[dev-dependencies] +docs_rs_types = { path = "../docs_rs_types" } + +[lints] +workspace = true diff --git a/crates/lib/docs_rs_crate_archive/src/lib.rs b/crates/lib/docs_rs_crate_archive/src/lib.rs new file mode 100644 index 000000000..a5a787595 --- /dev/null +++ b/crates/lib/docs_rs_crate_archive/src/lib.rs @@ -0,0 +1,107 @@ +//! Read crate package archives. + +use anyhow::{Context as _, Result, bail}; +use flate2::read::GzDecoder; +use std::{ + fs, + io::Read, + path::{Path, PathBuf}, +}; + +/// A crate archive extracted into a temporary source directory. +/// +/// Keeping this value alive keeps the source directory alive. +#[derive(Debug)] +pub struct SourceDir { + _temporary: tempfile::TempDir, + source_dir: PathBuf, +} + +impl SourceDir { + /// Return the root directory of the unpacked crate source. + pub fn path(&self) -> &Path { + &self.source_dir + } +} + +impl AsRef for SourceDir { + fn as_ref(&self) -> &Path { + self.path() + } +} + +/// Gzip-decompress and unpack a `.crate` archive. +/// +/// The archive must contain exactly one top-level directory, which is returned as [`SourceDir`]. +pub fn unpack_crate_archive(archive: impl Read) -> Result { + let temporary = tempfile::tempdir().context("creating temporary source directory")?; + tar::Archive::new(GzDecoder::new(archive)) + .unpack(temporary.path()) + .context("extracting crate archive")?; + + let entries = fs::read_dir(temporary.path()) + .context("reading extracted crate archive")? + .collect::, _>>()?; + + let source_dir = match entries.as_slice() { + [entry] if entry.file_type()?.is_dir() => entry.path(), + _ => bail!( + "expected the crate archive to contain one root directory, found {} entries", + entries.len() + ), + }; + + Ok(SourceDir { + _temporary: temporary, + source_dir, + }) +} + +#[cfg(any(test, feature = "testing"))] +/// Test utilities for creating crate package archives. +pub mod testing { + use super::*; + use docs_rs_types::{KrateName, Version}; + use flate2::write::GzEncoder; + + /// Create a gzip-compressed crate archive from a crate source root. + /// + /// The archive contains `root` beneath a single `-` top-level directory. + pub fn create_source_tarball( + name: &KrateName, + version: &Version, + root: impl AsRef, + ) -> Result> { + let root = root.as_ref(); + let encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); + let mut archive = tar::Builder::new(encoder); + archive.append_dir_all(format!("{name}-{version}"), root)?; + Ok(archive.into_inner()?.finish()?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn unpacks_the_single_source_root() -> Result<()> { + let root = tempfile::tempdir()?; + fs::write( + root.path().join("Cargo.toml"), + "[package]\nname = \"krate\"\n", + )?; + let name = "krate".parse()?; + let version = "1.0.0".parse()?; + let archive = testing::create_source_tarball(&name, &version, &root)?; + + let source = unpack_crate_archive(Cursor::new(archive))?; + assert_eq!( + fs::read_to_string(source.path().join("Cargo.toml"))?, + "[package]\nname = \"krate\"\n" + ); + + Ok(()) + } +} diff --git a/crates/lib/docs_rs_registry_api/Cargo.toml b/crates/lib/docs_rs_registry_api/Cargo.toml index 9f1a9cab1..80835b1e4 100644 --- a/crates/lib/docs_rs_registry_api/Cargo.toml +++ b/crates/lib/docs_rs_registry_api/Cargo.toml @@ -6,7 +6,7 @@ repository = "https://github.com/rust-lang/docs.rs" edition = "2024" [features] -testing = ["dep:mockito", "dep:tempfile", "dep:tokio"] +testing = ["dep:mockito"] [dependencies] anyhow = { workspace = true } @@ -14,9 +14,11 @@ bon = { workspace = true } chrono = { workspace = true } crates-index = { workspace = true, default-features = false, features = ["sparse"] } docs_rs_config = { path = "../docs_rs_config" } +docs_rs_crate_archive = { path = "../docs_rs_crate_archive" } docs_rs_env_vars = { path = "../docs_rs_env_vars" } docs_rs_types = { path = "../docs_rs_types" } docs_rs_utils = { path = "../docs_rs_utils" } +futures-util = { workspace = true } http = { workspace = true } mime = { workspace = true } mockito = { workspace = true, optional = true } @@ -29,14 +31,15 @@ serde_urlencoded = "0.7.1" serde_with = { workspace = true } sqlx = { workspace = true } strum = { workspace = true } -tempfile = { workspace = true, optional = true } +tempfile = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, optional = true } +tokio = { workspace = true } tracing = { workspace = true } url = { workspace = true } [dev-dependencies] docs_rs_config = { path = "../docs_rs_config", features = ["testing"] } +docs_rs_crate_archive = { path = "../docs_rs_crate_archive", features = ["testing"] } docs_rs_types = { path = "../docs_rs_types", features = ["testing"] } mockito = { workspace = true } tempfile = { workspace = true } diff --git a/crates/lib/docs_rs_registry_api/src/api.rs b/crates/lib/docs_rs_registry_api/src/api.rs index 4632aa4e4..96f2382b0 100644 --- a/crates/lib/docs_rs_registry_api/src/api.rs +++ b/crates/lib/docs_rs_registry_api/src/api.rs @@ -7,13 +7,16 @@ use crate::{ }, }; use anyhow::Context as _; +use docs_rs_crate_archive::{SourceDir, unpack_crate_archive}; use docs_rs_types::{KrateName, Version}; -use docs_rs_utils::APP_USER_AGENT; +use docs_rs_utils::{APP_USER_AGENT, spawn_blocking}; +use futures_util::StreamExt as _; use reqwest::header::ACCEPT; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff}; use serde::{Deserialize, de::DeserializeOwned}; -use std::{fmt, io, path::Path}; +use std::{ffi::OsStr, fmt, io, path::Path}; +use tokio::io::{AsyncSeekExt as _, AsyncWriteExt as _}; use tracing::instrument; use url::Url; @@ -166,6 +169,52 @@ impl RegistryApi { .map_err(Into::into) } + /// Download and unpack the source archive for a crate version. + /// + /// The returned directory and all extracted files are deleted when [`SourceDir`] is dropped. + #[instrument(skip(self))] + pub async fn download_and_extract_source( + &self, + name: &KrateName, + version: &Version, + ) -> Result { + let response = self + .client + .get(self.download_url(name, version)?) + .send() + .await? + .error_for_status()?; + + let mut archive = + tokio::fs::File::from_std(spawn_blocking(|| Ok(tempfile::tempfile()?)).await?); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + archive + .write_all(&chunk?) + .await + .map_err(anyhow::Error::from)?; + } + archive.sync_all().await.map_err(anyhow::Error::from)?; + archive + .seek(io::SeekFrom::Start(0)) + .await + .map_err(anyhow::Error::from)?; + let archive = archive.into_std().await; + + let source_dir = spawn_blocking(move || unpack_crate_archive(archive)).await?; + + let expected_root = format!("{name}-{version}"); + if source_dir.path().file_name() != Some(OsStr::new(&expected_root)) { + return Err(anyhow::anyhow!( + "broken crate archive, missing source directory {:?}", + source_dir.path() + ) + .into()); + } + + Ok(source_dir) + } + /// Fetch all published versions of a crate from the sparse index. /// /// Returns `None` when the index has no entry for `name`. @@ -365,10 +414,12 @@ mod tests { }; use chrono::{DateTime, Utc}; use crates_index::IndexConfig; + use docs_rs_crate_archive::testing::create_source_tarball; use docs_rs_types::testing::{KRATE, V1, V2}; use reqwest::{StatusCode, header::CONTENT_TYPE}; use serde::Serialize; use test_case::test_case; + use tokio::fs; const CHECKSUM: &str = "0000000000000000000000000000000000000000000000000000000000000000"; @@ -904,4 +955,26 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_download_and_extract_source() -> anyhow::Result<()> { + let env = TestRegistry::new().await?; + let root = tempfile::tempdir()?; + fs::write( + root.path().join("Cargo.toml"), + "[package]\nname = \"krate\"\n", + ) + .await?; + let archive = spawn_blocking(move || create_source_tarball(&KRATE, &V1, &root)).await?; + env.mock_download(&KRATE, &V1, archive).await; + + let source_dir = env.api().download_and_extract_source(&KRATE, &V1).await?; + assert_eq!( + fs::read_to_string(source_dir.path().join("Cargo.toml")).await?, + "[package]\nname = \"krate\"\n" + ); + env.assert_mocks().await; + + Ok(()) + } } diff --git a/crates/lib/docs_rs_registry_api/src/lib.rs b/crates/lib/docs_rs_registry_api/src/lib.rs index 9083d4148..3dd6f30d4 100644 --- a/crates/lib/docs_rs_registry_api/src/lib.rs +++ b/crates/lib/docs_rs_registry_api/src/lib.rs @@ -10,6 +10,7 @@ pub mod testing; pub use api::RegistryApi; pub use config::Config; +pub use docs_rs_crate_archive::SourceDir; pub use error::Error; pub use models::{ CrateData, CrateOwner, OwnerKind, ReleaseData, Search, SearchCursor, SearchQuery, SearchSort, diff --git a/crates/lib/docs_rs_registry_api/src/testing/test_env.rs b/crates/lib/docs_rs_registry_api/src/testing/test_env.rs index c28ab3991..fd7699953 100644 --- a/crates/lib/docs_rs_registry_api/src/testing/test_env.rs +++ b/crates/lib/docs_rs_registry_api/src/testing/test_env.rs @@ -156,6 +156,26 @@ impl TestRegistry { inner.mocks.push(index_mock); } + /// Mock downloading a crate archive from the URL advertised by the sparse index. + pub async fn mock_download( + &self, + krate: &KrateName, + version: &docs_rs_types::Version, + archive: Vec, + ) { + let url = self.api.download_url(krate, version).unwrap(); + + let mut inner = self.inner.lock().await; + let mock = inner + .download_server + .mock("GET", url.path()) + .with_status(StatusCode::OK.as_u16().into()) + .with_body(archive) + .create_async() + .await; + inner.mocks.push(mock); + } + /// Create a custom mock for a registry API `GET` request. /// /// `path` may include a query string, whose URL-encoded pairs are matched independently of