diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b53de9d..02e9361 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -47,9 +47,9 @@ jobs: - uses: actions/checkout@v4 - name: Install Azure CLI run: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - name: Start LocalStack (AWS S3) + - name: Start Floci (AWS S3) run: | - container=$(docker create -p 4566:4566 localstack/localstack:s3-latest) + container=$(docker create -p 4566:4566 floci/floci:latest) docker start $container sleep 5 docker logs $container @@ -61,7 +61,7 @@ jobs: docker logs $container - name: Create test AWS S3 bucket run: | - AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_DEFAULT_REGION=${DEFAULT_REGION:-$AWS_DEFAULT_REGION} aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket cloud-copy-test + AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket cloud-copy-test - name: Create test Azure Storage container run: | az storage container create --name cloud-copy-test --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1" diff --git a/CHANGELOG.md b/CHANGELOG.md index 28a67a6..91e9bf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +#### Fixed + +* Fixed `walk` implementation to filter "directory" blobs from Azure Blob ([#36](https://github.com/stjude-rust-labs/cloud-copy/pull/36)). + +#### Changed + +* Switched to `floci` for testing S3 in CI ([#36](https://github.com/stjude-rust-labs/cloud-copy/pull/36)). + ## 0.8.0 - 03-20-2026 #### Added diff --git a/README.md b/README.md index f87d11d..732d7f7 100644 --- a/README.md +++ b/README.md @@ -135,18 +135,18 @@ Automated tests rely on having the following cloud service emulators installed: * [Azurite](https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio%2Cblob-storage) for Azure Blob Storage. -* [Localstack](https://github.com/localstack/localstack) for AWS S3. +* [Floci](https://github.com/floci-io/floci) for AWS S3. Use the following command to run Azurite in a Docker container: ```bash -docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite -l /data --blobHost 0.0.0.0 --loose +docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite -l /data --blobHost 0.0.0.0 --loose --skipApiVersionCheck ``` -Use the following command to run Localstack in a Docker container: +Use the following command to run Floci in a Docker container: ```bash -docker run -p 4566:4566 localstack/localstack:s3-latest +docker run -p 4566:4566 floci/floci:latest ``` The tests expect a container/bucket with the name `cloud-copy-test` to be @@ -158,10 +158,10 @@ To create the container with Azurite, use the Azure CLI: az storage container create --name cloud-copy-test --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1" ``` -To create the bucket with Localstack, use the `awslocal` tool: +To create the bucket with Floci, use the `aws` tool: ```bash -awslocal s3api create-bucket --bucket cloud-copy-test +AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket cloud-copy-test ``` Finally, run the tests: diff --git a/src/backend/azure.rs b/src/backend/azure.rs index d9f5bbb..16b4fb1 100644 --- a/src/backend/azure.rs +++ b/src/backend/azure.rs @@ -746,6 +746,8 @@ impl StorageBackend for AzureBlobStorageBackend { pairs.append_pair("comp", "list"); // The prefix to use for listing blobs in the container. pairs.append_pair("prefix", &prefix); + // Only include files in the output. + pairs.append_pair("showonly", "files"); // Only return at most one result if we're returning the first only if first_only { diff --git a/src/backend/s3.rs b/src/backend/s3.rs index 0ba0e1b..c205106 100644 --- a/src/backend/s3.rs +++ b/src/backend/s3.rs @@ -42,9 +42,6 @@ use crate::streams::TransferStream; /// The root domain for AWS. const AWS_ROOT_DOMAIN: &str = "amazonaws.com"; -/// The root domain for localstack. -const LOCALSTACK_ROOT_DOMAIN: &str = "localhost.localstack.cloud"; - /// The maximum number of parts in an upload. const MAX_PARTS: u64 = 10000; @@ -555,8 +552,8 @@ impl StorageBackend for S3StorageBackend { // There must be at least two path segments !region.is_empty() && (domain.eq_ignore_ascii_case(AWS_ROOT_DOMAIN) - || (config.s3().use_localstack() - && domain.eq_ignore_ascii_case(LOCALSTACK_ROOT_DOMAIN))) + || (config.s3().use_floci() + && domain.eq_ignore_ascii_case("localhost"))) && url .path_segments() .map(|mut s| s.nth(1).is_some()) @@ -571,8 +568,8 @@ impl StorageBackend for S3StorageBackend { && !region.is_empty() && service.eq_ignore_ascii_case("s3") && (domain.eq_ignore_ascii_case(AWS_ROOT_DOMAIN) - || (config.s3().use_localstack() - && domain.eq_ignore_ascii_case(LOCALSTACK_ROOT_DOMAIN))) + || (config.s3().use_floci() + && domain.eq_ignore_ascii_case("localhost"))) && url .path_segments() .map(|mut s| s.next().is_some()) @@ -597,8 +594,8 @@ impl StorageBackend for S3StorageBackend { return Err(S3Error::InvalidScheme.into()); } - let (scheme, root, port) = if config.s3().use_localstack() { - ("http", LOCALSTACK_ROOT_DOMAIN, ":4566") + let (scheme, root, port) = if config.s3().use_floci() { + ("http", "localhost", ":4566") } else { ("https", AWS_ROOT_DOMAIN, "") }; diff --git a/src/config.rs b/src/config.rs index 67befd1..e84855b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -228,9 +228,9 @@ pub struct S3Config { /// If `None`, no authentication header will be put on requests. #[serde(default)] auth: Option, - /// Stores whether or not localstack is being used. + /// Stores whether or not Floci is being used. #[serde(default)] - use_localstack: bool, + use_floci: bool, } impl S3Config { @@ -265,15 +265,15 @@ impl S3Config { self } - /// Sets whether or not [localstack](https://github.com/localstack/localstack) is being used. + /// Sets whether or not [Floci](https://github.com/floci-io/floci) is being used. /// - /// The domain suffix is expected to be `localhost.localstack.cloud`. + /// The domain suffix is expected to be `localhost` for Floci requests. /// /// Any URLs that use the `s3` scheme will be rewritten to use that suffix. /// /// This setting is primarily intended for local testing. - pub fn with_use_localstack(mut self, use_localstack: bool) -> Self { - self.use_localstack = use_localstack; + pub fn with_use_floci(mut self, use_floci: bool) -> Self { + self.use_floci = use_floci; self } @@ -291,9 +291,9 @@ impl S3Config { self.auth.as_ref() } - /// Gets whether or not [localstack](https://github.com/localstack/localstack) is being used. - pub fn use_localstack(&self) -> bool { - self.use_localstack + /// Gets whether or not [Floci](https://github.com/floci-io/floci) is being used. + pub fn use_floci(&self) -> bool { + self.use_floci } } diff --git a/src/lib.rs b/src/lib.rs index 4a7556d..c41cea5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,7 @@ use std::borrow::Cow; use std::fmt; use std::io::ErrorKind; use std::ops::Deref; +use std::path::Component; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -114,6 +115,58 @@ fn sha256_hex_string(bytes: impl AsRef<[u8]>) -> String { hex::encode(hash.finalize()) } +/// Sorts a set of walk entries. +/// +/// Entries are expected to be resource subpaths, e.g. `foo` and `bar/baz`. +/// +/// Each entry is sorted by path segment. +/// +/// If an entry prefixes another, it means that a file would conflict with a +/// directory and an error is returned. +/// +/// Each entry may not contain a path segment that references the root or a +/// parent directory, otherwise an error is returned. +fn sort_walk_entries(url: &Url, entries: &mut [String]) -> Result<()> { + // Sort each entry by path components + entries.sort_by(|a, b| { + let a = Path::new(a); + let b = Path::new(b); + a.components().cmp(b.components()) + }); + + // Check for invalid or conflicting entries + let mut iter = entries.iter().peekable(); + while let Some(entry) = iter.next() { + // Ensure entry segments don't reference the root or parent directory + if Path::new(entry) + .components() + .any(|c| !matches!(c, Component::Normal(_) | Component::CurDir)) + { + return Err(Error::InvalidWalkEntry { + url: url.display().to_string(), + entry: entry.clone(), + }); + } + + // Ensure the next entry in the sorted list is not prefixed with this entry, + // otherwise a file conflicts with a directory + if let Some(next) = iter.peek() + && next + .strip_prefix(entry) + .and_then(|p| p.strip_prefix('/')) + .is_some() + { + return Err(Error::WalkEntryConflict { + url: url.display().to_string(), + first: entry.clone(), + second: (*next).clone(), + }); + } + } + + Ok(()) +} + trait DateTimeExt { /// Converts a [`DateTime`] to a HTTP date string. /// @@ -421,6 +474,24 @@ pub enum Error { /// The remote content was modified during a download. #[error("the remote content was modified during the download")] RemoteContentModified, + /// Failed to walk a URL due to an invalid entry. + #[error("failed to walk URL `{url}` due to invalid entry `{entry}`")] + InvalidWalkEntry { + /// The URL that failed to walk. + url: String, + /// The invalid entry + entry: String, + }, + /// Failed to walk a URL due to a conflict in its entries. + #[error("failed to walk URL `{url}` due to conflicting entries `{first}` and `{second}`")] + WalkEntryConflict { + /// The URL that failed to walk. + url: String, + /// The first conflicting entry. + first: String, + /// The second conflicting entry. + second: String, + }, /// Failed to create a directory. #[error("failed to create directory `{path}`: {error}", path = .path.display())] DirectoryCreationFailed { @@ -819,7 +890,8 @@ pub fn rewrite_url<'a>(config: &Config, url: &'a Url) -> Result> { /// Walks a given storage URL as if it were a directory. /// -/// Returns a list of relative paths from the given URL. +/// Returns a lexicographically-sorted list of relative paths from the given +/// URL. /// /// If the given storage URL is not a directory, an empty list is returned. pub async fn walk(config: Config, client: HttpClient, mut url: Url) -> Result> { @@ -829,7 +901,7 @@ pub async fn walk(config: Config, client: HttpClient, mut url: Url) -> Result Result Vec { vec![ // S3 URLs format!("s3://{TEST_BUCKET_NAME}/1/{test}").parse().unwrap(), - format!("http://s3.us-east-1.localhost.localstack.cloud:4566/{TEST_BUCKET_NAME}/2/{test}") + format!("http://s3.us-east-1.localhost:4566/{TEST_BUCKET_NAME}/2/{test}") .parse() .unwrap(), - format!("http://{TEST_BUCKET_NAME}.s3.us-east-1.localhost.localstack.cloud:4566/3/{test}") + format!("http://{TEST_BUCKET_NAME}.s3.us-east-1.localhost:4566/3/{test}") .parse() .unwrap(), @@ -184,7 +184,7 @@ fn azure_config() -> AzureConfig { fn s3_config() -> S3Config { S3Config::default() - .with_use_localstack(true) + .with_use_floci(true) .with_auth("test", "test") }