Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,8 @@ Options:
-C, --clean Delete local files after a successful upload
-V, --verbose Print uploader, object key, and URL to stderr
-w, --wait Do not exit after upload until the user presses a key
-n, --no-log Disable writing upgit.log (history.log is still written)
-n, --no-log Disable writing upgit.log and history.log
-j, --jobs <N> Concurrent uploads (1 = serial) [default: 1]
--application-path <PATH> Directory that owns config.toml / upgit.toml, history.log, and upgit.log
--version Print version
-h, --help Print help
Expand Down Expand Up @@ -318,7 +319,7 @@ The config file is searched in this order:
5. `~/.config/upgitrc`
6. `config.toml` and `upgit.toml` next to the binary

`--application-path` changes the binary-dir lookup. After each upload, `history.log` (and `upgit.log` unless `--no-log`) is written in that directory.
`--application-path` changes the binary-dir lookup. After each upload, `history.log` and `upgit.log` are written in that directory unless `--no-log`. Uploads run one at a time unless `--jobs` is set.

### Config via Environment Variables

Expand Down
2 changes: 1 addition & 1 deletion crates/upgit-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@ pub use key_policy::{KeyPolicy, KeyPolicyError};
pub use link_policy::LinkPolicy;
pub use locator::{Locator, PublicUrl};
pub use object_key::{ObjectKey, ObjectKeyError};
pub use publisher::{PublishError, Publisher};
pub use publisher::{BatchPublisher, PublishError, Publisher};
pub use registry::{Registry, RegistryError};
pub use uploader::{UploadError, Uploader};
130 changes: 130 additions & 0 deletions crates/upgit-core/src/publisher.rs

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

注意相当多的uploader实际上禁止并发,例如 Github。剩余的也都有限流。默认应该不并发,用户显式指定才能并;发。

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 43e13dc. Default concurrency is 1 (serial). Parallel only if the user passes --jobs N. GitHub Contents stays serial unless they opt in.

Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Mutex;
use std::thread;
use std::time::SystemTime;

use thiserror::Error;
Expand All @@ -23,6 +26,23 @@ pub struct Publisher {
linker: LinkPolicy,
}

/// Default is serial: many uploaders (GitHub Contents especially) forbid or
/// rate-limit concurrent PUTs. Callers opt in with [`BatchPublisher::with_concurrency`].
const DEFAULT_UPLOAD_CONCURRENCY: usize = 1;

type Slot = Mutex<Option<Result<(Locator, PublicUrl), PublishError>>>;

/// Sends one `upload` message per artifact.
///
/// Default concurrency is 1 (serial). `with_concurrency(n)` with n > 1 bounds
/// in-flight work. Results stay in input order. After the first failure, further
/// artifacts are not started; in-flight messages still finish. The error is the
/// failed artifact with the lowest input index.
pub struct BatchPublisher<'a> {
publisher: &'a Publisher,
concurrency: usize,
}

impl Publisher {
pub fn new(namer: KeyPolicy, linker: LinkPolicy) -> Self {
Self { namer, linker }
Expand Down Expand Up @@ -50,3 +70,113 @@ impl Publisher {
Ok((locator, url))
}
}

impl<'a> BatchPublisher<'a> {
pub fn new(publisher: &'a Publisher) -> Self {
Self {
publisher,
concurrency: DEFAULT_UPLOAD_CONCURRENCY,
}
}

pub fn with_concurrency(self, concurrency: usize) -> Self {
Self {
concurrency: concurrency.max(1),
..self
}
}

pub fn run(
&self,
uploader: &dyn Uploader,
artifacts: &[Artifact],
at: SystemTime,
) -> Result<Vec<(Locator, PublicUrl)>, PublishError> {
if artifacts.is_empty() {
return Ok(Vec::new());
}
if self.concurrency <= 1 || artifacts.len() == 1 {
return self.publish_serial(uploader, artifacts, at);
}
self.publish_parallel(uploader, artifacts, at)
}

fn publish_serial(
&self,
uploader: &dyn Uploader,
artifacts: &[Artifact],
at: SystemTime,
) -> Result<Vec<(Locator, PublicUrl)>, PublishError> {
let mut out = Vec::with_capacity(artifacts.len());
for artifact in artifacts {
out.push(self.publisher.publish_with_raw(uploader, artifact, at)?);
}
Ok(out)
}

fn publish_parallel(
&self,
uploader: &dyn Uploader,
artifacts: &[Artifact],
at: SystemTime,
) -> Result<Vec<(Locator, PublicUrl)>, PublishError> {
let n = artifacts.len();
let workers = self.concurrency.min(n);
let next = AtomicUsize::new(0);
let failed = AtomicBool::new(false);
let slots: Vec<Slot> = (0..n).map(|_| Mutex::new(None)).collect();

thread::scope(|scope| {
for _ in 0..workers {
scope.spawn(|| loop {
if failed.load(Ordering::Relaxed) {
break;
}
let i = next.fetch_add(1, Ordering::Relaxed);
if i >= n {
break;
}
if failed.load(Ordering::Relaxed) {
break;
}
let result = self.publisher.publish_with_raw(uploader, &artifacts[i], at);
if result.is_err() {
failed.store(true, Ordering::Relaxed);
}
*slots[i].lock().unwrap_or_else(|e| e.into_inner()) = Some(result);
});
}
});

Self::collect_slots(slots)
}

fn collect_slots(slots: Vec<Slot>) -> Result<Vec<(Locator, PublicUrl)>, PublishError> {
let n = slots.len();
let mut out = Vec::with_capacity(n);
let mut first_err = None;
let mut missing = false;
for slot in slots {
match slot.into_inner().unwrap_or_else(|e| e.into_inner()) {
Some(Ok(value)) => {
if first_err.is_none() {
out.push(value);
}
}
Some(Err(err)) => {
if first_err.is_none() {
first_err = Some(err);
}
}
None => missing = true,
}
}
if let Some(err) = first_err {
return Err(err);
}
if missing {
return Err(UploadError::message("upload did not finish").into());
}
Ok(out)
}
}
4 changes: 2 additions & 2 deletions crates/upgit-core/src/registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use crate::uploader::Uploader;
/// Runtime lookup of uploaders by id. The binary must not `match` on id.
#[derive(Default)]
pub struct Registry {
uploaders: HashMap<String, Box<dyn Uploader>>,
uploaders: HashMap<String, Box<dyn Uploader + Send + Sync>>,
}

#[derive(Debug, Error)]
Expand All @@ -21,7 +21,7 @@ impl Registry {
Self::default()
}

pub fn register(&mut self, id: impl Into<String>, uploader: Box<dyn Uploader>) {
pub fn register(&mut self, id: impl Into<String>, uploader: Box<dyn Uploader + Send + Sync>) {
self.uploaders.insert(id.into(), uploader);
}

Expand Down
4 changes: 3 additions & 1 deletion crates/upgit-core/src/uploader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ use crate::locator::Locator;
use crate::object_key::ObjectKey;

/// An object that accepts one message: put this artifact at this key.
pub trait Uploader {
///
/// `Send + Sync` so a batch sender can deliver that message from several threads.
pub trait Uploader: Send + Sync {
fn upload(&self, artifact: &Artifact, key: &ObjectKey) -> Result<Locator, UploadError>;
}

Expand Down
114 changes: 113 additions & 1 deletion crates/upgit-core/tests/upload_message.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ use std::sync::Mutex;
use std::time::{Duration, UNIX_EPOCH};

use upgit_core::{
Artifact, KeyPolicy, LinkPolicy, Locator, ObjectKey, Publisher, UploadError, Uploader,
Artifact, BatchPublisher, KeyPolicy, LinkPolicy, Locator, ObjectKey, Publisher, UploadError,
Uploader,
};

fn noon() -> std::time::SystemTime {
Expand Down Expand Up @@ -53,3 +54,114 @@ fn publisher_asks_uploader_with_the_computed_key() {
"https://cdn.example.com/2022/01/logo_1643630400.png"
);
}

struct BatchRecorder {
received: Mutex<Vec<(String, String)>>,
}

impl Uploader for BatchRecorder {
fn upload(&self, artifact: &Artifact, key: &ObjectKey) -> Result<Locator, UploadError> {
self.received
.lock()
.expect("lock")
.push((artifact.file_name().to_string(), key.as_str().to_string()));
Ok(Locator::new(format!(
"https://cdn.example.com/{}",
key.as_str()
)))
}
}

fn pngs(n: usize) -> Vec<Artifact> {
(0..n)
.map(|i| {
Artifact::from_name_and_size(&format!("f{i}.png"), 1024, Some(5 * 1024 * 1024))
.expect("artifact")
})
.collect()
}

#[test]
fn batch_publisher_sends_one_upload_per_artifact_and_keeps_order() {
let artifacts = pngs(12);
let uploader = BatchRecorder {
received: Mutex::new(Vec::new()),
};
let publisher = Publisher::new(KeyPolicy::keep_original_in("x"), LinkPolicy::identity());
let urls = BatchPublisher::new(&publisher)
.with_concurrency(4)
.run(&uploader, &artifacts, noon())
.expect("batch");

assert_eq!(urls.len(), 12);
for (i, (_raw, url)) in urls.iter().enumerate() {
assert_eq!(url.as_str(), format!("https://cdn.example.com/x/f{i}.png"));
}
let mut received = uploader.received.lock().expect("lock").clone();
received.sort();
let mut expected: Vec<_> = (0..12)
.map(|i| (format!("f{i}.png"), format!("x/f{i}.png")))
.collect();
expected.sort();
assert_eq!(received, expected);
}

struct FailOn {
name: String,
seen: Mutex<Vec<String>>,
}

impl Uploader for FailOn {
fn upload(&self, artifact: &Artifact, key: &ObjectKey) -> Result<Locator, UploadError> {
self.seen
.lock()
.expect("lock")
.push(artifact.file_name().to_string());
if artifact.file_name() == self.name {
return Err(UploadError::message("boom"));
}
std::thread::sleep(Duration::from_millis(30));
Ok(Locator::new(format!("https://ex/{}", key.as_str())))
}
}

#[test]
fn batch_publisher_defaults_to_serial() {
let artifacts = pngs(8);
let uploader = FailOn {
name: "f0.png".to_string(),
seen: Mutex::new(Vec::new()),
};
let publisher = Publisher::new(KeyPolicy::keep_original_in("x"), LinkPolicy::identity());
let err = BatchPublisher::new(&publisher)
.run(&uploader, &artifacts, noon())
.expect_err("must fail");
assert!(err.to_string().contains("boom"));
assert_eq!(
*uploader.seen.lock().expect("lock"),
vec!["f0.png".to_string()],
"default concurrency is 1: stop at the first error without starting the rest"
);
}

#[test]
fn batch_publisher_fails_the_run_on_first_error() {
let artifacts = pngs(20);
let uploader = FailOn {
name: "f0.png".to_string(),
seen: Mutex::new(Vec::new()),
};
let publisher = Publisher::new(KeyPolicy::keep_original_in("x"), LinkPolicy::identity());
let err = BatchPublisher::new(&publisher)
.with_concurrency(4)
.run(&uploader, &artifacts, noon())
.expect_err("must fail");
assert!(err.to_string().contains("boom"));
let seen = uploader.seen.lock().expect("lock");
assert!(!seen.is_empty());
assert!(
seen.len() < artifacts.len(),
"must not start every remaining file after a failure, started {}",
seen.len()
);
}
31 changes: 16 additions & 15 deletions crates/upgit/src/app.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ use std::error::Error;
use std::path::{Path, PathBuf};
use std::time::SystemTime;

use upgit::{application_dir, record_history, record_upload_log, Cli};
use upgit_core::{KeyPolicy, Publisher, Registry, RegistryError};
use upgit::{application_dir, Cli, History};
use upgit_core::{BatchPublisher, KeyPolicy, Publisher, Registry, RegistryError};
use upgit_uploaders::{AppConfig, HostCatalog};

use crate::emitter::Emitter;
Expand Down Expand Up @@ -70,10 +70,19 @@ impl App {
let mut intake = Intake::from_cli(cli, self.size_limit)?;
let artifacts = intake.collect()?;
let now = SystemTime::now();
let app_dir = application_dir(cli.application_path.as_deref());
let mut items = Vec::new();
for artifact in &artifacts {
let (raw, replaced) = self.publisher.publish_with_raw(uploader, artifact, now)?;
let history = if cli.no_log {
History::silent()
} else {
History::files(
application_dir(cli.application_path.as_deref()),
&self.uploader_id,
)
};
let published = BatchPublisher::new(&self.publisher)
.with_concurrency(cli.jobs)
.run(uploader, &artifacts, now)?;
let mut items = Vec::with_capacity(published.len());
for (artifact, (raw, replaced)) in artifacts.iter().zip(published) {
let shown = if cli.raw {
raw.as_str().to_string()
} else {
Expand All @@ -83,15 +92,7 @@ impl App {
if self.verbose {
eprintln!("key: {} url: {shown}", key.as_str());
}
record_history(app_dir.join("history.log"), raw.as_str(), replaced.as_str())?;
if !cli.no_log {
record_upload_log(
app_dir.join("upgit.log"),
&self.uploader_id,
key.as_str(),
&shown,
)?;
}
history.record(raw.as_str(), replaced.as_str(), key.as_str(), &shown)?;
items.push((shown, artifact.file_name().to_string()));
}
self.emitter.send(&items)?;
Expand Down
Loading
Loading