From 2bdb63d842429da24fc5663bba7145857cf2826f Mon Sep 17 00:00:00 2001 From: pluveto Date: Tue, 1 Sep 2026 22:08:41 +0800 Subject: [PATCH 1/2] feat: add GitLab uploader for self-hosted instances First-class GitlabUploader talks to the Repository Files API (POST create, PUT when the file already exists). Printed URLs default to instance raw and accept public_base/host so private projects are not a GitLab raw 404. --- README.md | 1 + config.sample.toml | 16 +- config.sample.zh-CN.toml | 16 +- crates/upgit-uploaders/Cargo.toml | 2 +- crates/upgit-uploaders/src/catalog.rs | 4 + crates/upgit-uploaders/src/gitlab.rs | 236 ++++++++++++++++++++++++ crates/upgit-uploaders/src/install.rs | 46 ++++- crates/upgit-uploaders/src/lib.rs | 1 + crates/upgit-uploaders/tests/gitlab.rs | 184 ++++++++++++++++++ crates/upgit-uploaders/tests/install.rs | 61 +++++- docs/README.zh-CN.md | 1 + 11 files changed, 563 insertions(+), 5 deletions(-) create mode 100644 crates/upgit-uploaders/src/gitlab.rs create mode 100644 crates/upgit-uploaders/tests/gitlab.rs diff --git a/README.md b/README.md index 572c9b0..4839e2e 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ This is also useful with [Typora](https://support.typora.io/Upload-Image/#image- ### Supported Upload Extensions + Github ++ GitLab + S3 Compatible Storages + AWS S3 diff --git a/config.sample.toml b/config.sample.toml index 6a5db10..a4f09ef 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -2,7 +2,7 @@ # GitHub is the default. Copy one table into config.toml and set `default` to its name. # Packed/init config is GitHub-only: config.github.toml in the upgit repository. # HTTP hosts: a table named after a file in recipes/ needs no `type = "http"`. -# Built-in types (github, s3, aliyunoss, qcloudcos, upyun, qiniu) use their real fields. +# Built-in types (github, gitlab, s3, aliyunoss, qcloudcos, upyun, qiniu) use their real fields. # `default_uploader` is an alias of `default`. `rename` is an alias of `naming`. # `prefix` is an alias of Qiniu `public_base`. `replacements` is an alias of `link`. @@ -35,6 +35,20 @@ repo = "repo-name" # Branch to commit into. Must match the repository default branch. branch = "main" +# GitLab (self-hosted or gitlab.com). Private projects make raw URLs 404 unless public_base/host is set. +[uploaders.gitlab] +# GitLab instance origin (alias: host when `url` is omitted). No trailing slash; not the /api/v4 URL. +url = "https://gitlab.example.com" +# Project path with namespace (group/name). Encoded on the wire as group%2Fname. +project = "group/name" +# Personal access token with api or write_repository. Sent as PRIVATE-TOKEN. +token = "YOUR_GITLAB_TOKEN" +# Branch to commit into. Defaults to main if omitted. +branch = "main" +# Optional public URL prefix for printed links (CDN, reverse proxy, or a public project raw prefix). +# Private GitLab `/-/raw/` URLs 404 without this. Alias: host when `url` is also set. +# public_base = "https://cdn.example.com/" + # SM.MS image host. Token: https://sm.ms/home/apitoken [uploaders.smms] # SM.MS API token from the dashboard. diff --git a/config.sample.zh-CN.toml b/config.sample.zh-CN.toml index d17e724..bbefb19 100644 --- a/config.sample.zh-CN.toml +++ b/config.sample.zh-CN.toml @@ -1,7 +1,7 @@ # 仓库里的完整示例,不会被 `upgit init` 写入。 # 默认是 GitHub。把某一段 [uploaders.*] 拷进 config.toml,并把 `default` 改成该表名。 # HTTP 图床:表名与 recipes/ 里的文件名相同即可,不必写 `type = "http"`。 -# 内置类型(github、s3、aliyunoss、qcloudcos、upyun、qiniu)填写各自的真实字段。 +# 内置类型(github、gitlab、s3、aliyunoss、qcloudcos、upyun、qiniu)填写各自的真实字段。 # `default_uploader` 是 `default` 的别名。`rename` 是 `naming` 的别名。 # 七牛的 `prefix` 等价于 `public_base`。`replacements` 等价于 `link`。 @@ -34,6 +34,20 @@ repo = "仓库名" # 提交到的分支。必须与仓库默认分支一致。 branch = "main" +# GitLab(自建或 gitlab.com)。私有项目的 raw 直链会 404,除非设置 public_base/host。 +[uploaders.gitlab] +# GitLab 实例 origin(未写 url 时可用 host)。不要末尾斜杠,不要填 /api/v4。 +url = "https://gitlab.example.com" +# 带命名空间的项目路径(group/name)。请求里会编码成 group%2Fname。 +project = "group/name" +# 个人访问令牌,需要 api 或 write_repository。请求头是 PRIVATE-TOKEN。 +token = "YOUR_GITLAB_TOKEN" +# 提交到的分支。省略则默认 main。 +branch = "main" +# 可选的公开 URL 前缀(CDN、反代,或公开项目的 raw 前缀),用于打印出的链接。 +# 私有 GitLab 的 `/-/raw/` 直链没有这项会 404。若已写 url,host 也可表示该前缀。 +# public_base = "https://cdn.example.com/" + # SM.MS:https://sm.ms/home/apitoken [uploaders.smms] # SM.MS 控制台里的 API token。 diff --git a/crates/upgit-uploaders/Cargo.toml b/crates/upgit-uploaders/Cargo.toml index becc037..53020c6 100644 --- a/crates/upgit-uploaders/Cargo.toml +++ b/crates/upgit-uploaders/Cargo.toml @@ -5,7 +5,7 @@ edition.workspace = true license.workspace = true repository.workspace = true publish.workspace = true -description = "Uploader objects for upgit: GitHub, S3, OSS, COS, Upyun, Qiniu, HTTP recipes." +description = "Uploader objects for upgit: GitHub, GitLab, S3, OSS, COS, Upyun, Qiniu, HTTP recipes." [dependencies] base64 = "0.22" diff --git a/crates/upgit-uploaders/src/catalog.rs b/crates/upgit-uploaders/src/catalog.rs index 9ff12e4..422345e 100644 --- a/crates/upgit-uploaders/src/catalog.rs +++ b/crates/upgit-uploaders/src/catalog.rs @@ -19,6 +19,10 @@ impl HostCatalog { id: "github", title: "GitHub", }, + Host { + id: "gitlab", + title: "GitLab", + }, Host { id: "s3", title: "Amazon S3 (MinIO / Cloudflare R2 / Backblaze / Wasabi / DigitalOcean Spaces / Ceph / Flexify.IO / IBM Cloud Object Storage)", diff --git a/crates/upgit-uploaders/src/gitlab.rs b/crates/upgit-uploaders/src/gitlab.rs new file mode 100644 index 0000000..9bdcee0 --- /dev/null +++ b/crates/upgit-uploaders/src/gitlab.rs @@ -0,0 +1,236 @@ +use base64::engine::general_purpose::STANDARD; +use base64::Engine; +use serde::Serialize; +use upgit_core::{Artifact, Locator, ObjectKey, UploadError, Uploader}; + +use crate::util::{ + could_not_reach, host_of, join_host_path, json_string_field, looks_like_rate_limit, read_bytes, +}; + +#[derive(Serialize)] +struct FileBody<'a> { + branch: &'a str, + content: String, + encoding: &'a str, + commit_message: String, +} + +fn encode_file_body(branch: &str, file_name: &str, data: &[u8]) -> Result { + serde_json::to_string(&FileBody { + branch, + content: STANDARD.encode(data), + encoding: "base64", + commit_message: format!("upload {file_name} via upgit"), + }) + .map_err(|e| UploadError::message(e.to_string())) +} + +fn file_already_exists(status: u16, body: &str) -> bool { + status == 400 && body.to_ascii_lowercase().contains("already exists") +} + +/// RFC 3986 unreserved stay literal; everything else is `%HH` (uppercase). +/// GitLab needs `/` in project and file_path as `%2F`, not as extra path segments. +fn percent_encode(s: &str) -> String { + let mut out = String::new(); + for &b in s.as_bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char) + } + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +fn with_scheme(url: &str) -> String { + let t = url.trim(); + if t.starts_with("http://") || t.starts_with("https://") { + t.to_string() + } else { + format!("https://{t}") + } +} + +fn normalize_instance(url: &str) -> String { + with_scheme(url).trim_end_matches('/').to_string() +} + +fn normalize_public_base(base: &str) -> Option { + let t = with_scheme(base).trim_end_matches('/').to_string(); + if t.is_empty() || t == "https:" || t == "http:" { + None + } else { + Some(t) + } +} + +#[derive(Debug, Clone)] +pub struct GitlabConfig { + pub url: String, + pub project: String, + pub token: String, + pub branch: String, + pub public_base: Option, +} + +#[derive(Debug, Clone)] +pub struct GitlabUploader { + config: GitlabConfig, +} + +impl GitlabUploader { + pub fn new(mut config: GitlabConfig) -> Self { + if config.branch.trim().is_empty() { + config.branch = "main".to_string(); + } + config.url = normalize_instance(&config.url); + config.project = config.project.trim().trim_matches('/').to_string(); + config.public_base = config + .public_base + .and_then(|base| normalize_public_base(&base)); + Self { config } + } + + pub fn files_url(&self, key: &ObjectKey) -> String { + format!( + "{}/api/v4/projects/{}/repository/files/{}", + self.config.url, + percent_encode(&self.config.project), + percent_encode(key.as_str()), + ) + } + + pub fn locator_for(&self, key: &ObjectKey) -> Locator { + if let Some(base) = &self.config.public_base { + Locator::new(join_host_path(base, key.as_str())) + } else { + Locator::new(format!( + "{}/{}/-/raw/{}/{}", + self.config.url, + self.config.project, + self.config.branch, + key.as_str() + )) + } + } + + /// Map a GitLab Repository Files API status + body to a user-facing error. + /// Never dumps JSON. + pub fn explain(&self, status: u16, body: &str) -> UploadError { + let project = self.config.project.as_str(); + match status { + 401 => UploadError::new( + "GitLab", + "GitLab personal access token is invalid or expired (HTTP 401).", + format!( + "Create a token with the \"api\" or \"write_repository\" scope at {}/-/user_settings/personal_access_tokens and put it in [uploaders.gitlab] `token`.", + self.config.url + ), + Some(status), + ), + 429 => Self::rate_limit_error(status), + 403 if looks_like_rate_limit(body) => Self::rate_limit_error(status), + 403 => UploadError::new( + "GitLab", + format!("GitLab denied access to `{project}` (HTTP 403)."), + format!( + "The token in [uploaders.gitlab] `token` lacks access to `{project}`. Grant the \"api\" or \"write_repository\" scope and a Developer role on this project." + ), + Some(status), + ), + 404 => UploadError::new( + "GitLab", + format!("GitLab project `{project}` was not found (HTTP 404)."), + "Check [uploaders.gitlab] url, project, and token. A missing project, or a token without access, also returns 404. Private projects make GitLab raw URLs 404; set public_base or host to a CDN, reverse proxy, or public raw prefix.", + Some(status), + ), + 400 | 409 | 422 => UploadError::new( + "GitLab", + format!("GitLab cannot create or update that path (HTTP {status})."), + "Check [uploaders.gitlab] branch and the file path.", + Some(status), + ), + 500..=599 => UploadError::new( + "GitLab", + format!("GitLab is failing (HTTP {status})."), + "Retry later; this is a GitLab server error, not a config problem.", + Some(status), + ), + _ => { + let what = match json_string_field(body, "message") { + Some(msg) => format!("GitLab upload failed (HTTP {status}): {msg}"), + None => format!("GitLab upload failed (HTTP {status})."), + }; + UploadError::new( + "GitLab", + what, + "Verify [uploaders.gitlab] url, project, token, and branch. Private projects make GitLab raw URLs 404; set public_base or host.", + Some(status), + ) + } + } + } + + fn rate_limit_error(status: u16) -> UploadError { + UploadError::new( + "GitLab", + format!("GitLab rate limit exceeded (HTTP {status})."), + "Wait and retry, or use a token with a higher rate limit in [uploaders.gitlab] `token`.", + Some(status), + ) + } + + fn authorized(&self, req: ureq::Request) -> ureq::Request { + req.set("PRIVATE-TOKEN", &self.config.token) + .set("User-Agent", "upgit") + } + + fn create_or_update( + &self, + key: &ObjectKey, + artifact: &Artifact, + data: &[u8], + ) -> Result { + let body = encode_file_body(&self.config.branch, artifact.file_name(), data)?; + let url = self.files_url(key); + match self + .authorized(ureq::post(&url)) + .set("Content-Type", "application/json") + .send_string(&body) + { + Ok(_) => Ok(self.locator_for(key)), + Err(ureq::Error::Status(code, resp)) => { + let text = resp.into_string().unwrap_or_default(); + if file_already_exists(code, &text) { + return self.update_file(&url, &body, key); + } + Err(self.explain(code, &text)) + } + Err(e) => Err(could_not_reach("GitLab", host_of(&self.config.url), e)), + } + } + + fn update_file(&self, url: &str, body: &str, key: &ObjectKey) -> Result { + match self + .authorized(ureq::put(url)) + .set("Content-Type", "application/json") + .send_string(body) + { + Ok(_) => Ok(self.locator_for(key)), + Err(ureq::Error::Status(code, resp)) => { + let text = resp.into_string().unwrap_or_default(); + Err(self.explain(code, &text)) + } + Err(e) => Err(could_not_reach("GitLab", host_of(&self.config.url), e)), + } + } +} + +impl Uploader for GitlabUploader { + fn upload(&self, artifact: &Artifact, key: &ObjectKey) -> Result { + let data = read_bytes(artifact)?; + self.create_or_update(key, artifact, &data) + } +} diff --git a/crates/upgit-uploaders/src/install.rs b/crates/upgit-uploaders/src/install.rs index 4459336..934ce95 100644 --- a/crates/upgit-uploaders/src/install.rs +++ b/crates/upgit-uploaders/src/install.rs @@ -9,6 +9,7 @@ use upgit_core::{KeyPolicy, KeyPolicyError, Registry}; use crate::catalog::RecipeCatalog; use crate::cos::{CosConfig, CosUploader}; use crate::github::{GithubConfig, GithubUploader}; +use crate::gitlab::{GitlabConfig, GitlabUploader}; use crate::oss::{OssConfig, OssUploader}; use crate::qiniu::{QiniuConfig, QiniuUploader}; use crate::recipe::{HttpRecipe, HttpRecipeUploader, RecipeError}; @@ -27,7 +28,7 @@ pub enum InstallError { MissingField { id: String, field: String }, #[error("uploader `{id}` still has a static Qiniu upload token, which expires. Set access_key, secret_key, bucket, and public_base instead.")] ExpiredQiniuToken { id: String }, - #[error("unknown uploader type `{kind}` for `{id}`. Built-in types: github, s3, aliyunoss, qcloudcos, upyun, qiniu.")] + #[error("unknown uploader type `{kind}` for `{id}`. Built-in types: github, gitlab, s3, aliyunoss, qcloudcos, upyun, qiniu.")] UnknownKind { id: String, kind: String }, #[error("unknown http recipe `{recipe}` for `{id}`")] UnknownRecipe { id: String, recipe: String }, @@ -220,6 +221,10 @@ impl AppConfig { let uploader = github_from_profile(id, profile)?; registry.register(id.clone(), Box::new(uploader)); } + "gitlab" => { + let uploader = gitlab_from_profile(id, profile)?; + registry.register(id.clone(), Box::new(uploader)); + } "s3" => { let uploader = s3_from_profile(id, profile)?; registry.register(id.clone(), Box::new(uploader)); @@ -318,6 +323,12 @@ fn resolved_kind(id: &str, profile: &UploaderProfile) -> Result Result { + let url = optional_string(profile, "url") + .filter(|s| !is_placeholder(s)) + .or_else(|| optional_string(profile, "host").filter(|s| !is_placeholder(s))) + .ok_or_else(|| InstallError::MissingField { + id: id.to_string(), + field: "url".to_string(), + })?; + let public_base = optional_string(profile, "public_base") + .filter(|s| !is_placeholder(s)) + .or_else(|| { + // `host` is the printed URL only when `url` already names the instance. + if optional_string(profile, "url") + .filter(|s| !is_placeholder(s)) + .is_some() + { + optional_string(profile, "host").filter(|s| !is_placeholder(s)) + } else { + None + } + }); + Ok(GitlabUploader::new(GitlabConfig { + url, + project: require_string(id, profile, "project")?, + token: require_string(id, profile, "token")?, + branch: optional_string(profile, "branch").unwrap_or_default(), + public_base, + })) +} + fn s3_from_profile(id: &str, profile: &UploaderProfile) -> Result { Ok(S3Uploader::new(S3Config { region: require_string(id, profile, "region")?, diff --git a/crates/upgit-uploaders/src/lib.rs b/crates/upgit-uploaders/src/lib.rs index a041ee5..dba85fb 100644 --- a/crates/upgit-uploaders/src/lib.rs +++ b/crates/upgit-uploaders/src/lib.rs @@ -2,6 +2,7 @@ mod catalog; pub mod cos; mod form; pub mod github; +pub mod gitlab; mod install; pub mod oss; pub mod qiniu; diff --git a/crates/upgit-uploaders/tests/gitlab.rs b/crates/upgit-uploaders/tests/gitlab.rs new file mode 100644 index 0000000..f08297c --- /dev/null +++ b/crates/upgit-uploaders/tests/gitlab.rs @@ -0,0 +1,184 @@ +//! Locator, encoding, explain, and POST-then-PUT for GitlabUploader. + +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread; +use std::time::Duration; + +use upgit_core::{Artifact, ObjectKey, Uploader}; +use upgit_uploaders::gitlab::{GitlabConfig, GitlabUploader}; +use upgit_uploaders::HostCatalog; + +fn uploader(public_base: Option<&str>) -> GitlabUploader { + GitlabUploader::new(GitlabConfig { + url: "https://gitlab.example.com".into(), + project: "group/name".into(), + token: "tok".into(), + branch: String::new(), + public_base: public_base.map(str::to_string), + }) +} + +fn read_http_request(stream: &mut std::net::TcpStream) -> Vec { + let mut buf = Vec::new(); + let mut tmp = [0u8; 4096]; + loop { + match stream.read(&mut tmp) { + Ok(0) => break, + Ok(n) => buf.extend_from_slice(&tmp[..n]), + Err(_) => break, + } + let Some(header_end) = buf.windows(4).position(|w| w == b"\r\n\r\n") else { + continue; + }; + let headers = String::from_utf8_lossy(&buf[..header_end]); + let content_len = headers + .lines() + .find(|line| line.to_ascii_lowercase().starts_with("content-length:")) + .and_then(|line| line.split(':').nth(1)) + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(0); + let body_start = header_end + 4; + if buf.len() >= body_start + content_len { + break; + } + } + buf +} + +#[test] +fn catalog_lists_gitlab_after_github() { + let hosts = HostCatalog::all(); + assert_eq!(hosts[0].id, "github"); + assert_eq!(hosts[1].id, "gitlab"); + assert_eq!(hosts[1].title, "GitLab"); +} + +#[test] +fn locator_uses_instance_raw_url_without_public_base() { + let key = ObjectKey::parse("2024/01/a.png").expect("key"); + assert_eq!( + uploader(None).locator_for(&key).as_str(), + "https://gitlab.example.com/group/name/-/raw/main/2024/01/a.png" + ); +} + +#[test] +fn locator_uses_public_base_when_set() { + let key = ObjectKey::parse("2024/01/a.png").expect("key"); + assert_eq!( + uploader(Some("https://cdn.example.com/")) + .locator_for(&key) + .as_str(), + "https://cdn.example.com/2024/01/a.png" + ); +} + +#[test] +fn files_url_percent_encodes_project_and_path() { + let key = ObjectKey::parse("2024/01/a.png").expect("key"); + assert_eq!( + uploader(None).files_url(&key), + "https://gitlab.example.com/api/v4/projects/group%2Fname/repository/files/2024%2F01%2Fa.png" + ); +} + +#[test] +fn explain_401_has_no_documentation_url_or_raw_json() { + let err = uploader(None).explain( + 401, + r#"{"message":"401 Unauthorized","documentation_url":"https://docs.gitlab.com/ee/api/index.html"}"#, + ); + let text = err.to_string(); + assert!(text.contains("401"), "got {text}"); + assert!(text.contains("token"), "got {text}"); + assert!( + !text.contains("documentation_url"), + "dumped GitLab JSON: {text}" + ); + assert!(!text.contains('{'), "dumped GitLab JSON: {text}"); + assert!( + !text.contains("401 Unauthorized"), + "leaked GitLab body: {text}" + ); +} + +fn serve_create_then_update() -> (String, thread::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + listener.set_nonblocking(false).expect("blocking listener"); + let addr = listener.local_addr().expect("addr"); + let handle = thread::spawn(move || { + let mut requests = Vec::new(); + for i in 0..2 { + let (mut stream, _) = listener.accept().expect("accept"); + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let buf = read_http_request(&mut stream); + requests.push(String::from_utf8_lossy(&buf).into_owned()); + let (status, body) = if i == 0 { + ( + "400 Bad Request", + r#"{"message":"A file with this name already exists"}"#, + ) + } else { + ("200 OK", r#"{"file_path":"2024/01/a.png","branch":"main"}"#) + }; + let resp = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(resp.as_bytes()).expect("write"); + stream.flush().ok(); + } + requests + }); + (format!("http://{addr}"), handle) +} + +#[test] +fn upload_posts_then_puts_when_file_exists() { + let (origin, server) = serve_create_then_update(); + let uploader = GitlabUploader::new(GitlabConfig { + url: origin, + project: "group/name".into(), + token: "tok".into(), + branch: "main".into(), + public_base: Some("https://cdn.example.com/".into()), + }); + + let dir = std::env::temp_dir(); + let file = dir.join(format!("upgit-gitlab-upload-{}.png", std::process::id())); + std::fs::write(&file, b"\x89PNG-fake").expect("write fixture"); + let artifact = Artifact::from_path(&file, None).expect("artifact"); + let key = ObjectKey::parse("2024/01/a.png").expect("key"); + + let locator = uploader.upload(&artifact, &key).expect("upload"); + assert_eq!(locator.as_str(), "https://cdn.example.com/2024/01/a.png"); + + let _ = std::fs::remove_file(&file); + let requests = server.join().expect("server"); + assert_eq!(requests.len(), 2, "expected POST then PUT"); + assert!( + requests[0].starts_with("POST "), + "first request should be create, got:\n{}", + requests[0] + ); + assert!( + requests[1].starts_with("PUT "), + "second request should be update, got:\n{}", + requests[1] + ); + for req in &requests { + assert!( + req.contains("PRIVATE-TOKEN: tok"), + "missing PRIVATE-TOKEN:\n{req}" + ); + assert!( + req.contains("/api/v4/projects/group%2Fname/repository/files/2024%2F01%2Fa.png"), + "project/path must stay percent-encoded:\n{req}" + ); + assert!( + req.contains("\"encoding\":\"base64\""), + "body must send base64 encoding:\n{req}" + ); + } +} diff --git a/crates/upgit-uploaders/tests/install.rs b/crates/upgit-uploaders/tests/install.rs index 5032483..0a1fa32 100644 --- a/crates/upgit-uploaders/tests/install.rs +++ b/crates/upgit-uploaders/tests/install.rs @@ -1,5 +1,5 @@ use upgit_core::Registry; -use upgit_uploaders::AppConfig; +use upgit_uploaders::{AppConfig, InstallError}; #[test] fn install_registers_qiniu_and_http_from_config() { @@ -36,3 +36,62 @@ token = "tok" assert!(msg.contains("qiniu"), "got {msg}"); assert!(msg.contains("smms"), "got {msg}"); } + +#[test] +fn install_registers_gitlab_from_profile_without_type() { + let cfg = AppConfig::from_toml( + r#" +default = "gitlab" + +[uploaders.gitlab] +url = "https://gitlab.example.com" +project = "group/name" +token = "tok" +branch = "main" +public_base = "https://cdn.example.com/" +"#, + ) + .expect("parse"); + let mut registry = Registry::new(); + cfg.install_into(&mut registry).expect("install"); + registry.get("gitlab").expect("gitlab registered"); +} + +#[test] +fn install_infers_gitlab_from_token_project_host() { + let cfg = AppConfig::from_toml( + r#" +[uploaders.corp] +host = "https://gitlab.corp.example" +project = "group/name" +token = "tok" +"#, + ) + .expect("parse"); + let mut registry = Registry::new(); + cfg.install_into(&mut registry).expect("install"); + registry.get("corp").expect("corp registered as gitlab"); +} + +#[test] +fn unknown_kind_lists_gitlab_among_built_ins() { + let cfg = AppConfig::from_toml( + r#" +[uploaders.webdav] +type = "webdav" +"#, + ) + .expect("parse"); + let mut registry = Registry::new(); + let err = cfg.install_into(&mut registry).expect_err("unknown kind"); + let msg = err.to_string(); + match err { + InstallError::UnknownKind { id, kind } => { + assert_eq!(id, "webdav"); + assert_eq!(kind, "webdav"); + } + other => panic!("expected UnknownKind, got {other}"), + } + assert!(msg.contains("gitlab"), "got {msg}"); + assert!(msg.contains("github"), "got {msg}"); +} diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index f957f06..bbb88d6 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -29,6 +29,7 @@ ### 上传扩展 + Github ++ GitLab + S3 兼容存储 + AWS S3 + MinIO From 008d218b1f7aab70137068a4304c21e35d86842c Mon Sep 17 00:00:00 2001 From: pluveto Date: Tue, 1 Sep 2026 22:36:28 +0800 Subject: [PATCH 2/2] fix: rewrite GitLab uploader after review Move URL encoding and origin normalization into util. GitLab install infers only from type or table id; host aliases url only. Drop the low-value GitLab tests. --- config.sample.toml | 6 +- config.sample.zh-CN.toml | 6 +- crates/upgit-uploaders/src/gitlab.rs | 99 +++++-------- crates/upgit-uploaders/src/install.rs | 35 ++--- crates/upgit-uploaders/src/util.rs | 34 +++++ crates/upgit-uploaders/tests/gitlab.rs | 184 ------------------------ crates/upgit-uploaders/tests/install.rs | 61 +------- 7 files changed, 82 insertions(+), 343 deletions(-) delete mode 100644 crates/upgit-uploaders/tests/gitlab.rs diff --git a/config.sample.toml b/config.sample.toml index a4f09ef..7a92aea 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -35,9 +35,9 @@ repo = "repo-name" # Branch to commit into. Must match the repository default branch. branch = "main" -# GitLab (self-hosted or gitlab.com). Private projects make raw URLs 404 unless public_base/host is set. +# GitLab (self-hosted or gitlab.com). Private projects make raw URLs 404 unless public_base is set. [uploaders.gitlab] -# GitLab instance origin (alias: host when `url` is omitted). No trailing slash; not the /api/v4 URL. +# GitLab instance origin (alias: host). No trailing slash; not the /api/v4 URL. url = "https://gitlab.example.com" # Project path with namespace (group/name). Encoded on the wire as group%2Fname. project = "group/name" @@ -46,7 +46,7 @@ token = "YOUR_GITLAB_TOKEN" # Branch to commit into. Defaults to main if omitted. branch = "main" # Optional public URL prefix for printed links (CDN, reverse proxy, or a public project raw prefix). -# Private GitLab `/-/raw/` URLs 404 without this. Alias: host when `url` is also set. +# Private GitLab `/-/raw/` URLs 404 without this. # public_base = "https://cdn.example.com/" # SM.MS image host. Token: https://sm.ms/home/apitoken diff --git a/config.sample.zh-CN.toml b/config.sample.zh-CN.toml index bbefb19..67726e5 100644 --- a/config.sample.zh-CN.toml +++ b/config.sample.zh-CN.toml @@ -34,9 +34,9 @@ repo = "仓库名" # 提交到的分支。必须与仓库默认分支一致。 branch = "main" -# GitLab(自建或 gitlab.com)。私有项目的 raw 直链会 404,除非设置 public_base/host。 +# GitLab(自建或 gitlab.com)。私有项目的 raw 直链会 404,除非设置 public_base。 [uploaders.gitlab] -# GitLab 实例 origin(未写 url 时可用 host)。不要末尾斜杠,不要填 /api/v4。 +# GitLab 实例 origin(别名 host)。不要末尾斜杠,不要填 /api/v4。 url = "https://gitlab.example.com" # 带命名空间的项目路径(group/name)。请求里会编码成 group%2Fname。 project = "group/name" @@ -45,7 +45,7 @@ token = "YOUR_GITLAB_TOKEN" # 提交到的分支。省略则默认 main。 branch = "main" # 可选的公开 URL 前缀(CDN、反代,或公开项目的 raw 前缀),用于打印出的链接。 -# 私有 GitLab 的 `/-/raw/` 直链没有这项会 404。若已写 url,host 也可表示该前缀。 +# 私有 GitLab 的 `/-/raw/` 直链没有这项会 404。 # public_base = "https://cdn.example.com/" # SM.MS:https://sm.ms/home/apitoken diff --git a/crates/upgit-uploaders/src/gitlab.rs b/crates/upgit-uploaders/src/gitlab.rs index 9bdcee0..1dd6e5f 100644 --- a/crates/upgit-uploaders/src/gitlab.rs +++ b/crates/upgit-uploaders/src/gitlab.rs @@ -4,7 +4,8 @@ use serde::Serialize; use upgit_core::{Artifact, Locator, ObjectKey, UploadError, Uploader}; use crate::util::{ - could_not_reach, host_of, join_host_path, json_string_field, looks_like_rate_limit, read_bytes, + could_not_reach, host_of, http_origin, join_host_path, json_string_field, + looks_like_rate_limit, percent_encode, read_bytes, }; #[derive(Serialize)] @@ -15,57 +16,6 @@ struct FileBody<'a> { commit_message: String, } -fn encode_file_body(branch: &str, file_name: &str, data: &[u8]) -> Result { - serde_json::to_string(&FileBody { - branch, - content: STANDARD.encode(data), - encoding: "base64", - commit_message: format!("upload {file_name} via upgit"), - }) - .map_err(|e| UploadError::message(e.to_string())) -} - -fn file_already_exists(status: u16, body: &str) -> bool { - status == 400 && body.to_ascii_lowercase().contains("already exists") -} - -/// RFC 3986 unreserved stay literal; everything else is `%HH` (uppercase). -/// GitLab needs `/` in project and file_path as `%2F`, not as extra path segments. -fn percent_encode(s: &str) -> String { - let mut out = String::new(); - for &b in s.as_bytes() { - match b { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(b as char) - } - _ => out.push_str(&format!("%{b:02X}")), - } - } - out -} - -fn with_scheme(url: &str) -> String { - let t = url.trim(); - if t.starts_with("http://") || t.starts_with("https://") { - t.to_string() - } else { - format!("https://{t}") - } -} - -fn normalize_instance(url: &str) -> String { - with_scheme(url).trim_end_matches('/').to_string() -} - -fn normalize_public_base(base: &str) -> Option { - let t = with_scheme(base).trim_end_matches('/').to_string(); - if t.is_empty() || t == "https:" || t == "http:" { - None - } else { - Some(t) - } -} - #[derive(Debug, Clone)] pub struct GitlabConfig { pub url: String, @@ -85,23 +35,15 @@ impl GitlabUploader { if config.branch.trim().is_empty() { config.branch = "main".to_string(); } - config.url = normalize_instance(&config.url); + config.url = http_origin(&config.url); config.project = config.project.trim().trim_matches('/').to_string(); config.public_base = config .public_base - .and_then(|base| normalize_public_base(&base)); + .map(|base| http_origin(&base)) + .filter(|s| !s.is_empty()); Self { config } } - pub fn files_url(&self, key: &ObjectKey) -> String { - format!( - "{}/api/v4/projects/{}/repository/files/{}", - self.config.url, - percent_encode(&self.config.project), - percent_encode(key.as_str()), - ) - } - pub fn locator_for(&self, key: &ObjectKey) -> Locator { if let Some(base) = &self.config.public_base { Locator::new(join_host_path(base, key.as_str())) @@ -143,7 +85,7 @@ impl GitlabUploader { 404 => UploadError::new( "GitLab", format!("GitLab project `{project}` was not found (HTTP 404)."), - "Check [uploaders.gitlab] url, project, and token. A missing project, or a token without access, also returns 404. Private projects make GitLab raw URLs 404; set public_base or host to a CDN, reverse proxy, or public raw prefix.", + "Check [uploaders.gitlab] url, project, and token. A missing project, or a token without access, also returns 404. Private projects make GitLab raw URLs 404; set public_base to a CDN, reverse proxy, or public raw prefix.", Some(status), ), 400 | 409 | 422 => UploadError::new( @@ -166,13 +108,36 @@ impl GitlabUploader { UploadError::new( "GitLab", what, - "Verify [uploaders.gitlab] url, project, token, and branch. Private projects make GitLab raw URLs 404; set public_base or host.", + "Verify [uploaders.gitlab] url, project, token, and branch. Private projects make GitLab raw URLs 404; set public_base.", Some(status), ) } } } + fn files_url(&self, key: &ObjectKey) -> String { + format!( + "{}/api/v4/projects/{}/repository/files/{}", + self.config.url, + percent_encode(&self.config.project), + percent_encode(key.as_str()), + ) + } + + fn encode_file_body(&self, file_name: &str, data: &[u8]) -> Result { + serde_json::to_string(&FileBody { + branch: &self.config.branch, + content: STANDARD.encode(data), + encoding: "base64", + commit_message: format!("upload {file_name} via upgit"), + }) + .map_err(|e| UploadError::message(e.to_string())) + } + + fn file_already_exists(status: u16, body: &str) -> bool { + status == 400 && body.to_ascii_lowercase().contains("already exists") + } + fn rate_limit_error(status: u16) -> UploadError { UploadError::new( "GitLab", @@ -193,7 +158,7 @@ impl GitlabUploader { artifact: &Artifact, data: &[u8], ) -> Result { - let body = encode_file_body(&self.config.branch, artifact.file_name(), data)?; + let body = self.encode_file_body(artifact.file_name(), data)?; let url = self.files_url(key); match self .authorized(ureq::post(&url)) @@ -203,7 +168,7 @@ impl GitlabUploader { Ok(_) => Ok(self.locator_for(key)), Err(ureq::Error::Status(code, resp)) => { let text = resp.into_string().unwrap_or_default(); - if file_already_exists(code, &text) { + if Self::file_already_exists(code, &text) { return self.update_file(&url, &body, key); } Err(self.explain(code, &text)) diff --git a/crates/upgit-uploaders/src/install.rs b/crates/upgit-uploaders/src/install.rs index 934ce95..ce6bb71 100644 --- a/crates/upgit-uploaders/src/install.rs +++ b/crates/upgit-uploaders/src/install.rs @@ -323,10 +323,7 @@ fn resolved_kind(id: &str, profile: &UploaderProfile) -> Result Result { - let url = optional_string(profile, "url") - .filter(|s| !is_placeholder(s)) - .or_else(|| optional_string(profile, "host").filter(|s| !is_placeholder(s))) - .ok_or_else(|| InstallError::MissingField { - id: id.to_string(), - field: "url".to_string(), - })?; - let public_base = optional_string(profile, "public_base") - .filter(|s| !is_placeholder(s)) - .or_else(|| { - // `host` is the printed URL only when `url` already names the instance. - if optional_string(profile, "url") - .filter(|s| !is_placeholder(s)) - .is_some() - { - optional_string(profile, "host").filter(|s| !is_placeholder(s)) - } else { - None - } - }); Ok(GitlabUploader::new(GitlabConfig { - url, + url: optional_string(profile, "url") + .or_else(|| optional_string(profile, "host")) + .filter(|s| !is_placeholder(s)) + .ok_or_else(|| InstallError::MissingField { + id: id.to_string(), + field: "url".to_string(), + })?, project: require_string(id, profile, "project")?, token: require_string(id, profile, "token")?, branch: optional_string(profile, "branch").unwrap_or_default(), - public_base, + public_base: optional_string(profile, "public_base").filter(|s| !is_placeholder(s)), })) } diff --git a/crates/upgit-uploaders/src/util.rs b/crates/upgit-uploaders/src/util.rs index f331713..6135b5c 100644 --- a/crates/upgit-uploaders/src/util.rs +++ b/crates/upgit-uploaders/src/util.rs @@ -158,6 +158,40 @@ pub(crate) fn host_of(url: &str) -> &str { rest.trim_end_matches('/') } +/// Absolute HTTP URL: add `https://` if no scheme, strip surrounding slashes. +pub(crate) fn http_origin(url: &str) -> String { + let t = url.trim(); + let (scheme, rest) = if let Some(rest) = t.strip_prefix("https://") { + ("https://", rest) + } else if let Some(rest) = t.strip_prefix("http://") { + ("http://", rest) + } else if t.is_empty() { + return String::new(); + } else { + ("https://", t) + }; + let rest = rest.trim_matches('/'); + if rest.is_empty() { + String::new() + } else { + format!("{scheme}{rest}") + } +} + +/// RFC 3986 unreserved stay literal; everything else is `%HH` (uppercase). +pub(crate) fn percent_encode(s: &str) -> String { + let mut out = String::new(); + for &b in s.as_bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char) + } + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + pub(crate) fn one_line(s: &str, max: usize) -> String { let line = s.lines().next().unwrap_or("").trim(); let n = line.chars().count(); diff --git a/crates/upgit-uploaders/tests/gitlab.rs b/crates/upgit-uploaders/tests/gitlab.rs deleted file mode 100644 index f08297c..0000000 --- a/crates/upgit-uploaders/tests/gitlab.rs +++ /dev/null @@ -1,184 +0,0 @@ -//! Locator, encoding, explain, and POST-then-PUT for GitlabUploader. - -use std::io::{Read, Write}; -use std::net::TcpListener; -use std::thread; -use std::time::Duration; - -use upgit_core::{Artifact, ObjectKey, Uploader}; -use upgit_uploaders::gitlab::{GitlabConfig, GitlabUploader}; -use upgit_uploaders::HostCatalog; - -fn uploader(public_base: Option<&str>) -> GitlabUploader { - GitlabUploader::new(GitlabConfig { - url: "https://gitlab.example.com".into(), - project: "group/name".into(), - token: "tok".into(), - branch: String::new(), - public_base: public_base.map(str::to_string), - }) -} - -fn read_http_request(stream: &mut std::net::TcpStream) -> Vec { - let mut buf = Vec::new(); - let mut tmp = [0u8; 4096]; - loop { - match stream.read(&mut tmp) { - Ok(0) => break, - Ok(n) => buf.extend_from_slice(&tmp[..n]), - Err(_) => break, - } - let Some(header_end) = buf.windows(4).position(|w| w == b"\r\n\r\n") else { - continue; - }; - let headers = String::from_utf8_lossy(&buf[..header_end]); - let content_len = headers - .lines() - .find(|line| line.to_ascii_lowercase().starts_with("content-length:")) - .and_then(|line| line.split(':').nth(1)) - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(0); - let body_start = header_end + 4; - if buf.len() >= body_start + content_len { - break; - } - } - buf -} - -#[test] -fn catalog_lists_gitlab_after_github() { - let hosts = HostCatalog::all(); - assert_eq!(hosts[0].id, "github"); - assert_eq!(hosts[1].id, "gitlab"); - assert_eq!(hosts[1].title, "GitLab"); -} - -#[test] -fn locator_uses_instance_raw_url_without_public_base() { - let key = ObjectKey::parse("2024/01/a.png").expect("key"); - assert_eq!( - uploader(None).locator_for(&key).as_str(), - "https://gitlab.example.com/group/name/-/raw/main/2024/01/a.png" - ); -} - -#[test] -fn locator_uses_public_base_when_set() { - let key = ObjectKey::parse("2024/01/a.png").expect("key"); - assert_eq!( - uploader(Some("https://cdn.example.com/")) - .locator_for(&key) - .as_str(), - "https://cdn.example.com/2024/01/a.png" - ); -} - -#[test] -fn files_url_percent_encodes_project_and_path() { - let key = ObjectKey::parse("2024/01/a.png").expect("key"); - assert_eq!( - uploader(None).files_url(&key), - "https://gitlab.example.com/api/v4/projects/group%2Fname/repository/files/2024%2F01%2Fa.png" - ); -} - -#[test] -fn explain_401_has_no_documentation_url_or_raw_json() { - let err = uploader(None).explain( - 401, - r#"{"message":"401 Unauthorized","documentation_url":"https://docs.gitlab.com/ee/api/index.html"}"#, - ); - let text = err.to_string(); - assert!(text.contains("401"), "got {text}"); - assert!(text.contains("token"), "got {text}"); - assert!( - !text.contains("documentation_url"), - "dumped GitLab JSON: {text}" - ); - assert!(!text.contains('{'), "dumped GitLab JSON: {text}"); - assert!( - !text.contains("401 Unauthorized"), - "leaked GitLab body: {text}" - ); -} - -fn serve_create_then_update() -> (String, thread::JoinHandle>) { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); - listener.set_nonblocking(false).expect("blocking listener"); - let addr = listener.local_addr().expect("addr"); - let handle = thread::spawn(move || { - let mut requests = Vec::new(); - for i in 0..2 { - let (mut stream, _) = listener.accept().expect("accept"); - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let buf = read_http_request(&mut stream); - requests.push(String::from_utf8_lossy(&buf).into_owned()); - let (status, body) = if i == 0 { - ( - "400 Bad Request", - r#"{"message":"A file with this name already exists"}"#, - ) - } else { - ("200 OK", r#"{"file_path":"2024/01/a.png","branch":"main"}"#) - }; - let resp = format!( - "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - stream.write_all(resp.as_bytes()).expect("write"); - stream.flush().ok(); - } - requests - }); - (format!("http://{addr}"), handle) -} - -#[test] -fn upload_posts_then_puts_when_file_exists() { - let (origin, server) = serve_create_then_update(); - let uploader = GitlabUploader::new(GitlabConfig { - url: origin, - project: "group/name".into(), - token: "tok".into(), - branch: "main".into(), - public_base: Some("https://cdn.example.com/".into()), - }); - - let dir = std::env::temp_dir(); - let file = dir.join(format!("upgit-gitlab-upload-{}.png", std::process::id())); - std::fs::write(&file, b"\x89PNG-fake").expect("write fixture"); - let artifact = Artifact::from_path(&file, None).expect("artifact"); - let key = ObjectKey::parse("2024/01/a.png").expect("key"); - - let locator = uploader.upload(&artifact, &key).expect("upload"); - assert_eq!(locator.as_str(), "https://cdn.example.com/2024/01/a.png"); - - let _ = std::fs::remove_file(&file); - let requests = server.join().expect("server"); - assert_eq!(requests.len(), 2, "expected POST then PUT"); - assert!( - requests[0].starts_with("POST "), - "first request should be create, got:\n{}", - requests[0] - ); - assert!( - requests[1].starts_with("PUT "), - "second request should be update, got:\n{}", - requests[1] - ); - for req in &requests { - assert!( - req.contains("PRIVATE-TOKEN: tok"), - "missing PRIVATE-TOKEN:\n{req}" - ); - assert!( - req.contains("/api/v4/projects/group%2Fname/repository/files/2024%2F01%2Fa.png"), - "project/path must stay percent-encoded:\n{req}" - ); - assert!( - req.contains("\"encoding\":\"base64\""), - "body must send base64 encoding:\n{req}" - ); - } -} diff --git a/crates/upgit-uploaders/tests/install.rs b/crates/upgit-uploaders/tests/install.rs index 0a1fa32..5032483 100644 --- a/crates/upgit-uploaders/tests/install.rs +++ b/crates/upgit-uploaders/tests/install.rs @@ -1,5 +1,5 @@ use upgit_core::Registry; -use upgit_uploaders::{AppConfig, InstallError}; +use upgit_uploaders::AppConfig; #[test] fn install_registers_qiniu_and_http_from_config() { @@ -36,62 +36,3 @@ token = "tok" assert!(msg.contains("qiniu"), "got {msg}"); assert!(msg.contains("smms"), "got {msg}"); } - -#[test] -fn install_registers_gitlab_from_profile_without_type() { - let cfg = AppConfig::from_toml( - r#" -default = "gitlab" - -[uploaders.gitlab] -url = "https://gitlab.example.com" -project = "group/name" -token = "tok" -branch = "main" -public_base = "https://cdn.example.com/" -"#, - ) - .expect("parse"); - let mut registry = Registry::new(); - cfg.install_into(&mut registry).expect("install"); - registry.get("gitlab").expect("gitlab registered"); -} - -#[test] -fn install_infers_gitlab_from_token_project_host() { - let cfg = AppConfig::from_toml( - r#" -[uploaders.corp] -host = "https://gitlab.corp.example" -project = "group/name" -token = "tok" -"#, - ) - .expect("parse"); - let mut registry = Registry::new(); - cfg.install_into(&mut registry).expect("install"); - registry.get("corp").expect("corp registered as gitlab"); -} - -#[test] -fn unknown_kind_lists_gitlab_among_built_ins() { - let cfg = AppConfig::from_toml( - r#" -[uploaders.webdav] -type = "webdav" -"#, - ) - .expect("parse"); - let mut registry = Registry::new(); - let err = cfg.install_into(&mut registry).expect_err("unknown kind"); - let msg = err.to_string(); - match err { - InstallError::UnknownKind { id, kind } => { - assert_eq!(id, "webdav"); - assert_eq!(kind, "webdav"); - } - other => panic!("expected UnknownKind, got {other}"), - } - assert!(msg.contains("gitlab"), "got {msg}"); - assert!(msg.contains("github"), "got {msg}"); -}