Repository navigation
feat: download http(s) file operands before upload (AI生成) - #107
Conversation
Typora passes remote image URLs as FILE arguments. Classify http(s) operands as UrlSource, download to a held temp file, and keep mixed local paths working in the same invocation.
| const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(30); | ||
| const DOWNLOAD_REDIRECTS: u32 = 10; |
There was a problem hiding this comment.
不要把策略、配置和机制混在一起,并且写到全局的常量里面。
There was a problem hiding this comment.
已改于 c48001a。删掉了 DOWNLOAD_TIMEOUT / DOWNLOAD_REDIRECTS。超时和重定向只出现在 Intake 构造 Agent 的现场,不再作为 UrlSource 的全局策略常量。
| let agent = ureq::AgentBuilder::new() | ||
| .timeout(DOWNLOAD_TIMEOUT) | ||
| .redirects(DOWNLOAD_REDIRECTS) | ||
| .user_agent(concat!("upgit/", env!("CARGO_PKG_VERSION"))) |
There was a problem hiding this comment.
为了反爬虫,User agent必须可以通过依赖注入。
There was a problem hiding this comment.
已改于 c48001a。新增 --user-agent;Intake 用它(缺省 upgit/<version>)构造 Agent。UrlSource 不再硬编码 UA。测试注入 upgit-test/1 并核对对端收到的头。
| .timeout(DOWNLOAD_TIMEOUT) | ||
| .redirects(DOWNLOAD_REDIRECTS) |
There was a problem hiding this comment.
已改于 c48001a。UrlSource::new(url, size_limit, agent) 接收整个 ureq::Agent,artifacts() 只对注入的 agent 发 GET。Intake 负责按 CLI 组装 Agent;测试自己注入短超时 + 自定义 UA 的 Agent。
| fn looks_like_http_url(operand: &str) -> bool { | ||
| strip_http_scheme(operand.trim()).is_some() | ||
| } | ||
|
|
||
| fn strip_http_scheme(s: &str) -> Option<&str> { | ||
| if starts_with_ignore_ascii_case(s, "https://") { | ||
| Some(&s["https://".len()..]) | ||
| } else if starts_with_ignore_ascii_case(s, "http://") { | ||
| Some(&s["http://".len()..]) | ||
| } else { | ||
| None | ||
| } | ||
| } | ||
|
|
||
| fn starts_with_ignore_ascii_case(s: &str, prefix: &str) -> bool { | ||
| s.len() >= prefix.len() && s.as_bytes()[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes()) | ||
| } | ||
|
|
||
| fn resolve_download_name(cd: Option<&str>, url: &str, content_type: Option<&str>) -> String { | ||
| if let Some(header) = cd { | ||
| if let Some(name) = filename_from_content_disposition(header) { | ||
| return name; | ||
| } | ||
| } | ||
| if let Some(name) = filename_from_url_path(url) { | ||
| return name; | ||
| } | ||
| let mut name = String::from("download"); | ||
| if let Some(ct) = content_type { | ||
| if let Some(ext) = extension_from_content_type(ct) { | ||
| name.push_str(ext); | ||
| } | ||
| } | ||
| name | ||
| } | ||
|
|
||
| fn filename_from_url_path(url: &str) -> Option<String> { | ||
| let rest = strip_http_scheme(url.trim())?; | ||
| let rest = rest.split_once('#').map(|(a, _)| a).unwrap_or(rest); | ||
| let rest = rest.split_once('?').map(|(a, _)| a).unwrap_or(rest); | ||
| let path = path_after_host(rest); | ||
| if path.is_empty() || path.ends_with('/') { | ||
| return None; | ||
| } | ||
| let segment = path.split('/').next_back().filter(|s| !s.is_empty())?; | ||
| let decoded = percent_decode(segment)?; | ||
| let decoded = decoded.trim(); | ||
| sane_filename(decoded).map(str::to_string) | ||
| } | ||
|
|
||
| fn path_after_host(rest: &str) -> &str { | ||
| if let Some(rest) = rest.strip_prefix('[') { | ||
| return rest | ||
| .split_once(']') | ||
| .map(|(_, after)| after.split_once('/').map(|(_, p)| p).unwrap_or("")) | ||
| .unwrap_or(""); | ||
| } | ||
| rest.split_once('/').map(|(_, p)| p).unwrap_or("") | ||
| } | ||
|
|
||
| fn filename_from_content_disposition(header: &str) -> Option<String> { | ||
| if let Some(value) = header_param(header, "filename*") { | ||
| if let Some(name) = decode_ext_value(&value) { | ||
| let name = name.trim(); | ||
| if let Some(name) = sane_filename(name) { | ||
| return Some(name.to_string()); | ||
| } | ||
| } | ||
| } | ||
| let value = header_param(header, "filename")?; | ||
| let decoded = percent_decode(&value).unwrap_or(value); | ||
| let decoded = decoded.trim(); | ||
| sane_filename(decoded).map(str::to_string) | ||
| } | ||
|
|
||
| fn header_param(header: &str, name: &str) -> Option<String> { | ||
| let lower = header.to_ascii_lowercase(); | ||
| let key = name.to_ascii_lowercase(); | ||
| let bytes = header.as_bytes(); | ||
| let mut i = 0; | ||
| while let Some(pos) = lower[i..].find(&key) { | ||
| let start = i + pos; | ||
| if start > 0 { | ||
| let prev = bytes[start - 1]; | ||
| if prev != b';' && !prev.is_ascii_whitespace() { | ||
| i = start + 1; | ||
| continue; | ||
| } | ||
| } | ||
| let mut j = start + key.len(); | ||
| while j < bytes.len() && bytes[j].is_ascii_whitespace() { | ||
| j += 1; | ||
| } | ||
| if j >= bytes.len() || bytes[j] != b'=' { | ||
| i = start + 1; | ||
| continue; | ||
| } | ||
| j += 1; | ||
| while j < bytes.len() && bytes[j].is_ascii_whitespace() { | ||
| j += 1; | ||
| } | ||
| return Some(parse_token_or_quoted(&header[j..])); | ||
| } | ||
| None | ||
| } | ||
|
|
||
| fn parse_token_or_quoted(s: &str) -> String { | ||
| let s = s.trim_start(); | ||
| if let Some(rest) = s.strip_prefix('"') { | ||
| let mut out = String::new(); | ||
| let mut chars = rest.chars(); | ||
| while let Some(c) = chars.next() { | ||
| if c == '\\' { | ||
| if let Some(n) = chars.next() { | ||
| out.push(n); | ||
| } | ||
| } else if c == '"' { | ||
| break; | ||
| } else { | ||
| out.push(c); | ||
| } | ||
| } | ||
| out | ||
| } else { | ||
| s.split(';').next().unwrap_or("").trim().to_string() | ||
| } | ||
| } | ||
|
|
||
| fn decode_ext_value(v: &str) -> Option<String> { | ||
| let mut parts = v.splitn(3, '\''); | ||
| let charset = parts.next()?; | ||
| let _lang = parts.next()?; | ||
| let value = parts.next()?; | ||
| if !charset.is_empty() && !charset.eq_ignore_ascii_case("utf-8") { | ||
| return None; | ||
| } | ||
| percent_decode(value) | ||
| } | ||
|
|
||
| fn extension_from_content_type(ct: &str) -> Option<&'static str> { | ||
| let mime = ct.split(';').next()?.trim().to_ascii_lowercase(); | ||
| match mime.as_str() { | ||
| "image/png" => Some(".png"), | ||
| "image/jpeg" | "image/jpg" => Some(".jpeg"), | ||
| "image/gif" => Some(".gif"), | ||
| "image/webp" => Some(".webp"), | ||
| "image/svg+xml" => Some(".svg"), | ||
| _ => None, | ||
| } | ||
| } | ||
|
|
||
| fn sane_filename(name: &str) -> Option<&str> { | ||
| if name.is_empty() || name == "." || name == ".." || name.len() > 255 { | ||
| return None; | ||
| } | ||
| if name.bytes().all(|b| b == b'.') { | ||
| return None; | ||
| } | ||
| let bad = [b'/', b'\\', 0, b':', b'*', b'?', b'"', b'<', b'>', b'|']; | ||
| if name | ||
| .bytes() | ||
| .any(|b| bad.contains(&b) || b.is_ascii_control()) | ||
| { | ||
| return None; | ||
| } | ||
| Some(name) | ||
| } | ||
|
|
||
| fn percent_decode(s: &str) -> Option<String> { | ||
| let bytes = s.as_bytes(); | ||
| let mut out = Vec::with_capacity(bytes.len()); | ||
| let mut i = 0; | ||
| while i < bytes.len() { | ||
| if bytes[i] == b'%' { | ||
| if i + 2 >= bytes.len() { | ||
| return None; | ||
| } | ||
| let hi = from_hex(bytes[i + 1])?; | ||
| let lo = from_hex(bytes[i + 2])?; | ||
| out.push((hi << 4) | lo); | ||
| i += 3; | ||
| } else { | ||
| out.push(bytes[i]); | ||
| i += 1; | ||
| } | ||
| } | ||
| String::from_utf8(out).ok() | ||
| } | ||
|
|
||
| fn temp_suffix(name: &str) -> Option<&str> { | ||
| let i = name.rfind('.')?; | ||
| if i == 0 || i + 1 == name.len() { | ||
| return None; | ||
| } | ||
| let ext = &name[i + 1..]; | ||
| if ext.len() <= 16 && ext.bytes().all(|b| b.is_ascii_alphanumeric()) { | ||
| Some(&name[i..]) | ||
| } else { | ||
| None | ||
| } | ||
| } | ||
|
|
||
| fn copy_limited<R: Read, W: Write>( | ||
| url: &str, | ||
| reader: &mut R, | ||
| writer: &mut W, | ||
| size_limit: Option<u64>, | ||
| ) -> Result<u64, Box<dyn Error>> { | ||
| let mut buf = [0u8; 8192]; | ||
| let mut total = 0u64; | ||
| loop { | ||
| let n = reader | ||
| .read(&mut buf) | ||
| .map_err(|e| format!("cannot download {url}: cannot read download: {e}"))?; | ||
| if n == 0 { | ||
| break; | ||
| } | ||
| total = total.saturating_add(n as u64); | ||
| if let Some(limit) = size_limit { | ||
| if total > limit { | ||
| return Err(over_download_limit(url, total, limit)); | ||
| } | ||
| } | ||
| writer | ||
| .write_all(&buf[..n]) | ||
| .map_err(|e| format!("cannot download {url}: cannot write download: {e}"))?; | ||
| } | ||
| Ok(total) | ||
| } | ||
|
|
||
| fn over_download_limit(url: &str, size: u64, limit: u64) -> Box<dyn Error> { | ||
| download_failed( | ||
| url, | ||
| ArtifactError::OverLimit { size, limit }, | ||
| Some("pass --size-limit BYTES (0 = unlimited); default is 5MiB"), | ||
| ) | ||
| } | ||
|
|
||
| fn annotate_artifact(url: &str, err: ArtifactError) -> Box<dyn Error> { | ||
| match &err { | ||
| ArtifactError::OverLimit { size, limit } => over_download_limit(url, *size, *limit), | ||
| _ => download_failed(url, err, None), | ||
| } | ||
| } | ||
|
|
||
| fn download_failed(url: &str, what: impl std::fmt::Display, hint: Option<&str>) -> Box<dyn Error> { | ||
| match hint { | ||
| Some(hint) if !hint.is_empty() => { | ||
| format!("cannot download {url}: {what}\nhint: {hint}").into() | ||
| } | ||
| _ => format!("cannot download {url}: {what}").into(), | ||
| } | ||
| } | ||
|
|
||
| fn http_what(err: ureq::Error) -> String { | ||
| match err { | ||
| ureq::Error::Status(code, _) => format!("HTTP {code}"), | ||
| ureq::Error::Transport(t) => { | ||
| let mut detail = t | ||
| .message() | ||
| .map(str::to_string) | ||
| .unwrap_or_else(|| t.kind().to_string()); | ||
| let mut src = Error::source(&t); | ||
| while let Some(s) = src { | ||
| detail = s.to_string(); | ||
| src = s.source(); | ||
| } | ||
| detail | ||
| } | ||
| } | ||
| } | ||
|
|
There was a problem hiding this comment.
此处大量FP无主带副作用函数,建议Kay式OOP化或者内联lambda,低价值测试可删除。
There was a problem hiding this comment.
已改于 c48001a。那些无主 FP 辅助函数收进 UrlSource 的方法;检测/文件名/404 等低价值单测已删。只留一条 loopback 测试:注入 Agent 下载并断言 UA。
| # 对于 Windows: .\upgit.exe logo.png -t /my_images/demo | ||
| ``` | ||
|
|
||
| FILE 可以是本地路径,也可以是 `http://` / `https://` 地址(先下载再上传)。Typora 把远程图片 URL 当路径传入时同样适用。 |
There was a problem hiding this comment.
已改于 c48001a。中英 Get-started 都收成一句,去掉 Typora 附句。
Build ureq::Agent in Intake from --user-agent (timeout/redirects at construction). UrlSource only GET on the given agent. Drop global download consts, free helper tests, and extra README clauses.
UrlSource::artifact_name asks ContentType::image_extension instead of matching MIME strings itself.
|
Addressed in SHA: 9b87816
|
Closes #72
Typora 把远程图片 URL 当 FILE 传给 upgit。本 PR 在 intake 侧把
http:///https://操作数变成UrlSource:下载到临时文件并持有,再按现有 Artifact 上传。Publisher / Uploader 不下载。Typora
Custom Command 仍是
upgit。远程图片会以 URL 作为 FILE 传入,例如:混合本地路径和 URL 可以一次调用。
--size-limit(默认 5MiB,0无限)对下载同样生效。下载超时 30s,跟随重定向。How to verify
手工(需已配置的 uploader):
失败路径:把 FILE 换成会 404 的 URL,stderr 应是
cannot download …: HTTP 404加可选hint:,不能出现 HTML。超大文件应在读完 body 前/中因 size limit 失败。单元测试不打公网:URL 识别、路径文件名、本地 mock 的 size-limit / 404 / 混合操作数。
三剑客(讨论 / 收敛)
Eric Evans:
领域语言是 uploader / config / init,加上本上下文的 FILE 操作数 和 Source → Artifact。Issue #72 的业务事实:Typora 把远程图片 URL 当路径交给 upgit,希望「先下载再上传到图床」。底线:这是 intake 的规则,不是 publish / upload 的规则。Artifact 已经定义为「通过 size check 的本地文件」;远程 URL 必须在进入 Publisher 之前变成 Artifact。选项:(1) 在 intake 引入 UrlSource,FILE = 本地路径 | http(s) URL;(2) 让 Uploader 自己 GET URL,污染每个 host 的上传消息;(3) 做一层 Go 0.2 兼容适配。不能让步:不把「下载」写进 Publisher/Uploader;不引入 0.2 前向兼容;统一语言里 FILE 可以是 URL,但 Artifact 仍然是本地文件。Size limit 是 intake 不变量,下载必须遵守。
John Ousterhout:
Intake 已经是深模块:
from_cli+collect,调用方(App)不必知道 clipboard vs 文件。完成标准:混合操作数保持顺序;http(s) 大小写不敏感;下载进临时文件且生命周期盖过 upload;CD / URL path /download+content-type 命名;跟随重定向;超时;size-limit 在读 oversized body 之前或之中失败;错误是{what}+ 可选hint:,不倒 HTML;测试不打公网。选项:(A) FileSource 内部识别 URL 并代理下载——FileSource 会被迫持有 temp file,接口变浅、信息泄漏;(B) Intake 按操作数分类,UrlSource 自己藏 HTTP/临时文件;(C) 抽出 Fetcher trait 方便测试——对一个生产实现是浅抽象。怎样算做完:Intake::from_cliAPI 不变所以不用改 app.rs;现有测试仍过;有 mock 覆盖检测、文件名、size-limit。复杂度风险:Content-Disposition 解析容易长成迷你 RFC;只解析filename/filename*,不健全则回退。Alan Kay:
Source 已经是对象:
artifacts()一条消息,FileSource / ClipboardImageSource / ClipboardFilesSource 各自封装状态。URL 必须是另一个对象 UrlSource(持有 URL、下载、NamedTempFile),而不是 FileSource 里的if。调用方只发artifacts(),不读 URL、不管理临时文件。选项:FileSource 委派给内部 UrlSource——委派可以,但 FileSource 仍要 hold 别人的临时文件,封装破了;Intake 创建 UrlSource 并放进sources向量,生命周期自然跟 clipboard PNG 一样。极端延迟绑定:不需要 Fetcher 协议,ureq 是 UrlSource 的私有实现。简单的事(本地路径)仍走 FileSource;复杂的事(下载、命名、限速、超时)关在 UrlSource 里。三个人的讨论
Kay 否决 Evans 若把 URL 只做成值对象/newtype 而不给行为:没有对象就没有「谁持有临时文件」。Evans 同意身份在 Source,不在字符串;并认为 Kay 若把下载放进 Uploader 是跨上下文,过松。Ousterhout 同意 Kay 的 UrlSource,但认为 Kay 若再加 Fetcher trait 是过严——测试用本机 TcpListener 已是仓库惯例(recipe_upload)。Kay 认为 Ousterhout 的「FileSource 内部 if URL」过松:那是过程式分类,不是对象。三人合并:Intake 分类操作数(Evans 的限界:intake vs publish),每个 http(s) FILE 一个 UrlSource(Kay),App 仍然只
from_cli/collect(Ousterhout 深模块,不改 app.rs)。冲突点「是否批量连续本地路径」:Ousterhout 可接受每个本地操作数一个 FileSource,Typora 通常一两个 FILE,不值得 flush 分组。SSRF:Evans 记为「用户显式传入的本地 CLI」,接受;仍拒绝无 scheme 的空字符串,走 FileSource 既有缺文件错误。收敛结论
状态:已收敛。
推荐路径:
Intake::from_cli对每个 FILE 判断http:///https://(大小写不敏感)→UrlSource或FileSource;UrlSource用已有 ureq 下载到 NamedTempFile 并 hold;文件名:健全的 Content-Disposition,否则 URL path 最后一段,否则download+ content-type 扩展名(png/jpeg/gif/webp/svg);Content-Length 超限则先失败,读 body 时再兜底;30s 超时、最多 10 次重定向;错误不包含响应体。否决:
--clean语义:Intake API 未变;--clean仍按本地路径删除cli.files,URL 操作数删文件会失败,不在本 issue 范围。残留风险:用户传入的 URL 可指向内网(SSRF 已接受);部分图床对非浏览器 UA 不友好(UA 为
upgit/<version>,与 update 一致)。决策与后续
已决定:
待验证:真实 Typora 自定义命令把远程图片 URL 原样作为 argv 传入(假设成立则本 PR 即所需行为)。
--clean+ URL 仍会在 App 里remove_file(url)失败,若以后要支持,应改 App 只删本地路径,而不是让 UrlSource 假装自己是路径。