Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 6 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -290,9 +290,12 @@ For Windows user:
| -- `{unix_tsms}` | -- Unix timestamp in millisecond. Like `1644212979622`. |
| --- `{ext}` | -- Extension like `.png`, and empty when the original file has no extension |
| -- `{fname}` | -- Original file base name like `demo` (without extension) |
| -- `{fname_hash}` | -- MD5 Hash in hex of `{fname}` (name without extension) |
| -- `{fname_hash4}` | -- MD5 Hash in hex of `{fname}`, first 4 digits |
| -- `{fname_hash8}` | -- MD5 Hash in hex of `{fname}`, first 8 digits |
| -- `{fname_hash}` | -- xxHash32 (seed 0) hex of `{fname}` (name without extension) |
| -- `{fname_hash4}` | -- xxHash32 hex of `{fname}`, first 4 digits |
| -- `{fname_hash8}` | -- xxHash32 hex of `{fname}` (full 8-digit digest) |
| -- `{content_hash}` | -- xxHash32 (seed 0) hex of the file contents |
| -- `{content_hash4}` | -- xxHash32 hex of the file contents, first 4 digits |
| -- `{content_hash8}` | -- xxHash32 hex of the file contents (full 8-digit digest) |
| -- `{hmac}` | -- HMAC-SHA256 hash of `hmac_format`, truncated to `hmac_len` |
| hmac_key | Secret key for calculation `{hmac}` |
| hmac_format | Format string for `{hmac}` calculation. Supporting all above placeholders. |
Expand Down
2 changes: 1 addition & 1 deletion config.github.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# Which uploader table to use.
default = "github"

# Remote object key template. Placeholders: {year} {month} {day} {hour} {minute} {second} {unix} {unix_tsms} {stem} {fname} {fullname} {ext} {hmac} {fname_hash} {fname_hash4} {fname_hash8}.
# Remote object key template. Placeholders: {year} {month} {day} {hour} {minute} {second} {unix} {unix_tsms} {stem} {fname} {fullname} {ext} {hmac} {fname_hash} {fname_hash4} {fname_hash8} {content_hash} {content_hash4} {content_hash8}. {fname_hash} and {content_hash} are xxHash32 (seed 0); {hmac} is HMAC-SHA256.
naming = "{year}/{month}/upgit_{year}{month}{day}_{unix}{ext}"

# Optional HMAC secret for the {hmac} placeholder in naming. Any random string works.
Expand Down
2 changes: 1 addition & 1 deletion config.sample.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
# Which uploader table to use (must match a [uploaders.*] table below).
default = "github"

# Remote object key template. Placeholders: {year} {month} {day} {hour} {minute} {second} {unix} {unix_tsms} {stem} {fname} {fullname} {ext} {hmac} {fname_hash} {fname_hash4} {fname_hash8}.
# Remote object key template. Placeholders: {year} {month} {day} {hour} {minute} {second} {unix} {unix_tsms} {stem} {fname} {fullname} {ext} {hmac} {fname_hash} {fname_hash4} {fname_hash8} {content_hash} {content_hash4} {content_hash8}. {fname_hash} and {content_hash} are xxHash32 (seed 0); {hmac} is HMAC-SHA256.
naming = "{year}/{month}/upgit_{year}{month}{day}_{unix}_{hmac}{ext}"

# Secret used to compute {hmac} (HMAC-SHA256). Required only if naming contains {hmac}.
Expand Down
2 changes: 1 addition & 1 deletion config.sample.zh-CN.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
# 使用哪一张上传器表(必须对应下面某个 [uploaders.*])。
default = "github"

# 远程对象键模板。占位符:{year} {month} {day} {hour} {minute} {second} {unix} {unix_tsms} {stem} {fname} {fullname} {ext} {hmac} {fname_hash} {fname_hash4} {fname_hash8}。
# 远程对象键模板。占位符:{year} {month} {day} {hour} {minute} {second} {unix} {unix_tsms} {stem} {fname} {fullname} {ext} {hmac} {fname_hash} {fname_hash4} {fname_hash8} {content_hash} {content_hash4} {content_hash8}。{fname_hash} 与 {content_hash} 为 xxHash32(种子 0);{hmac} 为 HMAC-SHA256。
naming = "{year}/{month}/upgit_{year}{month}{day}_{unix}_{hmac}{ext}"

# 用于计算 {hmac} 的密钥(HMAC-SHA256)。仅当 naming 含 {hmac} 时需要。
Expand Down
2 changes: 1 addition & 1 deletion crates/upgit-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,6 @@ description = "Domain objects for upgit: artifacts, keys, publisher, registry."
[dependencies]
chrono = { version = "0.4", default-features = false, features = ["std"] }
hmac = "0.12"
md-5 = "0.10"
sha2 = "0.10"
thiserror = "2"
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh32"] }
10 changes: 10 additions & 0 deletions crates/upgit-core/src/artifact.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,16 @@ impl Artifact {
self.path.as_deref()
}

/// File bytes when this artifact has a path. `Ok(None)` if there is no path.
pub fn bytes(&self) -> Result<Option<Vec<u8>>, ArtifactError> {
let Some(path) = self.path.as_deref() else {
return Ok(None);
};
std::fs::read(path)
.map(Some)
.map_err(|e| ArtifactError::Io(e.to_string()))
}

pub fn file_name(&self) -> &str {
&self.name
}
Expand Down
98 changes: 81 additions & 17 deletions crates/upgit-core/src/key_policy.rs

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

并非所有源都能提供content,因此content_hash可能为空,千万不要用空串去生成哈希值。应当调用方显式提供fallback

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

空字节不再参与散列。KeyPolicy::apply(artifact, at, content_hash_fallback: Option<&str>):有非空内容时 xxHash32;没有可读内容时用调用方传入的 fallback(原样插入,不哈希)。未提供 fallback 才 MissingContent。Publisher / App 显式传入 Some(artifact.stem())。

2777635

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已 OOP 化:resolve_content_hash 收到 KeyPolicy 上;HMAC 的 hex 编码收到 HmacSpec::hex_lower。文件级游离函数已去掉。

fcbd360

Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ use std::time::{SystemTime, UNIX_EPOCH};

use chrono::DateTime;
use hmac::{Hmac, Mac};
use md5::{Digest, Md5};
use sha2::Sha256;
use thiserror::Error;
use xxhash_rust::xxh32::xxh32;

use crate::artifact::Artifact;
use crate::artifact::{Artifact, ArtifactError};
use crate::object_key::{ObjectKey, ObjectKeyError};

type HmacSha256 = Hmac<Sha256>;
Expand Down Expand Up @@ -36,12 +36,22 @@ impl HmacSpec {
let mut mac = HmacSha256::new_from_slice(self.key.as_bytes())
.map_err(|_| KeyPolicyError::InvalidHmacKey)?;
mac.update(material.as_bytes());
let mut hex = hex_lower(&mac.finalize().into_bytes());
let mut hex = Self::hex_lower(&mac.finalize().into_bytes());
if let Some(n) = self.len {
hex.truncate(n.min(hex.len()));
}
Ok(hex)
}

fn hex_lower(bytes: &[u8]) -> String {
const LUT: &[u8; 16] = b"0123456789abcdef";
let mut out = String::with_capacity(bytes.len() * 2);
for &b in bytes {
out.push(LUT[(b >> 4) as usize] as char);
out.push(LUT[(b & 0x0f) as usize] as char);
}
out
}
}

#[derive(Debug, Error, Clone, PartialEq, Eq)]
Expand All @@ -56,6 +66,10 @@ pub enum KeyPolicyError {
InvalidHmacKey,
#[error("naming template uses `{{hmac}}` but `hmac_key` is not set")]
MissingHmacKey,
#[error("naming template uses `{{content_hash}}` but the artifact has no readable bytes")]
MissingContent,
#[error("cannot read artifact content: {0}")]
ContentIo(String),
}

impl KeyPolicy {
Expand Down Expand Up @@ -87,7 +101,15 @@ impl KeyPolicy {
}
}

pub fn apply(&self, artifact: &Artifact, at: SystemTime) -> Result<ObjectKey, KeyPolicyError> {
/// `content_hash_fallback` is inserted as `{content_hash*}` when the artifact
/// has no readable bytes. It is never hashed. Required if the template (or
/// `hmac_format`) uses those placeholders and content is missing.
pub fn apply(
&self,
artifact: &Artifact,
at: SystemTime,
content_hash_fallback: Option<&str>,
) -> Result<ObjectKey, KeyPolicyError> {
match &self.kind {
KeyKind::KeepOriginal { dir } => {
let dir = dir.trim().trim_matches('/');
Expand All @@ -106,7 +128,14 @@ impl KeyPolicy {
if self.hmac.is_none() && template.contains("{hmac}") {
return Err(KeyPolicyError::MissingHmacKey);
}
let fields = Fields::from_artifact(artifact, at)?;
let xxh32_hex = |data: &[u8]| format!("{:08x}", xxh32(data, 0));
let content_hash = self.resolve_content_hash(
artifact,
template,
content_hash_fallback,
xxh32_hex,
)?;
let fields = Fields::from_artifact(artifact, at, xxh32_hex, content_hash)?;
let hmac = match &self.hmac {
Some(spec) => {
let material = fields.interpolate(&spec.format, None);
Expand All @@ -119,6 +148,33 @@ impl KeyPolicy {
}
}
}

fn resolve_content_hash(
&self,
artifact: &Artifact,
template: &str,
fallback: Option<&str>,
xxh32_hex: impl Fn(&[u8]) -> String,
) -> Result<Option<String>, KeyPolicyError> {
let uses_content_hash = |s: &str| s.contains("{content_hash") || s.contains("{contenthash");
let needs_content = uses_content_hash(template)
|| self
.hmac
.as_ref()
.is_some_and(|spec| uses_content_hash(&spec.format));
if !needs_content {
return Ok(None);
}
match artifact.bytes() {
Ok(Some(bytes)) if !bytes.is_empty() => Ok(Some(xxh32_hex(&bytes))),
Ok(_) => match fallback {
Some(fb) => Ok(Some(fb.to_string())),
None => Err(KeyPolicyError::MissingContent),
},
Err(ArtifactError::Io(msg)) => Err(KeyPolicyError::ContentIo(msg)),
Err(other) => Err(KeyPolicyError::ContentIo(other.to_string())),
}
}
}

struct Fields<'a> {
Expand All @@ -134,16 +190,22 @@ struct Fields<'a> {
ext: &'a str,
fullname: &'a str,
fname_hash: String,
content_hash: Option<String>,
}

impl<'a> Fields<'a> {
fn from_artifact(artifact: &'a Artifact, at: SystemTime) -> Result<Self, KeyPolicyError> {
fn from_artifact(
artifact: &'a Artifact,
at: SystemTime,
xxh32_hex: impl Fn(&[u8]) -> String,
content_hash: Option<String>,
) -> Result<Self, KeyPolicyError> {
let duration = at
.duration_since(UNIX_EPOCH)
.map_err(|_| KeyPolicyError::InvalidTime)?;
let datetime = DateTime::from_timestamp(duration.as_secs() as i64, 0)
.ok_or(KeyPolicyError::InvalidTime)?;
let fname_hash = hex_lower(&Md5::digest(artifact.stem().as_bytes()));
let fname_hash = xxh32_hex(artifact.stem().as_bytes());
Ok(Self {
year: datetime.format("%Y").to_string(),
month: datetime.format("%m").to_string(),
Expand All @@ -157,6 +219,7 @@ impl<'a> Fields<'a> {
ext: artifact.ext(),
fullname: artifact.file_name(),
fname_hash,
content_hash,
})
}

Expand Down Expand Up @@ -187,19 +250,20 @@ impl<'a> Fields<'a> {
.replace("{stem}", self.stem)
.replace("{fname}", self.stem)
.replace("{ext}", self.ext);
if let Some(content_hash) = &self.content_hash {
let content4 = &content_hash[..content_hash.len().min(4)];
let content8 = &content_hash[..content_hash.len().min(8)];
out = out
.replace("{content_hash4}", content4)
.replace("{contenthash4}", content4)
.replace("{content_hash8}", content8)
.replace("{contenthash8}", content8)
.replace("{content_hash}", content_hash)
.replace("{contenthash}", content_hash);
}
if let Some(hmac) = hmac {
out = out.replace("{hmac}", hmac);
}
out
}
}

fn hex_lower(bytes: &[u8]) -> String {
const LUT: &[u8; 16] = b"0123456789abcdef";
let mut out = String::with_capacity(bytes.len() * 2);
for &b in bytes {
out.push(LUT[(b >> 4) as usize] as char);
out.push(LUT[(b & 0x0f) as usize] as char);
}
out
}
8 changes: 6 additions & 2 deletions crates/upgit-core/src/publisher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,11 @@ impl Publisher {
uploader: &dyn Uploader,
artifact: &Artifact,
at: SystemTime,
content_hash_fallback: Option<&str>,
) -> Result<PublicUrl, PublishError> {
Ok(self.publish_with_raw(uploader, artifact, at)?.1)
Ok(self
.publish_with_raw(uploader, artifact, at, content_hash_fallback)?
.1)
}

/// Locator before `[link]` replacements, and the rewritten public URL.
Expand All @@ -43,8 +46,9 @@ impl Publisher {
uploader: &dyn Uploader,
artifact: &Artifact,
at: SystemTime,
content_hash_fallback: Option<&str>,
) -> Result<(Locator, PublicUrl), PublishError> {
let key = self.namer.apply(artifact, at)?;
let key = self.namer.apply(artifact, at, content_hash_fallback)?;
let locator = uploader.upload(artifact, &key)?;
let url = self.linker.apply(&locator);
Ok((locator, url))
Expand Down
Loading