feat: storage streaming (#22)

## Summary

- add memory, local filesystem, and S3-compatible paste storage backends
- run SeaweedFS as the development S3 service through Docker Compose
- stream raw API uploads through bounded private spool files while
calculating size and SHA-256
- store UTF-8/NUL-free textual pastes up to 8 KiB inline in PostgreSQL
and use blob storage for larger or binary content
- persist normalized content types and safely represent binary bodies in
the API, Rust client, CLI, and LiveView
- harden upload finalization, read integrity, cleanup, and raw browser
responses

## Storage and upload behavior

Paste bodies are limited to 1 MiB by default. Raw request bodies are
read in bounded 64 KiB chunks into private temporary files while size
and SHA-256 are calculated.

Bodies are stored inline in PostgreSQL only when they:

- are at or below `TEXTBIN_INLINE_PASTE_BYTES` (8 KiB by default)
- have a textual content type
- contain valid UTF-8 without NUL bytes

All other bodies use the configured local or S3-compatible backend.
Development uses SeaweedFS through Docker Compose; simple self-hosting
can select local filesystem storage.

## Safety and durability

- synchronize local file contents, renames, directory creation,
deletion, and temporary-file cleanup before acknowledging operations
- journal pending blob uploads in PostgreSQL so interrupted uploads can
be recovered after process or host failure
- atomically coordinate cleanup claims with paste insertion to prevent
committed objects from being deleted
- register active spool files by owner process and periodically remove
only stale, unowned files
- verify persisted size and SHA-256 whenever paste content is read
- disable redirects and retries for manually signed one-shot S3 streams
- limit persisted content types to the database boundary

Raw browser responses only allow passive `text/plain` and
`application/json` content inline. HTML, XML, SVG, JavaScript, binary,
and other user-provided types are attachments. Responses include
`X-Content-Type-Options: nosniff` and a sandboxed CSP.

## API and CLI compatibility

Text responses retain the existing `data` string. Binary responses use:

```json
{
  \"data\": null,
  \"data_base64\": \"/wAB\",
  \"data_encoding\": \"base64\",
  \"content_type\": \"application/octet-stream\"
}
```

The Rust client now exposes paste data as bytes and preserves the
text/binary discriminator. The workspace version is bumped to `0.2.0` to
document this deliberate pre-1.0 compatibility change. CLI raw output
preserves arbitrary bytes exactly.

## Database changes

- add storage metadata and location fields to pastes
- add normalized `content_type`
- add `pending_uploads` cleanup journal with claim leases

## CI and verification

GitHub Actions exercises memory, local filesystem, and real
S3-compatible storage through a matrix. SeaweedFS is only started for
the S3 matrix entry.

Validated locally with:

- `mix precommit` — 268 tests
- memory backend — 268 tests
- local backend — 268 tests
- SeaweedFS/S3 backend — 268 tests
- `cargo test --workspace` — 53 tests
- `make lint` — Credo and Clippy pass

A final Oracle review returned **SHIP** with no remaining blockers.

---------

Co-authored-by: Darwin D Wu <darwin67@users.noreply.github.com>

Author: darwin67

Cargo.lock

@@ -3,12 +3,13 @@ members = ["crates/cli", "crates/client"]
 resolver = "3"
 
 [workspace.package]
-version = "0.1.0"
+version = "0.2.0"
 edition = "2024"
 license = "GPL-3.0-or-later"
 
 [workspace.dependencies]
 anyhow = "1.0.100"
+base64 = "0.22.1"
 clap = { version = "4.6.1", features = ["derive", "env"] }
 directories = "6.0.0"
 keyring = "4.1.5"

Cargo.toml

@@ -19,6 +19,25 @@ Then run `mix setup` and `mix phx.server` locally as usual.
 
 Ready to run in production? Please [check our deployment guides](https://hexdocs.pm/phoenix/deployment.html).
 
+## API binary content
+
+API v1 returns UTF-8 textual paste bodies in `data`. Arbitrary binary bodies use
+an explicit Base64 representation so JSON remains valid:
+
+```json
+{
+  "data": null,
+  "data_base64": "/wAB",
+  "data_encoding": "base64",
+  "content_type": "application/octet-stream"
+}
+```
+
+The `textbin-client` 0.2 release reflects this binary-capable contract by
+exposing `Paste.data` as bytes instead of a Rust `String`. This is a deliberate
+pre-1.0 breaking change; consumers that only accept text should use
+`Paste::text()`.
+
 ## Learn more
 
 * Official website: https://www.phoenixframework.org/

README.md

@@ -26,6 +26,11 @@ config :textbin,
   allow_guest_pastes: false,
   guest_paste_ttl: "6h",
   max_paste_bytes: 1_048_576,
+  inline_paste_bytes: 8_192,
+  upload_tmp_dir: Path.join(System.tmp_dir!(), "textbin-uploads"),
+  upload_cleanup_enabled: true,
+  upload_cleanup_interval_ms: :timer.minutes(15),
+  upload_cleanup_stale_after_seconds: :timer.hours(1) |> div(1_000),
   expiration_cleanup_enabled: true,
   expiration_cleanup_interval_ms: :timer.minutes(15),
   expiration_cleanup_batch_size: 500

config/config.exs

@@ -20,6 +20,17 @@ if System.get_env("PHX_SERVER") do
   config :textbin, TextbinWeb.Endpoint, server: true
 end
 
+if upload_tmp_dir = System.get_env("TEXTBIN_UPLOAD_TMP_DIR") do
+  config :textbin, upload_tmp_dir: upload_tmp_dir
+end
+
+if inline_paste_bytes = System.get_env("TEXTBIN_INLINE_PASTE_BYTES") do
+  case Integer.parse(inline_paste_bytes) do
+    {value, ""} when value >= 0 -> config :textbin, inline_paste_bytes: value
+    _result -> raise "TEXTBIN_INLINE_PASTE_BYTES must be a non-negative integer"
+  end
+end
+
 storage_backend = System.get_env("TEXTBIN_STORAGE_BACKEND")
 
 case storage_backend do

config/runtime.exs

@@ -43,7 +43,7 @@ pub fn handle(args: &ShowArgs, settings: &Settings) -> Result<()> {
     let client = settings.client()?;
     let paste = client.get_paste(&args.id)?;
 
-    if args.raw {
+    if args.raw || paste.text().is_none() {
         write_raw(io::stdout().lock(), &paste.data)?;
         return Ok(());
     }
@@ -56,8 +56,8 @@ pub fn handle(args: &ShowArgs, settings: &Settings) -> Result<()> {
     Ok(())
 }
 
-fn write_raw(mut writer: impl Write, data: &str) -> io::Result<()> {
-    writer.write_all(data.as_bytes())
+fn write_raw(mut writer: impl Write, data: &[u8]) -> io::Result<()> {
+    writer.write_all(data)
 }
 
 fn print_code_area(content: &str) {
@@ -73,23 +73,27 @@ fn format_code_area(content: &str) -> String {
 }
 
 fn render_paste(paste: &Paste, use_color: bool) -> Result<String> {
+    let data = paste
+        .text()
+        .context("binary paste cannot be rendered as text")?;
+
     if use_color {
-        highlight_paste(paste)
+        highlight_paste(paste, data)
     } else {
-        Ok(paste.data.clone())
+        Ok(data.to_string())
     }
 }
 
-fn highlight_paste(paste: &Paste) -> Result<String> {
-    let language = Language::guess(Some(&paste.syntax_highlight), &paste.data);
+fn highlight_paste(paste: &Paste, data: &str) -> Result<String> {
+    let language = Language::guess(Some(&paste.syntax_highlight), data);
     let theme = themes::get("onedark").context("failed to load Lumis theme: onedark")?;
     let formatter = TerminalBuilder::new()
         .language(language)
         .theme(Some(theme))
         .build()
         .context("failed to build terminal syntax highlighter")?;
 
-    Ok(lumis::highlight(&paste.data, formatter))
+    Ok(lumis::highlight(data, formatter))
 }
 
 #[cfg(test)]
@@ -98,9 +102,11 @@ mod tests {
 
     fn paste(data: &str, syntax_highlight: &str) -> Paste {
         Paste {
-            data: data.to_string(),
+            data: data.as_bytes().to_vec(),
+            content_type: "text/plain".to_string(),
             syntax_highlight: syntax_highlight.to_string(),
             visibility: "private".to_string(),
+            is_text: true,
         }
     }
 
@@ -118,7 +124,7 @@ mod tests {
     fn write_raw_preserves_content_without_adding_a_newline() {
         let mut output = Vec::new();
 
-        write_raw(&mut output, "paste without newline").unwrap();
+        write_raw(&mut output, b"paste without newline").unwrap();
 
         assert_eq!(output, b"paste without newline");
     }
@@ -127,11 +133,20 @@ mod tests {
     fn write_raw_preserves_trailing_newlines() {
         let mut output = Vec::new();
 
-        write_raw(&mut output, "paste\n\n").unwrap();
+        write_raw(&mut output, b"paste\n\n").unwrap();
 
         assert_eq!(output, b"paste\n\n");
     }
 
+    #[test]
+    fn write_raw_preserves_arbitrary_binary_data() {
+        let mut output = Vec::new();
+
+        write_raw(&mut output, &[255, 0, 1]).unwrap();
+
+        assert_eq!(output, [255, 0, 1]);
+    }
+
     #[test]
     fn render_paste_without_color_returns_raw_data() {
         let paste = paste("fn main() {}\n", "rust");

crates/cli/src/base/show.rs

@@ -5,6 +5,7 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
+base64.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true

crates/client/Cargo.toml

@@ -1,7 +1,10 @@
+use base64::Engine as _;
 use reqwest::StatusCode;
 use reqwest::blocking::Body;
 use reqwest::header::{AUTHORIZATION, CONTENT_TYPE};
 use serde::Deserialize;
+use serde::Deserializer;
+use serde::de::Error as _;
 use std::collections::BTreeMap;
 use std::error;
 use std::fmt;
@@ -309,16 +312,74 @@ pub struct ApiTokenMetadata {
     pub inserted_at: String,
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone)]
 pub struct Paste {
-    pub data: String,
+    pub data: Vec<u8>,
+    pub content_type: String,
     pub syntax_highlight: String,
     pub visibility: String,
+    pub is_text: bool,
+}
+
+impl Paste {
+    pub fn text(&self) -> Option<&str> {
+        if self.is_text && !self.data.contains(&0) {
+            std::str::from_utf8(&self.data).ok()
+        } else {
+            None
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for Paste {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let paste = PasteResponse::deserialize(deserializer)?;
+        let (data, is_text) = match (
+            paste.data,
+            paste.data_base64,
+            paste.data_encoding.as_deref(),
+        ) {
+            (Some(data), None, None) => (data.into_bytes(), true),
+            (None, Some(data), Some("base64")) => base64::engine::general_purpose::STANDARD
+                .decode(data)
+                .map(|data| (data, false))
+                .map_err(D::Error::custom)?,
+            _ => return Err(D::Error::custom("invalid paste data representation")),
+        };
+
+        Ok(Self {
+            data,
+            content_type: paste.content_type,
+            syntax_highlight: paste.syntax_highlight,
+            visibility: paste.visibility,
+            is_text,
+        })
+    }
+}
+
+#[derive(Deserialize)]
+struct PasteResponse {
+    data: Option<String>,
+    data_base64: Option<String>,
+    data_encoding: Option<String>,
+    #[serde(default = "default_content_type")]
+    content_type: String,
+    syntax_highlight: String,
+    visibility: String,
+}
+
+fn default_content_type() -> String {
+    "text/plain".to_string()
 }
 
 #[derive(Debug, Clone, Deserialize)]
 pub struct CreatedPaste {
     pub id: String,
+    #[serde(default = "default_content_type")]
+    pub content_type: String,
     pub syntax_highlight: String,
     pub visibility: String,
     pub expires_at: Option<String>,
@@ -559,6 +620,48 @@ mod tests {
         assert_eq!(created.token.name, "Laptop");
     }
 
+    #[test]
+    fn decodes_text_and_binary_paste_representations() {
+        let text = decode_json::<ShowResponse>(
+            r#"{"data":{"data":"hello","content_type":"text/plain","syntax_highlight":"plain","visibility":"public"}}"#,
+        )
+        .unwrap()
+        .data;
+        let binary = decode_json::<ShowResponse>(
+            r#"{"data":{"data":null,"data_base64":"/wAB","data_encoding":"base64","content_type":"application/octet-stream","syntax_highlight":"plain","visibility":"public"}}"#,
+        )
+        .unwrap()
+        .data;
+
+        assert_eq!(text.data, b"hello");
+        assert_eq!(text.text(), Some("hello"));
+        assert_eq!(binary.data, [255, 0, 1]);
+        assert_eq!(binary.text(), None);
+    }
+
+    #[test]
+    fn base64_representation_remains_binary_for_utf8_shaped_bytes() {
+        for encoded in ["aGVsbG8=", "AA=="] {
+            let body = format!(
+                r#"{{"data":{{"data":null,"data_base64":"{encoded}","data_encoding":"base64","content_type":"application/octet-stream","syntax_highlight":"plain","visibility":"public"}}}}"#
+            );
+            let paste = decode_json::<ShowResponse>(&body).unwrap().data;
+
+            assert_eq!(paste.text(), None);
+        }
+    }
+
+    #[test]
+    fn defaults_content_type_for_older_server_responses() {
+        let paste = decode_json::<ShowResponse>(
+            r#"{"data":{"data":"hello","syntax_highlight":"plain","visibility":"public"}}"#,
+        )
+        .unwrap()
+        .data;
+
+        assert_eq!(paste.content_type, "text/plain");
+    }
+
     #[test]
     fn builds_canonical_paste_url() {
         let client = Client::new("https://demo.textbin.com/");

crates/client/src/lib.rs

@@ -13,7 +13,9 @@ defmodule Textbin.Application do
         Textbin.Repo,
         {DNSCluster, query: Application.get_env(:textbin, :dns_cluster_query) || :ignore},
         {Phoenix.PubSub, name: Textbin.PubSub}
-      ] ++ storage_children() ++ expiration_cleanup_children() ++ [TextbinWeb.Endpoint]
+      ] ++
+        storage_children() ++
+        expiration_cleanup_children() ++ upload_cleanup_children() ++ [TextbinWeb.Endpoint]
 
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options
@@ -37,6 +39,14 @@ defmodule Textbin.Application do
     end
   end
 
+  defp upload_cleanup_children do
+    if Application.get_env(:textbin, :upload_cleanup_enabled, true) do
+      [Textbin.Pastes.UploadCleaner]
+    else
+      []
+    end
+  end
+
   defp storage_children do
     case Textbin.Storage.child_spec() do
       nil -> []