Expose the transform-stream layer; UTF-8 text fast path; O(active) end-tag dispatch

.github/workflows/bun.yml

@@ -0,0 +1,42 @@
+name: bun
+
+# Runs the crate's own test suite against the `bun` branch. Upstream's
+# `ci.yml` only triggers on `master`; this file is additive so rebasing the
+# fork onto a new upstream tag does not have to re-edit `ci.yml`.
+on:
+  push:
+    branches:
+      - bun
+  pull_request:
+    branches:
+      - bun
+
+env:
+  RUST_BACKTRACE: 1
+  CARGO_INCREMENTAL: 0
+
+jobs:
+  test:
+    name: cargo test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          key: index-${{ hashFiles('Cargo.toml') }}
+          path: |
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+      - name: Fetch dependencies
+        run: cargo fetch
+      - name: Cache target directory
+        uses: actions/cache@v4
+        with:
+          key: target-bun-${{ hashFiles('Cargo.lock') }}
+          path: target
+      - name: Run tests
+        run: cargo test --features=integration_test --target-dir=target

README.md

@@ -1,5 +1,16 @@
 # LOL HTML
 
+> [!NOTE]
+> **This is Bun's fork of [`cloudflare/lol-html`](https://github.com/cloudflare/lol-html).** The
+> [`bun` branch](https://github.com/oven-sh/lol-html/tree/bun) adds **content-handler suspension**:
+> a handler can return `Err(SuspensionRequest)` to deep-copy the in-flight rewritable unit onto the
+> heap and exit `write()`/`end()` with the non-poisoning `RewritingError::Suspended`;
+> `HtmlRewriter::resume()` continues from a state-machine bookmark. Bun's `HTMLRewriter` uses this
+> so an `async` JS content handler can `await` and then keep mutating the element, without running
+> the parser on a side stack or spinning a nested event loop inside `write()`. See
+> [oven-sh/bun#36733](https://github.com/oven-sh/bun/pull/36733). The `main` branch tracks
+> upstream unchanged.
+
 <p align="center">
     <a href="https://github.com/cloudflare/lol-html">
         <img src="media/logo.png" alt="The logo is generated from https://openmoji.org/data/color/svg/1F602.svg by Emily Jäger which is licensed under CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)" style="width:472px; height: 375px" />

src/base/bytes.rs

@@ -59,6 +59,16 @@ impl<'b> BytesCow<'b> {
         BytesCow(Cow::Owned(self.0.into_owned()))
     }
 
+    /// Detaches the bytes from the input buffer they may borrow, leaving an
+    /// empty value behind. Used when a token has to outlive the parser's
+    /// input buffer (handler suspension).
+    #[inline]
+    pub(crate) fn take_owned(&mut self) -> BytesCow<'static> {
+        BytesCow(Cow::Owned(
+            std::mem::replace(&mut self.0, Cow::Borrowed(&[])).into_owned(),
+        ))
+    }
+
     #[inline]
     pub(crate) fn as_ref(&self) -> Bytes<'_> {
         Bytes(&self.0)
@@ -75,7 +85,7 @@ impl<'b> BytesCow<'b> {
 
 impl<'b> Bytes<'b> {
     #[inline]
-    pub(crate) fn new(bytes: &'b [u8]) -> Self {
+    pub(crate) const fn new(bytes: &'b [u8]) -> Self {
         Self(bytes)
     }
 

src/base/encoding.rs

@@ -67,6 +67,8 @@ fn encoding_to_index(encoding: AsciiCompatibleEncoding) -> usize {
 /// encounters a `meta` tag that specifies the charset (that behavior is dependent on
 /// [`crate::Settings::adjust_charset_on_meta_tag`]).
 // Pub only for integration tests
+/// An encoding handle shared by a [`TransformStream`](crate::transform::TransformStream)
+/// and the tokens it produces; it can change mid-document.
 #[derive(Clone)]
 pub struct SharedEncoding {
     encoding: Arc<AtomicUsize>,
@@ -75,20 +77,23 @@ pub struct SharedEncoding {
 impl SharedEncoding {
     #[must_use]
     #[cfg_attr(debug_assertions, track_caller)]
+    /// Wraps `encoding` for sharing between the stream and its tokens.
     pub fn new(encoding: AsciiCompatibleEncoding) -> Self {
         Self {
             encoding: Arc::new(AtomicUsize::new(encoding_to_index(encoding))),
         }
     }
 
     #[must_use]
+    /// The current encoding.
     pub fn get(&self) -> &'static Encoding {
         let encoding = self.encoding.load(Ordering::Relaxed);
         // it will never be out of range, but get() avoids a panic branch
         ALL_ENCODINGS.get(encoding).unwrap_or(&ALL_ENCODINGS[0])
     }
 
     #[cfg_attr(debug_assertions, track_caller)]
+    /// Switches the encoding (a `<meta charset>` was seen).
     pub fn set(&self, encoding: AsciiCompatibleEncoding) {
         self.encoding
             .store(encoding_to_index(encoding), Ordering::Relaxed);

src/base/spanned.rs

@@ -48,19 +48,24 @@ pub(crate) struct Spanned<B> {
     source_location_byte_start: usize,
 }
 
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Clone)]
 pub(crate) enum RawBytes<'input> {
     Original(&'input [u8]),
+    /// Raw bytes copied out of the input buffer so that they can outlive it.
+    /// Produced by [`SpannedRawBytes::take_owned`] when a token is detached
+    /// from the parser (handler suspension).
+    Owned(Box<[u8]>),
     /// Keeps the len of the original
     Modified(usize),
 }
 
 impl<'input> SpannedRawBytes<'input> {
     #[inline]
     pub fn len(&self) -> usize {
-        match self.bytes {
+        match &self.bytes {
             RawBytes::Original(s) => s.len(),
-            RawBytes::Modified(l) => l,
+            RawBytes::Owned(s) => s.len(),
+            RawBytes::Modified(l) => *l,
         }
     }
 
@@ -71,13 +76,30 @@ impl<'input> SpannedRawBytes<'input> {
     }
 
     #[inline]
-    pub fn original(&self) -> Option<&'input [u8]> {
-        match self.bytes {
+    pub fn original(&self) -> Option<&[u8]> {
+        match &self.bytes {
             RawBytes::Original(s) => Some(s),
+            RawBytes::Owned(s) => Some(s),
             RawBytes::Modified(_) => None,
         }
     }
 
+    /// Detaches the raw bytes from the input buffer, leaving `self` behind as
+    /// `Modified(len)` (the source is abandoned after a `take_owned`, so the
+    /// leftover only has to keep `len()`/`source_location()` correct).
+    pub(crate) fn take_owned(&mut self) -> SpannedRawBytes<'static> {
+        let len = self.len();
+        let bytes = match std::mem::replace(&mut self.bytes, RawBytes::Modified(len)) {
+            RawBytes::Original(s) => RawBytes::Owned(s.into()),
+            RawBytes::Owned(s) => RawBytes::Owned(s),
+            RawBytes::Modified(l) => RawBytes::Modified(l),
+        };
+        Spanned {
+            bytes,
+            source_location_byte_start: self.source_location_byte_start,
+        }
+    }
+
     #[inline]
     pub fn source_location(&self) -> SourceLocation {
         SourceLocation::from_start_len(self.source_location_byte_start, self.len())

src/html/local_name.rs

@@ -31,7 +31,8 @@ use std::fmt;
 // to be able to invalidate itself if parsing an unrepresentable name.
 // `EMPTY_HASH` is used as a sentinel value.
 //
-// Pub only for integration tests
+/// A tag name of up to twelve `[a-zA-Z1-6]` characters packed five bits per
+/// character into a `u64`; compares as an integer. See the notes above.
 #[derive(PartialEq, Eq, Copy, Clone, Default, Hash)]
 pub struct LocalNameHash(u64);
 
@@ -40,18 +41,21 @@ const EMPTY_HASH: u64 = !0;
 impl LocalNameHash {
     #[inline]
     #[must_use]
+    /// The hash of the empty name.
     pub const fn new() -> Self {
         Self(0)
     }
 
     #[inline]
     #[must_use]
+    /// Whether the name was not representable (too long, or other characters).
     pub const fn is_empty(&self) -> bool {
         self.0 == EMPTY_HASH
     }
 
+    /// Appends one character.
     #[inline]
-    pub fn update(&mut self, ch: u8) {
+    pub const fn update(&mut self, ch: u8) {
         let h = self.0;
 
         // NOTE: check if we still have space for yet another
@@ -67,12 +71,12 @@ impl LocalNameHash {
                 // upper bits which we eliminate with the mask). Then add
                 // 5, since numbers from 0 to 5 are reserved for digits.
                 // Aftwerards put result as 5 lower bits of the hash.
-                b'a'..=b'z' | b'A'..=b'Z' => (h << 5) | ((u64::from(ch) & 0x1F) + 5),
+                b'a'..=b'z' | b'A'..=b'Z' => (h << 5) | ((ch as u64 & 0x1F) + 5),
 
                 // NOTE: apply 0x0F mask on ASCII digit to convert it to number
                 // from 1 to 6. Then subtract 1 to make it zero-based.
                 // Afterwards, put result as lower bits of the hash.
-                b'1'..=b'6' => (h << 5) | ((u64::from(ch) & 0x0F) - 1),
+                b'1'..=b'6' => (h << 5) | ((ch as u64 & 0x0F) - 1),
 
                 // NOTE: for any other characters hash function is not
                 // applicable, so we completely invalidate the hash.
@@ -84,43 +88,70 @@ impl LocalNameHash {
     }
 }
 
-impl fmt::Debug for LocalNameHash {
-    #[cold]
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if self.is_empty() {
-            return f.write_str("N/A");
+impl LocalNameHash {
+    /// Hashes a complete name; usable in `const` context, e.g. to build
+    /// `match` arms over [`as_u64`](Self::as_u64) values.
+    #[must_use]
+    pub const fn from_ascii(name: &[u8]) -> Self {
+        let mut hash = Self::new();
+        let mut i = 0;
+        while i < name.len() {
+            hash.update(name[i]);
+            i += 1;
         }
+        hash
+    }
 
-        let mut reverse_buf = [0u8; 12];
-        let mut pos = 11;
+    /// The packed value (see the encoding notes above); `!0` when the name
+    /// was not representable.
+    #[inline]
+    #[must_use]
+    pub const fn as_u64(&self) -> u64 {
+        self.0
+    }
+
+    /// Unpacks the hash back into the ASCII-lowercase tag name it encodes.
+    /// The encoding is reversible for every representable name (the first
+    /// character of a tag name is always a letter, so leading zero groups are
+    /// unambiguous padding). Returns `None` for the empty/invalid hash.
+    #[inline]
+    pub fn decode<'b>(&self, buf: &'b mut [u8; 12]) -> Option<&'b str> {
+        if self.is_empty() || self.0 == 0 {
+            return None;
+        }
+        let mut pos = 12;
         let mut h = self.0;
         loop {
-            reverse_buf[pos] = match (h & 31) as u8 {
+            pos -= 1;
+            buf[pos] = match (h & 31) as u8 {
                 v @ 6.. => v + (b'a' - 6),
                 v => v + b'1',
             };
             h >>= 5;
             if h == 0 || pos == 0 {
                 break;
             }
-            pos -= 1;
         }
-        std::str::from_utf8(&reverse_buf[pos..])
-            .unwrap_or_default()
-            .fmt(f)
+        // Only ASCII letters and digits were written.
+        std::str::from_utf8(&buf[pos..]).ok()
+    }
+}
+
+impl fmt::Debug for LocalNameHash {
+    #[cold]
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut buf = [0u8; 12];
+        match self.decode(&mut buf) {
+            Some(name) => name.fmt(f),
+            None => f.write_str("N/A"),
+        }
     }
 }
 
 impl From<&str> for LocalNameHash {
     #[inline]
     fn from(string: &str) -> Self {
-        let mut hash = Self::new();
-
-        for ch in string.bytes() {
-            hash.update(ch);
-        }
-
-        hash
+        Self::from_ascii(string.as_bytes())
     }
 }
 
@@ -134,12 +165,26 @@ impl PartialEq<Tag> for LocalNameHash {
 /// `LocalName` is used for the comparison of tag names.
 /// In the majority of cases it will be represented as a hash, however for long
 /// non-standard tag names it fallsback to the Name representation.
-#[derive(Clone, Debug, Eq, Hash)]
+#[derive(Clone, Debug, Eq)]
 pub enum LocalName<'i> {
+    /// A name representable as a [`LocalNameHash`].
     Hash(LocalNameHash),
+    /// Any other name, as written (compared ASCII case-insensitively).
     Bytes(BytesCow<'i>),
 }
 
+// `PartialEq` compares `Bytes` case-insensitively, so `Hash` must case-fold too.
+impl std::hash::Hash for LocalName<'_> {
+    #[inline]
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        core::mem::discriminant(self).hash(state);
+        match self {
+            LocalName::Hash(h) => h.hash(state),
+            LocalName::Bytes(b) => b.iter().for_each(|c| c.to_ascii_lowercase().hash(state)),
+        }
+    }
+}
+
 impl<'i> LocalName<'i> {
     #[inline]
     #[must_use]
@@ -153,6 +198,7 @@ impl<'i> LocalName<'i> {
 
     #[inline]
     #[must_use]
+    /// Detaches the name from the input buffer.
     pub fn into_owned(self) -> LocalName<'static> {
         match self {
             LocalName::Bytes(b) => LocalName::Bytes(b.into_owned()),
@@ -161,6 +207,8 @@ impl<'i> LocalName<'i> {
     }
 
     #[inline]
+    /// Builds a name for comparison; fails if `string` is not representable
+    /// in `encoding`.
     pub fn from_str_without_replacements<'s>(
         string: &'s str,
         encoding: &'static Encoding,
@@ -219,4 +267,15 @@ mod tests {
     fn hash_invalidation_for_long_values() {
         assert!(LocalNameHash::from("aaaaaaaaaaaaaa").is_empty());
     }
+
+    #[test]
+    fn bytes_variant_hash_matches_case_insensitive_eq() {
+        use std::hash::{BuildHasher, RandomState};
+        let s = RandomState::new();
+        let a = LocalName::from_str_without_replacements("My-Widget", encoding_rs::UTF_8).unwrap();
+        let b = LocalName::from_str_without_replacements("my-widget", encoding_rs::UTF_8).unwrap();
+        assert!(matches!(a, LocalName::Bytes(_)));
+        assert_eq!(a, b);
+        assert_eq!(s.hash_one(&a), s.hash_one(&b));
+    }
 }