implement text-{en,de}coder-streams

Cargo.toml

@@ -55,6 +55,7 @@ web-globals = { path = "crates/builtins/web-globals" }
 web-streams = { path = "crates/builtins/web-streams" }
 web-url = { path = "crates/builtins/web-url" }
 cpp-builtins = { path = "crates/builtins/cpp-builtins" }
+encoding = { path = "crates/builtins/encoding" }
 
 
 [profile.release]

crates/builtins/encoding/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "encoding"
+version.workspace = true
+repository.workspace = true
+license.workspace = true
+authors.workspace = true
+edition.workspace = true
+
+[dependencies]
+core-runtime.workspace = true
+encoding_rs.workspace = true
+js.workspace = true
+web-streams.workspace = true
+
+[dev-dependencies]
+js.workspace = true
+libstarling.workspace = true
+web-globals.workspace = true
+web-streams.workspace = true
+
+[lints]
+workspace = true

crates/builtins/encoding/src/decoder_common.rs

@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception
+
+/// Creates a new streaming decoder for the given encoding, with or without BOM handling.
+///
+/// Per the Encoding spec: ignoreBOM=true means "ignore the BOM's special meaning"
+/// (treat it as a regular U+FEFF character). ignoreBOM=false (default) means the
+/// BOM is detected and removed.
+pub fn new_decoder(
+    encoding: &'static encoding_rs::Encoding,
+    ignore_bom: bool,
+) -> encoding_rs::Decoder {
+    if ignore_bom {
+        encoding.new_decoder_without_bom_handling()
+    } else {
+        encoding.new_decoder_with_bom_removal()
+    }
+}
+
+/// Options dictionary for TextDecoder and TextDecoderStream constructors.
+#[core_runtime::webidl_dictionary]
+#[derive(Default)]
+pub struct TextDecoderOptions {
+    /// <https://encoding.spec.whatwg.org/#dom-textdecoderoptions-fatal>
+    /// When true, sets the error mode to "fatal", causing the decoder to throw
+    /// a TypeError on any malformed input. When false (default), the error mode
+    /// is "replacement", replacing malformed sequences with U+FFFD.
+    #[webidl(default = false)]
+    pub fatal: bool,
+
+    /// <https://encoding.spec.whatwg.org/#dom-textdecoderoptions-ignorebom>
+    /// When true, the decoder ignores the byte order mark (BOM). When false
+    /// (default), the decoder detects and removes the BOM. For UTF-16 encodings,
+    /// the BOM also determines byte order.
+    #[webidl(default = false, name = "ignoreBOM")]
+    pub ignore_bom: bool,
+}

crates/builtins/encoding/src/lib.rs

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception
+
+//! WHATWG Encoding Standard
+//! <https://encoding.spec.whatwg.org/>
+//!
+//! Provides `TextEncoder`, `TextDecoder`, `TextEncoderStream`, and `TextDecoderStream`.
+
+mod decoder_common;
+mod text_decoder;
+mod text_decoder_stream;
+mod text_encoder;
+mod text_encoder_stream;
+
+use js::gc::scope::Scope;
+use js::Object;
+
+pub fn add_to_global(scope: &Scope<'_>, global: Object<'_>) {
+    text_encoder::TextEncoder::add_to_global(scope, global);
+    text_decoder::TextDecoder::add_to_global(scope, global);
+    text_decoder_stream::TextDecoderStream::add_to_global(scope, global);
+    text_encoder_stream::TextEncoderStream::add_to_global(scope, global);
+}

crates/builtins/web-globals/src/encoding/text_decoder.rs → crates/builtins/encoding/src/text_decoder.rs

@@ -7,6 +7,9 @@ use js::error::{ExnThrown, RangeError};
 use js::gc::scope::Scope;
 use js::{ArrayBuffer, ArrayBufferView, JSString};
 
+use crate::decoder_common::new_decoder;
+use crate::decoder_common::TextDecoderOptions;
+
 /// <https://encoding.spec.whatwg.org/#textdecoder>
 #[webidl_interface]
 pub struct TextDecoder {
@@ -107,84 +110,88 @@ impl TextDecoder<'_> {
 
         // Step 3: If _input_ is given, then `push` a `copy of` _input_ to `this`'s `I/O queue`.
         //         (We decode directly from _input_'s bytes. Nothing retains them after this call.)
-        // SAFETY: `input` is rooted via the Uint8Array handle and the slice
-        // is only borrowed for the duration of the decode call below, which
-        // is pure CPU work in `encoding_rs` and cannot trigger GC or detach
-        // the underlying buffer.
+        // SAFETY: `input` is rooted, and the helper consumes the slice or copies it into
+        // Rust-owned output before any JS allocation can GC and move the input buffer.
         let bytes: &[u8] = match input {
             Some(ArrayBufferViewOrArrayBuffer::Buffer(input)) => unsafe { input.bytes() },
             Some(ArrayBufferViewOrArrayBuffer::View(input)) => unsafe { input.bytes() },
             None => &[],
         };
 
-        // windows-1252 decodes every byte outside 0x80..=0x9F to the code point of the same
-        // value, and its decoder keeps no state between calls, so such input is its own
-        // Latin-1 output.
-        if encoding == encoding_rs::WINDOWS_1252 && !bytes.iter().any(|b| (0x80..=0x9F).contains(b))
-        {
-            return JSString::from_latin1(scope, bytes);
-        }
-        // A fresh decoder that decodes its whole input at once turns ASCII into the same
-        // characters in every ASCII-compatible encoding.
-        if fresh
-            && last
-            && encoding.is_ascii_compatible()
-            && encoding_rs::Encoding::ascii_valid_up_to(bytes) == bytes.len()
-        {
-            return JSString::from_latin1(scope, bytes);
-        }
-
-        // Step 4: Let _output_ be the `I/O queue` of scalar values « `end-of-queue` ».
-        // Step 5: While true:
-        // Step 5.1: Let _item_ be the result of `reading` from `this`'s `I/O queue`.
-        // Step 5.2: If _item_ is `end-of-queue` and `this`'s `do not flush` is true, then return
-        //           the result of running `serialize I/O queue` with `this` and _output_.
-        // Step 5.3: Otherwise:
-        // Step 5.3.1: Let _result_ be the result of `processing an item` with _item_, `this`'s
-        //             `decoder`, `this`'s `I/O queue`, _output_, and `this`'s `error mode`.
-        // Step 5.3.2: If _result_ is `finished`, then return the result of running `serialize
-        //             I/O queue` with `this` and _output_.
-        // Step 5.3.3: Otherwise, if _result_ is `error`, `throw` a ``TypeError``.
-        //
-        // (One `encoding_rs` call processes every item. `last` is true when `end-of-queue` is
-        // processed rather than returned at step 5.2. The decoder's BOM removal performs
-        // `serialize I/O queue`'s BOM handling. We decode to UTF-16, which SpiderMonkey copies
-        // as is, or narrows to Latin-1 if every code unit fits.)
-        let max_len = decoder
-            .max_utf16_buffer_length(bytes.len())
-            .unwrap_or(bytes.len().saturating_add(1));
-        let mut output = vec![0u16; max_len];
-        if fatal {
-            let (result, _read, written) =
-                decoder.decode_to_utf16_without_replacement(bytes, &mut output, last);
-
-            match result {
-                encoding_rs::DecoderResult::InputEmpty => {
-                    JSString::from_utf16(scope, &output[..written])
-                }
-                encoding_rs::DecoderResult::Malformed(_, _) => Err(js::error::throw_type_error(
-                    scope,
-                    c"The encoded data was not valid.",
-                )),
-                encoding_rs::DecoderResult::OutputFull => {
-                    unreachable!("output buffer was pre-allocated to max size");
-                }
-            }
-        } else {
-            // Malformed sequences become U+FFFD.
-            let (_result, _read, written, _had_errors) =
-                decoder.decode_to_utf16(bytes, &mut output, last);
-
-            JSString::from_utf16(scope, &output[..written])
-        }
+        decode_bytes_to_js_string(scope, encoding, decoder, fatal, bytes, last, fresh)
     }
 }
 
-fn new_decoder(encoding: &'static encoding_rs::Encoding, ignore_bom: bool) -> encoding_rs::Decoder {
-    if ignore_bom {
-        encoding.new_decoder_without_bom_handling()
+pub(crate) fn decode_bytes_to_js_string<'r>(
+    scope: &'r Scope<'_>,
+    encoding: &'static encoding_rs::Encoding,
+    decoder: &mut encoding_rs::Decoder,
+    fatal: bool,
+    bytes: &[u8],
+    last: bool,
+    fresh: bool,
+) -> Result<JSString<'r>, ExnThrown> {
+    // windows-1252 decodes every byte outside 0x80..=0x9F to the code point of the same
+    // value, and its decoder keeps no state between calls, so such input is its own
+    // Latin-1 output.
+    if encoding == encoding_rs::WINDOWS_1252 && !bytes.iter().any(|b| (0x80..=0x9F).contains(b)) {
+        // GC during string allocation can move inline input buffer data.
+        let output = bytes.to_vec();
+        return JSString::from_latin1(scope, &output);
+    }
+    // A fresh decoder that decodes its whole input at once turns ASCII into the same
+    // characters in every ASCII-compatible encoding.
+    if fresh
+        && last
+        && encoding.is_ascii_compatible()
+        && encoding_rs::Encoding::ascii_valid_up_to(bytes) == bytes.len()
+    {
+        let output = bytes.to_vec();
+        return JSString::from_latin1(scope, &output);
+    }
+
+    // Step 4: Let _output_ be the `I/O queue` of scalar values « `end-of-queue` ».
+    // Step 5: While true:
+    // Step 5.1: Let _item_ be the result of `reading` from `this`'s `I/O queue`.
+    // Step 5.2: If _item_ is `end-of-queue` and `this`'s `do not flush` is true, then return
+    //           the result of running `serialize I/O queue` with `this` and _output_.
+    // Step 5.3: Otherwise:
+    // Step 5.3.1: Let _result_ be the result of `processing an item` with _item_, `this`'s
+    //             `decoder`, `this`'s `I/O queue`, _output_, and `this`'s `error mode`.
+    // Step 5.3.2: If _result_ is `finished`, then return the result of running `serialize
+    //             I/O queue` with `this` and _output_.
+    // Step 5.3.3: Otherwise, if _result_ is `error`, `throw` a ``TypeError``.
+    //
+    // (One `encoding_rs` call processes every item. `last` is true when `end-of-queue` is
+    // processed rather than returned at step 5.2. The decoder's BOM removal performs
+    // `serialize I/O queue`'s BOM handling. We decode to UTF-16, which SpiderMonkey copies
+    // as is, or narrows to Latin-1 if every code unit fits.)
+    let max_len = decoder
+        .max_utf16_buffer_length(bytes.len())
+        .unwrap_or(bytes.len().saturating_add(1));
+    let mut output = vec![0u16; max_len];
+    if fatal {
+        let (result, _read, written) =
+            decoder.decode_to_utf16_without_replacement(bytes, &mut output, last);
+
+        match result {
+            encoding_rs::DecoderResult::InputEmpty => {
+                JSString::from_utf16(scope, &output[..written])
+            }
+            encoding_rs::DecoderResult::Malformed(_, _) => Err(js::error::throw_type_error(
+                scope,
+                c"The encoded data was not valid.",
+            )),
+            encoding_rs::DecoderResult::OutputFull => {
+                unreachable!("output buffer was pre-allocated to max size");
+            }
+        }
     } else {
-        encoding.new_decoder_with_bom_removal()
+        // Malformed sequences become U+FFFD.
+        let (_result, _read, written, _had_errors) =
+            decoder.decode_to_utf16(bytes, &mut output, last);
+
+        JSString::from_utf16(scope, &output[..written])
     }
 }
 
@@ -199,12 +206,3 @@ pub struct TextDecodeOptions {
     #[webidl(default = false)]
     pub stream: bool,
 }
-
-#[webidl_dictionary]
-#[derive(Default)]
-pub struct TextDecoderOptions {
-    #[webidl(default = false)]
-    pub fatal: bool,
-    #[webidl(default = false, name = "ignoreBOM")]
-    pub ignore_bom: bool,
-}