From 5d6c69da25453e8d64e8c259b8c697573340bd56 Mon Sep 17 00:00:00 2001 From: Karthik Ganeshram Date: Tue, 14 Jul 2026 13:48:20 +0200 Subject: [PATCH 1/5] implement text-{en,de}coder-streams This commit moves the existing TextDecoder and TextEncoder implementations into a new `encoding` crate, and adds the corresponding stream classes. Signed-off-by: Karthik Ganeshram --- Cargo.lock | 14 +- Cargo.toml | 1 + crates/builtins/encoding/Cargo.toml | 22 + .../builtins/encoding/src/decoder_common.rs | 37 + crates/builtins/encoding/src/lib.rs | 22 + .../encoding => encoding/src}/text_decoder.rs | 22 +- .../encoding/src/text_decoder_stream.rs | 365 +++++++++ .../encoding => encoding/src}/text_encoder.rs | 0 .../encoding/src/text_encoder_stream.rs | 272 +++++++ crates/builtins/encoding/tests/encoding.rs | 707 ++++++++++++++++++ crates/builtins/web-globals/Cargo.toml | 1 - .../builtins/web-globals/src/encoding/mod.rs | 12 - crates/builtins/web-globals/src/lib.rs | 2 - crates/builtins/web-globals/tests/encoding.rs | 37 - crates/builtins/web-streams/src/lib.rs | 1 + .../builtins/web-streams/src/transform/mod.rs | 4 +- .../src/transform/transform_stream.rs | 18 +- crates/libstarling/Cargo.toml | 1 + crates/libstarling/src/lib.rs | 1 + .../encoding/idlharness.any.js.json | 34 +- .../encoding/streams/backpressure.any.js.json | 14 + .../streams/decode-attributes.any.js.json | 86 +++ .../streams/decode-bad-chunks.any.js.json | 17 + .../streams/decode-ignore-bom.any.js.json | 38 + .../decode-incomplete-input.any.js.json | 8 + .../streams/decode-non-utf8.any.js.json | 41 + .../decode-split-character.any.js.json | 35 + .../streams/encode-bad-chunks.any.js.json | 20 + .../encoding/streams/encode-utf8.any.js.json | 59 ++ .../readable-writable-properties.any.js.json | 8 + tests/wpt-harness/tests.json | 11 + 31 files changed, 1817 insertions(+), 93 deletions(-) create mode 100644 crates/builtins/encoding/Cargo.toml create mode 100644 crates/builtins/encoding/src/decoder_common.rs create mode 100644 crates/builtins/encoding/src/lib.rs rename crates/builtins/{web-globals/src/encoding => encoding/src}/text_decoder.rs (94%) create mode 100644 crates/builtins/encoding/src/text_decoder_stream.rs rename crates/builtins/{web-globals/src/encoding => encoding/src}/text_encoder.rs (100%) create mode 100644 crates/builtins/encoding/src/text_encoder_stream.rs create mode 100644 crates/builtins/encoding/tests/encoding.rs delete mode 100644 crates/builtins/web-globals/src/encoding/mod.rs delete mode 100644 crates/builtins/web-globals/tests/encoding.rs create mode 100644 tests/wpt-harness/expectations/encoding/streams/backpressure.any.js.json create mode 100644 tests/wpt-harness/expectations/encoding/streams/decode-attributes.any.js.json create mode 100644 tests/wpt-harness/expectations/encoding/streams/decode-bad-chunks.any.js.json create mode 100644 tests/wpt-harness/expectations/encoding/streams/decode-ignore-bom.any.js.json create mode 100644 tests/wpt-harness/expectations/encoding/streams/decode-incomplete-input.any.js.json create mode 100644 tests/wpt-harness/expectations/encoding/streams/decode-non-utf8.any.js.json create mode 100644 tests/wpt-harness/expectations/encoding/streams/decode-split-character.any.js.json create mode 100644 tests/wpt-harness/expectations/encoding/streams/encode-bad-chunks.any.js.json create mode 100644 tests/wpt-harness/expectations/encoding/streams/encode-utf8.any.js.json create mode 100644 tests/wpt-harness/expectations/encoding/streams/readable-writable-properties.any.js.json diff --git a/Cargo.lock b/Cargo.lock index e01c821c..a282825f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -532,6 +532,18 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "encoding" +version = "0.3.0" +dependencies = [ + "core-runtime", + "encoding_rs", + "js", + "libstarling", + "web-globals", + "web-streams", +] + [[package]] name = "encoding_c" version = "0.9.8" @@ -1508,6 +1520,7 @@ dependencies = [ "bytes", "core-runtime", "cpp-builtins", + "encoding", "event-listener", "fetch-event", "futures-channel", @@ -3103,7 +3116,6 @@ dependencies = [ "bitflags", "core-runtime", "data-url", - "encoding_rs", "js", "libstarling", "platform", diff --git a/Cargo.toml b/Cargo.toml index 8dfddadb..3a8834a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ web-globals = { path = "crates/builtins/web-globals" } web-streams = { path = "crates/builtins/web-streams" } web-url = { path = "crates/builtins/web-url" } cpp-builtins = { path = "crates/builtins/cpp-builtins" } +encoding = { path = "crates/builtins/encoding" } [profile.release] lto = "thin" diff --git a/crates/builtins/encoding/Cargo.toml b/crates/builtins/encoding/Cargo.toml new file mode 100644 index 00000000..22e2d62d --- /dev/null +++ b/crates/builtins/encoding/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "encoding" +version.workspace = true +repository.workspace = true +license.workspace = true +authors.workspace = true +edition.workspace = true + +[dependencies] +core-runtime.workspace = true +encoding_rs.workspace = true +js.workspace = true +web-streams.workspace = true + +[dev-dependencies] +js.workspace = true +libstarling.workspace = true +web-globals.workspace = true +web-streams.workspace = true + +[lints] +workspace = true diff --git a/crates/builtins/encoding/src/decoder_common.rs b/crates/builtins/encoding/src/decoder_common.rs new file mode 100644 index 00000000..927153bc --- /dev/null +++ b/crates/builtins/encoding/src/decoder_common.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +/// Creates a new streaming decoder for the given encoding, with or without BOM handling. +/// +/// Per the Encoding spec: ignoreBOM=true means "ignore the BOM's special meaning" +/// (treat it as a regular U+FEFF character). ignoreBOM=false (default) means the +/// BOM is detected and removed. +pub fn new_decoder( + encoding: &'static encoding_rs::Encoding, + ignore_bom: bool, +) -> encoding_rs::Decoder { + if ignore_bom { + encoding.new_decoder_without_bom_handling() + } else { + encoding.new_decoder_with_bom_removal() + } +} + + +/// Options dictionary for TextDecoder and TextDecoderStream constructors. +#[core_runtime::webidl_dictionary] +#[derive(Default)] +pub struct TextDecoderOptions { + /// + /// When true, sets the error mode to "fatal", causing the decoder to throw + /// a TypeError on any malformed input. When false (default), the error mode + /// is "replacement", replacing malformed sequences with U+FFFD. + #[webidl(default = false)] + pub fatal: bool, + + /// + /// When true, the decoder ignores the byte order mark (BOM). When false + /// (default), the decoder detects and removes the BOM. For UTF-16 encodings, + /// the BOM also determines byte order. + #[webidl(default = false, name = "ignoreBOM")] + pub ignore_bom: bool, +} diff --git a/crates/builtins/encoding/src/lib.rs b/crates/builtins/encoding/src/lib.rs new file mode 100644 index 00000000..d1e9d812 --- /dev/null +++ b/crates/builtins/encoding/src/lib.rs @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +//! WHATWG Encoding Standard +//! +//! +//! Provides `TextEncoder`, `TextDecoder`, `TextEncoderStream`, and `TextDecoderStream`. + +mod decoder_common; +mod text_decoder; +mod text_decoder_stream; +mod text_encoder; +mod text_encoder_stream; + +use js::gc::scope::Scope; +use js::Object; + +pub fn add_to_global(scope: &Scope<'_>, global: Object<'_>) { + text_encoder::TextEncoder::add_to_global(scope, global); + text_decoder::TextDecoder::add_to_global(scope, global); + text_decoder_stream::TextDecoderStream::add_to_global(scope, global); + text_encoder_stream::TextEncoderStream::add_to_global(scope, global); +} diff --git a/crates/builtins/web-globals/src/encoding/text_decoder.rs b/crates/builtins/encoding/src/text_decoder.rs similarity index 94% rename from crates/builtins/web-globals/src/encoding/text_decoder.rs rename to crates/builtins/encoding/src/text_decoder.rs index e7242a44..cf74d91c 100644 --- a/crates/builtins/web-globals/src/encoding/text_decoder.rs +++ b/crates/builtins/encoding/src/text_decoder.rs @@ -7,6 +7,9 @@ use js::error::{ExnThrown, OutOfMemory, RangeError, ThrowException}; use js::gc::scope::Scope; use js::{ArrayBuffer, ArrayBufferView, JSString}; +use crate::decoder_common::new_decoder; +use crate::decoder_common::TextDecoderOptions; + /// #[webidl_interface] pub struct TextDecoder { @@ -172,16 +175,6 @@ enum Decoded { Utf16(Result, DecodeError>), } -/// A new decoder of `encoding`, which removes a byte order mark at the start of its input unless -/// `ignore_bom` is true. -fn new_decoder(encoding: &'static encoding_rs::Encoding, ignore_bom: bool) -> encoding_rs::Decoder { - if ignore_bom { - encoding.new_decoder_without_bom_handling() - } else { - encoding.new_decoder_with_bom_removal() - } -} - /// Why [`decode_to_utf16`] failed. enum DecodeError { /// The input is malformed, in the "fatal" error mode. @@ -248,12 +241,3 @@ pub struct TextDecodeOptions { #[webidl(default = false)] pub stream: bool, } - -#[webidl_dictionary] -#[derive(Default)] -pub struct TextDecoderOptions { - #[webidl(default = false)] - pub fatal: bool, - #[webidl(default = false, name = "ignoreBOM")] - pub ignore_bom: bool, -} diff --git a/crates/builtins/encoding/src/text_decoder_stream.rs b/crates/builtins/encoding/src/text_decoder_stream.rs new file mode 100644 index 00000000..b6592345 --- /dev/null +++ b/crates/builtins/encoding/src/text_decoder_stream.rs @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +//! +//! +//! TextDecoderStream is a TransformStream that decodes bytes to strings using +//! a configurable encoding. It maintains decoder state across chunks to handle +//! incomplete multi-byte sequences at chunk boundaries. + +use core_runtime::{webidl_interface, webidl_methods}; +use js::error::ExnThrown; +use js::gc::handle::Heap; +use js::gc::scope::Scope; +use js::native::Value; +use js::prelude::{CallbackArgs, HandleValue}; +use js::{ArrayBuffer, ArrayBufferView, Function, Object, value}; + +use web_streams::readable::ReadableStream; +use web_streams::{TransformStream, TransformStreamImpl}; +use web_streams::writable::WritableStream; + +use crate::decoder_common::{new_decoder, TextDecoderOptions}; + +/// +#[webidl_interface] +pub struct TextDecoderStream { + /// The encoding_rs encoding (always `Some` after construction). + #[no_trace] + encoding: Option<&'static encoding_rs::Encoding>, + + /// The encoding name, ASCII lowercased. + encoding_name: String, + + /// + /// true = "fatal" mode (throw on error), false = "replacement" mode. + fatal: bool, + + /// + #[no_trace(name = "ignoreBOM")] + ignore_bom: bool, + + // Note: The spec defines an I/O queue (https://encoding.spec.whatwg.org/#textdecodercommon-i-o-queue) + // to buffer incomplete multi-byte sequences across chunks. We omit it because encoding_rs's + // Decoder is stateful and tracks partial sequences internally. Per the Decoder contract + // (https://docs.rs/encoding_rs/0.8/encoding_rs/struct.Decoder.html), `src` is only + // partially consumed on `OutputFull` or `Malformed`; since we pre-allocate to prevent + // `OutputFull` and treat `Malformed` as a fatal error, the result is always `InputEmpty` + // with `read == src.len()`. Passing each chunk's bytes directly to the decoder is + // equivalent to the spec's queue-then-process pattern. + + /// + /// The decoder instance, preserving state between chunks. + #[no_trace] + decoder: Option, + + /// The underlying TransformStream. + transform: Heap, +} + +#[webidl_methods] +impl TextDecoderStream { + /// + /// 4. Set up a text decoder stream with this, encoding, errorMode, and + /// options["ignoreBOM"]. + /// + /// "Set up a text decoder stream" initializes all internal slots and creates + /// the TransformStream with the decode and flush algorithms. + #[constructor] + fn new( + &self, + scope: &Scope<'_>, + label: Option, + options: Option, + ) -> Result<(), ExnThrown> { + // Step 1: Let encoding be the result of getting an encoding from label. + let label = label.as_deref().unwrap_or("utf-8"); + let encoding = + encoding_rs::Encoding::for_label_no_replacement(label.as_bytes()).ok_or_else(|| { + js::error::throw_range_error(scope, c"The encoding label provided is invalid.") + })?; + + // Step 2: If encoding is failure or replacement, then throw a RangeError. + if encoding == encoding_rs::REPLACEMENT { + return Err(js::error::throw_range_error( + scope, + c"The replacement or failure encoding cannot be used.", + )); + } + + // Step 3: Let errorMode be "fatal" if options["fatal"] is true; otherwise "replacement". + let options = options.unwrap_or_default(); + let fatal = options.fatal; + let ignore_bom = options.ignore_bom; + + // Step 4: Set up a text decoder stream with this, encoding, errorMode, and options["ignoreBOM"]. + // Step 4.1: Assert: encoding is not replacement. (checked above when we throw) + // Step 4.2 Set stream's encoding to encoding. + self.data_mut().encoding = Some(encoding); + self.data_mut().encoding_name = encoding.name().to_ascii_lowercase(); + // Step 4.3 Set stream's error mode to errorMode. + self.data_mut().fatal = fatal; + // Step 4.4 Set stream's ignore BOM to options["ignoreBOM"]. + self.data_mut().ignore_bom = ignore_bom; + // Step 4.5 Set stream’s decoder to a new instance of encoding’s decoder. + self.data_mut().decoder = Some(new_decoder(encoding, ignore_bom)); + // Step 4.6: Set stream's I/O queue to a new I/O queue. + // (Omitted — encoding_rs's Decoder buffers partial sequences internally; + // see the comment on the struct definition.) + + let self_payload = scope.root_value(self.as_value()); + // Step 4.7: Let transformAlgorithm be an algorithm which takes a chunk argument and runs the decode and enqueue a chunk algorithm with stream and chunk. + let transform_cb = Function::new_callback( + scope, + c"textDecoderTransform", + 1, + decoder_transform_cb, + self_payload, + )?; + + // Step 4.8: Let flushAlgorithm be an algorithm which takes no arguments and runs the flush and enqueue algorithm with stream. + let flush_cb = Function::new_callback( + scope, + c"textDecoderFlush", + 0, + decoder_flush_cb, + self_payload, + )?; + + // Step 4.9: Let transformStream be a new TransformStream. + // Step 4.10: Set up transformStream with transformAlgorithm set to transformAlgorithm and flushAlgorithm set to flushAlgorithm. + // We pass both algorithms as properties on a plain transformer object so that the + // TransformStream constructor sees a real transformer and does not set the + // identity-transform shortcut — otherwise `pipeTo` would bypass decoding for native + // byte sources. + let transformer = Object::new_plain(scope)?; + transformer.set_property(scope, c"transform", transform_cb)?; + transformer.set_property(scope, c"flush", flush_cb)?; + let transform_stream = TransformStream::new(scope, Some(scope.root_value(transformer.as_value())), None, None)?; + + // Step 4.11: Set stream’s transform to transformStream. + self.data_mut().transform.set(transform_stream); + + Ok(()) + } + + /// + #[getter] + fn encoding(&self) -> String { + self.data().encoding_name.clone() + } + + /// + #[getter] + fn fatal(&self) -> bool { + self.data().fatal + } + + /// + #[getter(name = "ignoreBOM")] + fn ignore_bom(&self) -> bool { + self.data().ignore_bom + } + + /// + #[getter] + fn readable<'r>(&self, scope: &'r Scope<'_>) -> ReadableStream<'r> { + self.data().transform.get(scope).readable(scope) + } + + /// + #[getter] + fn writable<'r>(&self, scope: &'r Scope<'_>) -> WritableStream<'r> { + self.data().transform.get(scope).writable(scope) + } +} + +/// The algorithm. +fn decoder_transform_cb( + scope: &Scope<'_>, + args: CallbackArgs<'_>, + payload: HandleValue<'_>, +) -> Result { + let stream: TextDecoderStream<'_> = Object::from_value(scope, *payload) + .expect("payload is an object") + .cast::>() + .expect("payload is a TextDecoderStream"); + + // Step 1: Let bufferSource be the result of converting chunk to an AllowSharedBufferSource. + let chunk = args.get(0); + let bytes = buffer_source_to_bytes(scope, chunk)?; + + // Step 2: Push a copy of bufferSource to decoder's I/O queue. + // Step 3: Let output be the I/O queue of scalar values « end-of-queue ». + // Step 4: Process I/O queue. + // (Steps 2-4 are collapsed: per the encoding_rs Decoder contract + // (https://docs.rs/encoding_rs/0.8/encoding_rs/struct.Decoder.html), `src` is fully + // consumed when the result is `InputEmpty`, which is guaranteed here by pre-allocating + // via `max_utf8_buffer_length`. The Decoder tracks partial multi-byte sequences + // internally, so we pass the chunk directly without queuing.) + let mut output = String::new(); + + if !bytes.is_empty() { + let mut data = stream.data_mut(); + let fatal = data.fatal; + let decoder = data.decoder.as_mut().expect("decoder must be initialized"); + + if fatal { + // Step 4.3: Process through fatal decoder. + if let Some(needed) = decoder.max_utf8_buffer_length_without_replacement(bytes.len()) { + output.reserve(needed); + } + let (result, _read) = + decoder.decode_to_string_without_replacement(&bytes, &mut output, false); + + match result { + encoding_rs::DecoderResult::InputEmpty => {} + // Step 4.4: If error, throw a TypeError. + encoding_rs::DecoderResult::Malformed(_, _) => { + return Err(js::error::throw_type_error( + scope, + c"The encoded data was not valid.", + )); + } + encoding_rs::DecoderResult::OutputFull => { + unreachable!("Output buffer was too small despite max_utf8_buffer_length_without_replacement allocation"); + } + } + } else { + // Step 4.3: Process through replacement decoder. + if let Some(needed) = decoder.max_utf8_buffer_length(bytes.len()) { + output.reserve(needed); + } + match decoder.decode_to_string(&bytes, &mut output, false) { + (encoding_rs::CoderResult::InputEmpty, _, _) => {} + (encoding_rs::CoderResult::OutputFull, _, _) => { + unreachable!("Output buffer was too small despite max_utf8_buffer_length allocation"); + } + } + } + } + + // Step 4.2.1 & 4.2.2: If outputChunk is not the empty string, then enqueue outputChunk. + if !output.is_empty() { + let chunk_val = js::JSString::from_str(scope, &output) + .map_err(|_| js::error::throw_type_error(scope, c"unable to create string"))?; + stream + .data() + .transform + .get(scope) + .enqueue(scope, scope.root_value(chunk_val.as_value()))?; + } + + // Step 4.2.3: Return. + Ok(value::undefined()) +} + +/// The algorithm. +fn decoder_flush_cb( + scope: &Scope<'_>, + _args: CallbackArgs<'_>, + payload: HandleValue<'_>, +) -> Result { + let stream: TextDecoderStream<'_> = Object::from_value(scope, *payload) + .expect("payload is an object") + .cast::>() + .expect("payload is a TextDecoderStream"); + + // Step 1: Let output be the I/O queue of scalar values « end-of-queue ». + // (No I/O queue to drain — per the encoding_rs Decoder contract + // (https://docs.rs/encoding_rs/0.8/encoding_rs/struct.Decoder.html), partial sequences + // are held in the Decoder's internal state. We pass an empty slice with `last=true` + // to flush that state.) + let mut output = String::new(); + + { + let mut data = stream.data_mut(); + let fatal = data.fatal; + let decoder = data.decoder.as_mut().expect("decoder must be initialized"); + + // Step 2: While true: + // (Instead of a manual byte-by-byte loop, we pass `last = true` to encoding_rs. + // This instructs the native decoder to finalize, process any dangling bytes, + // and reset its shift or multibyte state. This acts as our loop termination). + if fatal { + // Pre-reserve using the without-replacement bound. + if let Some(needed) = decoder.max_utf8_buffer_length_without_replacement(0) { + output.reserve(needed); + } + // Step 2.2: Let result be processing an item with item, decoder, output, and error mode. + let (result, _read) = + decoder.decode_to_string_without_replacement(&[], &mut output, true); + + match result { + // Step 2.3: If result is finished + encoding_rs::DecoderResult::InputEmpty => {} + // Step 2.4: If result is error, throw a TypeError. + encoding_rs::DecoderResult::Malformed(_, _) => { + return Err(js::error::throw_type_error( + scope, + c"The encoded data was not valid.", + )); + } + encoding_rs::DecoderResult::OutputFull => { + unreachable!("Output buffer was too small during fatal flush"); + } + } + } else { + // Pre-reserve using the with-replacement bound (accounts for pending U+FFFD output). + if let Some(needed) = decoder.max_utf8_buffer_length(0) { + output.reserve(needed); + } + // Step 2.2: Let result be processing an item with item, decoder, output, and error mode. + match decoder.decode_to_string(&[], &mut output, true) { + (encoding_rs::CoderResult::InputEmpty, _, _) => {} + (encoding_rs::CoderResult::OutputFull, _, _) => { + unreachable!("Output buffer was too small during replacement flush"); + } + } + } + } + + // Step 2.3.1: Let outputChunk be the result of running serialize I/O queue with decoder and output.(handled by encoding_rs) + // Step 2.3.2: If outputChunk is not the empty string, then enqueue outputChunk in decoder's transform. + if !output.is_empty() { + let chunk_val = js::JSString::from_str(scope, &output) + .map_err(|_| js::error::throw_type_error(scope, c"unable to create string"))?; + stream + .data() + .transform + .get(scope) + .enqueue(scope, scope.root_value(chunk_val.as_value()))?; + } + + // Step 2.3.3: Return. + Ok(value::undefined()) +} + +fn buffer_source_to_bytes( + scope: &Scope<'_>, + value: HandleValue<'_>, +) -> Result, ExnThrown> { + let obj = Object::from_value(scope, *value).map_err(|_| { + js::error::throw_type_error(scope, c"argument is not an ArrayBuffer or ArrayBufferView") + })?; + + // Try ArrayBufferView first (typed arrays, DataView). + if let Ok(view) = obj.cast::>() { + // SAFETY: `view` is rooted; we copy the bytes immediately so the + // borrow doesn't outlive any GC-triggering operation. The copy also + // satisfies the spec requirement for SharedArrayBuffer-backed views. + return Ok(unsafe { view.bytes() }.to_vec()); + } + + // Try ArrayBuffer. + if let Ok(buf) = obj.cast::>() { + // SAFETY: `buf` is rooted; same constraints as above. + // Note: If the ArrayBuffer is detached, bytes() returns an empty slice. + return Ok(unsafe { buf.bytes() }.to_vec()); + } + + // Not a valid buffer source. + Err(js::error::throw_type_error( + scope, + c"argument is not an ArrayBuffer or ArrayBufferView", + )) +} diff --git a/crates/builtins/web-globals/src/encoding/text_encoder.rs b/crates/builtins/encoding/src/text_encoder.rs similarity index 100% rename from crates/builtins/web-globals/src/encoding/text_encoder.rs rename to crates/builtins/encoding/src/text_encoder.rs diff --git a/crates/builtins/encoding/src/text_encoder_stream.rs b/crates/builtins/encoding/src/text_encoder_stream.rs new file mode 100644 index 00000000..2eabc5b3 --- /dev/null +++ b/crates/builtins/encoding/src/text_encoder_stream.rs @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +//! + +use core_runtime::{webidl_interface, webidl_methods}; +use js::error::ExnThrown; +use js::gc::handle::Heap; +use js::gc::scope::Scope; +use js::native::Value; +use js::prelude::{CallbackArgs, HandleValue}; +use js::{value, Function, JSString, Object, Uint8Array}; + +use std::collections::VecDeque; +use web_streams::readable::ReadableStream; +use web_streams::{TransformStream, TransformStreamImpl}; +use web_streams::writable::WritableStream; + +/// Extract raw UTF-16 code units from a JS string, preserving surrogates. +/// +/// step 1: +/// "Let input be the result of converting chunk to a DOMString." +fn js_string_to_utf16(scope: &Scope<'_>, js_string: &JSString) -> Vec { + let length = js_string.len(); + if length == 0 { + return Vec::new(); + } + + let mut result = Vec::with_capacity(length); + for i in 0..length { + result.push(js_string.char_at(scope, i).unwrap_or(0xFFFD)); + } + result +} + +/// +#[webidl_interface] +pub struct TextEncoderStream { + /// + /// Represents the slot [[pending high surrogate]]. + leading_surrogate: Option, + + /// The underlying TransformStream that handles the streaming pipeline. + transform: Heap, +} + +#[webidl_methods] +impl TextEncoderStream { + /// + /// + /// The `new TextEncoderStream()` constructor steps: + #[constructor] + fn new(&self, scope: &Scope<'_>) -> Result<(), ExnThrown> { + // Step 1: Set this’s [[encoder]] to an instance of the UTF-8 encoder. + // (Note: The UTF-8 encoder is stateless, so no explicit assignment is required). + + let self_payload = scope.root_value(self.as_value()); + + // Step 2: Let transformAlgorithm be an algorithm which takes a chunk argument + // and runs the encode and enqueue a chunk algorithm with this and chunk. + let transform_cb = Function::new_callback( + scope, + c"textEncoderTransform", + 1, + encoder_transform_cb, + self_payload, + )?; + + // Step 3: Let flushAlgorithm be an algorithm which takes no arguments + // and runs the encode and flush algorithm with this. + let flush_cb = Function::new_callback( + scope, + c"textEncoderFlush", + 0, + encoder_flush_cb, + self_payload, + )?; + + // Step 4: Let transformStream be a new TransformStream. + // Step 5: Set up transformStream with transformAlgorithm set to transformAlgorithm + // and flushAlgorithm set to flushAlgorithm. + // We pass both algorithms as properties on a plain transformer object so that the + // TransformStream constructor sees a real transformer and does not set the + // identity-transform shortcut — otherwise `pipeTo` would bypass encoding for native + // byte sources. + let transformer = Object::new_plain(scope)?; + transformer.set_property(scope, c"transform", transform_cb)?; + transformer.set_property(scope, c"flush", flush_cb)?; + let transform_stream = TransformStream::new(scope, Some(scope.root_value(transformer.as_value())), None, None)?; + + // Step 6: Set this’s [[TransformStream]] to transformStream. + self.data_mut().transform.set(transform_stream); + + // Step 7: Set this’s [[pending high surrogate]] to null. + self.data_mut().leading_surrogate = None; + + Ok(()) + } + + /// + #[getter] + fn encoding(&self) -> String { + "utf-8".into() + } + + /// + #[getter] + fn readable<'r>(&self, scope: &'r Scope<'_>) -> ReadableStream<'r> { + self.data().transform.get(scope).readable(scope) + } + + /// + #[getter] + fn writable<'r>(&self, scope: &'r Scope<'_>) -> WritableStream<'r> { + self.data().transform.get(scope).writable(scope) + } +} + +/// The algorithm. +fn encoder_transform_cb( + scope: &Scope<'_>, + args: CallbackArgs<'_>, + payload: HandleValue<'_>, +) -> Result { + let stream: TextEncoderStream<'_> = Object::from_value(scope, *payload) + .expect("payload is an object") + .cast::>() + .expect("payload is a TextEncoderStream"); + + // Step 1: Let input be the result of converting chunk to a DOMString. + let js_string = JSString::from_value(scope, args.get(0))?; + + // Step 2: Convert input into an I/O queue of code units. + let mut input = VecDeque::from(js_string_to_utf16(scope, &js_string)); + + // Step 3: Let output be the I/O queue of bytes « end-of-queue ». + let mut output: Vec = Vec::new(); + + // Step 4: While true: + loop { + // Step 4.1: Let item be the result of reading from input. + let item = input.pop_front(); + + // Step 4.2: If item is end-of-queue: + let Some(item) = item else { + // Step 4.2.1: Convert output into a byte sequence. + // Step 4.2.2: If output is not empty, then enqueue Uint8Array(output) in encoder’s [[TransformStream]]. + if !output.is_empty() { + let u8array = Uint8Array::with_data(scope, &output)?; + enqueue_to_transform(scope, &stream, u8array)?; + } + // Step 4.2.3: Return. + break; + }; + + // Step 4.3: Let result be the result of running convert code unit to scalar value with encoder, item, and input. + // Scope the mutable borrow so it is dropped before any JS reentry (enqueue_to_transform). + let result = { + let mut data = stream.data_mut(); + convert_code_unit_to_scalar_value(&mut data, item, &mut input) + }; + + // Step 4.4: If result is not continue, then process an item with result, encoder’s encoder, input, output, and "fatal". + // (Note: For UTF-8, "processing an item" is encoding the scalar value to its byte sequence and appending to output) + match result { + EncodeResult::Continue => { + // Do nothing, continue the loop. + } + EncodeResult::Scalar(sv) => { + encode_scalar_value(&mut output, sv); + } + } + } + + Ok(value::undefined()) +} + +/// The algorithm. +fn encoder_flush_cb( + scope: &Scope<'_>, + _args: CallbackArgs<'_>, + payload: HandleValue<'_>, +) -> Result { + let stream: TextEncoderStream<'_> = Object::from_value(scope, *payload) + .expect("payload is an object") + .cast::>() + .expect("payload is a TextEncoderStream"); + + // Step 1: If encoder’s [[pending high surrogate]] is non-null: + // Scope the mutable borrow so it is dropped before any JS reentry. + let has_pending = { + let mut data = stream.data_mut(); + data.leading_surrogate.take().is_some() + }; + if has_pending { + // Step 1.1: Let chunk be a Uint8Array object wrapping « 0xEF, 0xBF, 0xBD ». (U+FFFD in UTF-8) + let chunk = Uint8Array::with_data(scope, &[0xEF, 0xBF, 0xBD])?; + // Step 1.2: Enqueue chunk in encoder’s [[TransformStream]]. + enqueue_to_transform(scope, &stream, chunk)?; + } + + Ok(value::undefined()) +} + +/// Result of the convert code unit to scalar value algorithm. +enum EncodeResult { + /// Return `continue` per spec step 2. + Continue, + /// A scalar value to encode. + Scalar(u32), +} + +/// The algorithm. +fn convert_code_unit_to_scalar_value( + data: &mut TextEncoderStreamImpl, + item: u16, + input: &mut VecDeque, +) -> EncodeResult { + // Step 1: If encoder’s leading surrogate is non-null: + if let Some(leading) = data.leading_surrogate.take() { + // Step 1.1: Let high be encoder’s leading surrogate. + // Step 1.2: Set encoder’s leading surrogate to null. (Done above by take()) + + // Step 1.3: If item is a trailing surrogate (U+DC00 to U+DFFF), then return + // the scalar value from surrogates given leadingSurrogate and item. + if (0xDC00..=0xDFFF).contains(&item) { + // To obtain a scalar value from surrogates, given a leading surrogate leading and a trailing surrogate trailing, + // return 0x10000 + ((leading − 0xD800) << 10) + (trailing − 0xDC00). + let sv = 0x10000 + ((leading as u32 - 0xD800) << 10) + (item as u32 - 0xDC00); + return EncodeResult::Scalar(sv); + } + + // Step 1.4: Restore item to input. + input.push_front(item); + + // Step 1.5: Return U+FFFD. + return EncodeResult::Scalar(0xFFFD); + } + + // Step 2: If item is a leading surrogate (U+D800 to U+DBFF): + if (0xD800..=0xDBFF).contains(&item) { + // Step 2.1: Set encoder’s [[pending high surrogate]] to item. + data.leading_surrogate = Some(item); + // Step 2.2: Return continue. + return EncodeResult::Continue; + } + + // Step 3: If item is a trailing surrogate (U+DC00 to U+DFFF), then return U+FFFD. + if (0xDC00..=0xDFFF).contains(&item) { + return EncodeResult::Scalar(0xFFFD); + } + + // Step 4: Return the scalar value item. + EncodeResult::Scalar(item as u32) +} + +/// Encode a scalar value as UTF-8 and append to output. +fn encode_scalar_value(output: &mut Vec, sv: u32) { + let mut buf = [0u8; 4]; + let c = char::from_u32(sv).unwrap_or('\u{FFFD}'); + let len = c.encode_utf8(&mut buf).len(); + output.extend_from_slice(&buf[..len]); +} + +/// Enqueue a chunk into the TextEncoderStream's TransformStream. +fn enqueue_to_transform( + scope: &Scope<'_>, + stream: &TextEncoderStream<'_>, + chunk: Uint8Array<'_>, +) -> Result<(), ExnThrown> { + let chunk_val = scope.root_value(chunk.as_value()); + stream.data().transform.get(scope).enqueue(scope, chunk_val) +} diff --git a/crates/builtins/encoding/tests/encoding.rs b/crates/builtins/encoding/tests/encoding.rs new file mode 100644 index 00000000..16ec7f8f --- /dev/null +++ b/crates/builtins/encoding/tests/encoding.rs @@ -0,0 +1,707 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +//! Integration tests for the Encoding API (TextEncoder, TextDecoder, +//! TextEncoderStream, TextDecoderStream). + +use core_runtime::config::RuntimeConfig; +use core_runtime::event_loop::run_microtasks; +use core_runtime::runtime::{clear_global_initializers, register_global_initializer, Runtime}; +use core_runtime::test_util::{eval_with_setup, throws_with_setup}; +use js::conversion::FromJSVal; + +fn setup() { + clear_global_initializers(); + register_global_initializer(|scope, global| { + web_globals::add_to_global(scope, global); + }); + register_global_initializer(|scope, global| { + encoding::add_to_global(scope, global); + }); + register_global_initializer(|scope, global| { + web_streams::add_to_global(scope, global); + }); +} + +fn eval(code: &str) -> String { + eval_with_setup(setup, code) +} + +fn throws(code: &str) -> bool { + throws_with_setup(setup, code) +} + +/// Runs async JS code and returns the value of `globalThis.__out` after +/// microtasks have settled. +fn run_async(code: &str) -> String { + clear_global_initializers(); + register_global_initializer(|scope, global| web_globals::add_to_global(scope, global)); + register_global_initializer(|scope, global| encoding::add_to_global(scope, global)); + register_global_initializer(|scope, global| web_streams::add_to_global(scope, global)); + let rt = Runtime::init(&RuntimeConfig::default()).expect("initialize test runtime"); + let scope = rt.default_global(); + js::compile::evaluate_with_filename(&scope, code, "test.js", 1).ok(); + run_microtasks(&scope); + let out = js::compile::evaluate_with_filename(&scope, "globalThis.__out", "out.js", 1).unwrap(); + String::from_jsval(&scope, out, ()).unwrap() +} + +/// Drains a ReadableStringStream into an array of string chunks. +const DRAIN_STR: &str = r#" + async function drainStr(stream) { + const r = stream.getReader(); + const acc = []; + for (;;) { + const {value, done} = await r.read(); + if (done) break; + acc.push(value); + } + return acc; + } +"#; + +/// Drains a ReadableStream into a flat array of byte numbers. +const DRAIN_BYTES: &str = r#" + async function drainBytes(stream) { + const r = stream.getReader(); + const acc = []; + for (;;) { + const {value, done} = await r.read(); + if (done) break; + for (const b of value) acc.push(b); + } + return acc; + } +"#; + +// ── TextEncoder existence ── + +#[test] +fn text_encoder_exists() { + assert_eq!(eval("typeof TextEncoder"), "function"); +} + +#[test] +fn text_encoder_construct() { + assert_eq!(eval("new TextEncoder() instanceof TextEncoder"), "true"); +} + +// ── TextEncoder.encoding ── + +#[test] +fn text_encoder_encoding() { + assert_eq!(eval("new TextEncoder().encoding"), "utf-8"); +} + +// ── TextEncoder.encode() ── + +#[test] +fn text_encoder_encode_empty() { + assert_eq!(eval("new TextEncoder().encode('').length"), "0"); +} + +#[test] +fn text_encoder_encode_ascii() { + assert_eq!( + eval("Array.from(new TextEncoder().encode('abc')).join(',')"), + "97,98,99" + ); +} + +#[test] +fn text_encoder_encode_returns_uint8array() { + assert_eq!( + eval("new TextEncoder().encode('a') instanceof Uint8Array"), + "true" + ); +} + +#[test] +fn text_encoder_encode_no_args() { + // encode() with no arguments encodes empty string + assert_eq!(eval("new TextEncoder().encode().length"), "0"); +} + +#[test] +fn text_encoder_encode_multibyte() { + // "€" is U+20AC, encoded as [0xE2, 0x82, 0xAC] in UTF-8 + assert_eq!( + eval(r#"Array.from(new TextEncoder().encode('€')).join(',')"#), + "226,130,172" + ); +} + +#[test] +fn text_encoder_encode_emoji() { + // "😀" is U+1F600, encoded as [0xF0, 0x9F, 0x98, 0x80] + assert_eq!( + eval(r#"Array.from(new TextEncoder().encode('😀')).join(',')"#), + "240,159,152,128" + ); +} + +// ── TextEncoder.encodeInto() ── + +#[test] +fn text_encoder_encode_into_basic() { + assert_eq!( + eval( + r#" + const enc = new TextEncoder(); + const buf = new Uint8Array(10); + const result = enc.encodeInto('abc', buf); + result.read + ',' + result.written + "# + ), + "3,3" + ); +} + +#[test] +fn text_encoder_encode_into_writes_bytes() { + assert_eq!( + eval( + r#" + const enc = new TextEncoder(); + const buf = new Uint8Array(10); + enc.encodeInto('abc', buf); + buf[0] + ',' + buf[1] + ',' + buf[2] + "# + ), + "97,98,99" + ); +} + +#[test] +fn text_encoder_encode_into_truncation() { + // Buffer too small to fit all characters + assert_eq!( + eval( + r#" + const enc = new TextEncoder(); + const buf = new Uint8Array(2); + const result = enc.encodeInto('abcdef', buf); + result.read + ',' + result.written + "# + ), + "2,2" + ); +} + +#[test] +fn text_encoder_encode_into_multibyte_truncation() { + // "€" needs 3 bytes; buffer of 2 can't fit it + assert_eq!( + eval( + r#" + const enc = new TextEncoder(); + const buf = new Uint8Array(2); + const result = enc.encodeInto('€', buf); + result.read + ',' + result.written + "# + ), + "0,0" + ); +} + +#[test] +fn text_encoder_encode_into_surrogate_pair_count() { + // "😀" is U+1F600, takes 4 UTF-8 bytes and 2 UTF-16 code units + assert_eq!( + eval( + r#" + const enc = new TextEncoder(); + const buf = new Uint8Array(10); + const result = enc.encodeInto('😀', buf); + result.read + ',' + result.written + "# + ), + "2,4" + ); +} + +// ── TextDecoder existence ── + +#[test] +fn text_decoder_exists() { + assert_eq!(eval("typeof TextDecoder"), "function"); +} + +#[test] +fn text_decoder_construct() { + assert_eq!(eval("new TextDecoder() instanceof TextDecoder"), "true"); +} + +// ── TextDecoder.encoding ── + +#[test] +fn text_decoder_encoding_default() { + assert_eq!(eval("new TextDecoder().encoding"), "utf-8"); +} + +#[test] +fn text_decoder_encoding_explicit_utf8() { + assert_eq!(eval("new TextDecoder('utf-8').encoding"), "utf-8"); +} + +#[test] +fn text_decoder_encoding_label_normalization() { + // "UTF-8" should be normalized to "utf-8" + assert_eq!(eval("new TextDecoder('UTF-8').encoding"), "utf-8"); +} + +#[test] +fn text_decoder_encoding_windows_1252() { + assert_eq!( + eval("new TextDecoder('windows-1252').encoding"), + "windows-1252" + ); +} + +// ── TextDecoder.fatal ── + +#[test] +fn text_decoder_fatal_default() { + assert_eq!(eval("new TextDecoder().fatal"), "false"); +} + +#[test] +fn text_decoder_fatal_true() { + assert_eq!( + eval("new TextDecoder('utf-8', { fatal: true }).fatal"), + "true" + ); +} + +// ── TextDecoder.ignoreBOM ── + +#[test] +fn text_decoder_ignore_bom_default() { + assert_eq!(eval("new TextDecoder().ignoreBOM"), "false"); +} + +#[test] +fn text_decoder_ignore_bom_true() { + assert_eq!( + eval("new TextDecoder('utf-8', { ignoreBOM: true }).ignoreBOM"), + "true" + ); +} + +// ── TextDecoder.decode() ── + +#[test] +fn text_decoder_decode_utf8() { + assert_eq!( + eval(r#"new TextDecoder().decode(new Uint8Array([97, 98, 99]))"#), + "abc" + ); +} + +#[test] +fn text_decoder_decode_no_args() { + // decode() with no arguments returns empty string + assert_eq!(eval("new TextDecoder().decode()"), ""); +} + +#[test] +fn text_decoder_decode_empty_array() { + assert_eq!(eval("new TextDecoder().decode(new Uint8Array([]))"), ""); +} + +#[test] +fn text_decoder_decode_multibyte_utf8() { + // "€" = [0xE2, 0x82, 0xAC] + assert_eq!( + eval("new TextDecoder().decode(new Uint8Array([0xE2, 0x82, 0xAC]))"), + "€" + ); +} + +#[test] +fn text_decoder_decode_emoji() { + // "😀" = [0xF0, 0x9F, 0x98, 0x80] + assert_eq!( + eval("new TextDecoder().decode(new Uint8Array([0xF0, 0x9F, 0x98, 0x80]))"), + "😀" + ); +} + +// ── TextDecoder error handling ── + +#[test] +fn text_decoder_invalid_label_throws() { + assert!(throws("new TextDecoder('invalid-encoding')")); +} + +#[test] +fn text_decoder_replacement_encoding_throws() { + // "replacement" is not a valid label for TextDecoder (excluded by spec) + assert!(throws("new TextDecoder('replacement')")); +} + +#[test] +fn text_decoder_fatal_invalid_bytes() { + // Invalid UTF-8 sequence with fatal: true should throw + assert!(throws( + "new TextDecoder('utf-8', { fatal: true }).decode(new Uint8Array([0xFF, 0xFE]))" + )); +} + +#[test] +fn text_decoder_replacement_mode_invalid_bytes() { + // Invalid UTF-8 in replacement mode should produce U+FFFD + assert_eq!( + eval( + r#" + const decoder = new TextDecoder('utf-8'); + const result = decoder.decode(new Uint8Array([0xFF])); + result === '\uFFFD' + "# + ), + "true" + ); +} + +// ── TextDecoder streaming ── + +#[test] +fn text_decoder_streaming() { + // Multi-byte character split across two decode() calls + // "€" = [0xE2, 0x82, 0xAC] + assert_eq!( + eval( + r#" + const decoder = new TextDecoder(); + let result = decoder.decode(new Uint8Array([0xE2, 0x82]), { stream: true }); + result += decoder.decode(new Uint8Array([0xAC])); + result + "# + ), + "€" + ); +} + +// ── TextDecoder with non-UTF-8 encodings ── + +#[test] +fn text_decoder_windows_1252() { + // 0xE9 in windows-1252 is "é" + assert_eq!( + eval(r#"new TextDecoder('windows-1252').decode(new Uint8Array([0xE9]))"#), + "é" + ); +} + +// ── WebIDL interface compliance ── + +#[test] +fn text_encoder_to_string_tag() { + assert_eq!( + eval("Object.prototype.toString.call(new TextEncoder())"), + "[object TextEncoder]" + ); +} + +#[test] +fn text_decoder_to_string_tag() { + assert_eq!( + eval("Object.prototype.toString.call(new TextDecoder())"), + "[object TextDecoder]" + ); +} + +// ── TextEncoderStream ── + +#[test] +fn text_encoder_stream_exists() { + assert_eq!(eval("typeof TextEncoderStream"), "function"); +} + +#[test] +fn text_encoder_stream_encoding() { + assert_eq!(eval("new TextEncoderStream().encoding"), "utf-8"); +} + +#[test] +fn text_encoder_stream_readable_writable() { + assert_eq!( + eval( + r#" + const s = new TextEncoderStream(); + (s.readable instanceof ReadableStream) + ',' + (s.writable instanceof WritableStream) + "# + ), + "true,true" + ); +} + +#[test] +fn text_encoder_stream_ascii() { + let out = run_async(&format!( + r#"{DRAIN_BYTES} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue("abc"); c.close(); }} }}); + const os = rs.pipeThrough(new TextEncoderStream()); + drainBytes(os).then(a => {{ globalThis.__out = a.join(","); }}); + "# + )); + assert_eq!(out, "97,98,99"); +} + +#[test] +fn text_encoder_stream_multibyte() { + // "€" = [0xE2, 0x82, 0xAC] + let out = run_async(&format!( + r#"{DRAIN_BYTES} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue("€"); c.close(); }} }}); + const os = rs.pipeThrough(new TextEncoderStream()); + drainBytes(os).then(a => {{ globalThis.__out = a.join(","); }}); + "# + )); + assert_eq!(out, "226,130,172"); +} + +#[test] +fn text_encoder_stream_unpaired_surrogate() { + // Lone leading surrogate produces replacement character [0xEF, 0xBF, 0xBD] + let out = run_async(&format!( + r#"{DRAIN_BYTES} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue("\uD83D"); c.close(); }} }}); + const os = rs.pipeThrough(new TextEncoderStream()); + drainBytes(os).then(a => {{ globalThis.__out = a.join(","); }}); + "# + )); + assert_eq!(out, "239,191,189"); +} + +#[test] +fn text_encoder_stream_surrogate_pair_split() { + // Surrogates in separate chunks should still produce the correct encoding + let out = run_async(&format!( + r#"{DRAIN_BYTES} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue("\uD83D"); c.enqueue("\uDE00"); c.close(); }} }}); + const os = rs.pipeThrough(new TextEncoderStream()); + drainBytes(os).then(a => {{ globalThis.__out = a.join(","); }}); + "# + )); + assert_eq!(out, "240,159,152,128"); +} + +// ── TextDecoderStream ── + +#[test] +fn text_decoder_stream_exists() { + assert_eq!(eval("typeof TextDecoderStream"), "function"); +} + +#[test] +fn text_decoder_stream_encoding() { + assert_eq!(eval("new TextDecoderStream().encoding"), "utf-8"); +} + +#[test] +fn text_decoder_stream_fatal() { + assert_eq!(eval("new TextDecoderStream().fatal"), "false"); +} + +#[test] +fn text_decoder_stream_ignore_bom() { + assert_eq!(eval("new TextDecoderStream().ignoreBOM"), "false"); +} + +#[test] +fn text_decoder_stream_readable_writable() { + assert_eq!( + eval( + r#" + const s = new TextDecoderStream(); + (s.readable instanceof ReadableStream) + ',' + (s.writable instanceof WritableStream) + "# + ), + "true,true" + ); +} + +#[test] +fn text_decoder_stream_invalid_label_throws() { + assert!(throws("new TextDecoderStream('invalid-encoding')")); +} + +#[test] +fn text_decoder_stream_ascii() { + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([97, 98, 99])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream()); + drainStr(os).then(a => {{ globalThis.__out = a.join("|"); }}); + "# + )); + assert_eq!(out, "abc"); +} + +#[test] +fn text_decoder_stream_multibyte() { + // "€" = [0xE2, 0x82, 0xAC] + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xE2, 0x82, 0xAC])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream()); + drainStr(os).then(a => {{ globalThis.__out = a.join("|"); }}); + "# + )); + assert_eq!(out, "€"); +} + +#[test] +fn text_decoder_stream_split_multibyte() { + // "€" = [0xE2, 0x82, 0xAC], split across two chunks + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xE2, 0x82])); c.enqueue(new Uint8Array([0xAC])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream()); + drainStr(os).then(a => {{ globalThis.__out = a.join("|"); }}); + "# + )); + assert_eq!(out, "€"); +} + +#[test] +fn text_decoder_stream_split_4byte() { + // "😀" = [0xF0, 0x9F, 0x98, 0x80], split after first byte + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xF0])); c.enqueue(new Uint8Array([0x9F, 0x98, 0x80])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream()); + drainStr(os).then(a => {{ globalThis.__out = a.join("|"); }}); + "# + )); + assert_eq!(out, "😀"); +} + +#[test] +fn text_decoder_stream_incomplete_at_eof() { + // Incomplete 4-byte sequence at EOF produces replacement character(s) + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xF0, 0x9F, 0x98])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream()); + drainStr(os).then(a => {{ globalThis.__out = a.join("").length; }}); + "# + )); + assert_eq!(out, "1"); +} + +#[test] +fn text_decoder_stream_fatal_invalid_bytes() { + // fatal: true should reject the readable stream on invalid bytes + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xFF, 0xFE])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream("utf-8", {{ fatal: true }})); + drainStr(os).then( + () => {{ globalThis.__out = "fulfilled"; }}, + e => {{ globalThis.__out = "rejected:" + e.name; }} + ); + "# + )); + assert_eq!(out, "rejected:TypeError"); +} + +// -- BOM handling -- + +#[test] +fn text_decoder_stream_bom_default() { + // ignoreBOM: false (default) → BOM is removed + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xEF, 0xBB, 0xBF, 97])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream()); + drainStr(os).then(a => {{ globalThis.__out = a.join(""); }}); + "# + )); + assert_eq!(out, "a"); +} + +#[test] +fn text_decoder_stream_bom_ignored() { + // ignoreBOM: true → BOM is kept as U+FEFF + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xEF, 0xBB, 0xBF, 97])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream("utf-8", {{ ignoreBOM: true }})); + drainStr(os).then(a => {{ globalThis.__out = a[0].charCodeAt(0) + "," + a[0][1]; }}); + "# + )); + assert_eq!(out, "65279,a"); +} + +#[test] +fn text_decoder_stream_bom_split_across_chunks() { + // BOM bytes split across two chunks, ignoreBOM: true → BOM kept as U+FEFF + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ + c.enqueue(new Uint8Array([0xEF, 0xBB])); + c.enqueue(new Uint8Array([0xBF, 97])); + c.close(); + }} }}); + const os = rs.pipeThrough(new TextDecoderStream("utf-8", {{ ignoreBOM: true }})); + drainStr(os).then(a => {{ globalThis.__out = a[0].charCodeAt(0) + "," + a[0][1]; }}); + "# + )); + assert_eq!(out, "65279,a"); +} + +#[test] +fn text_decoder_stream_bom_only() { + // BOM only, ignoreBOM: true → emits U+FEFF + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xEF, 0xBB, 0xBF])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream("utf-8", {{ ignoreBOM: true }})); + drainStr(os).then(a => {{ globalThis.__out = a[0].charCodeAt(0); }}); + "# + )); + assert_eq!(out, "65279"); +} + +#[test] +fn text_decoder_stream_bom_only_default() { + // BOM only, ignoreBOM: false (default) → no output + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xEF, 0xBB, 0xBF])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream()); + drainStr(os).then(a => {{ globalThis.__out = a.length; }}); + "# + )); + assert_eq!(out, "0"); +} + +#[test] +fn text_decoder_stream_windows_1252() { + // 0xE9 in windows-1252 is "é" + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xE9])); c.close(); }} }}); + const os = rs.pipeThrough(new TextDecoderStream("windows-1252")); + drainStr(os).then(a => {{ globalThis.__out = a.join("|"); }}); + "# + )); + assert_eq!(out, "é"); +} diff --git a/crates/builtins/web-globals/Cargo.toml b/crates/builtins/web-globals/Cargo.toml index 979f8b72..fe96ac38 100644 --- a/crates/builtins/web-globals/Cargo.toml +++ b/crates/builtins/web-globals/Cargo.toml @@ -14,7 +14,6 @@ test = false bitflags.workspace = true core-runtime.workspace = true data-url.workspace = true -encoding_rs.workspace = true js.workspace = true platform.workspace = true smallvec.workspace = true diff --git a/crates/builtins/web-globals/src/encoding/mod.rs b/crates/builtins/web-globals/src/encoding/mod.rs deleted file mode 100644 index b43fd31a..00000000 --- a/crates/builtins/web-globals/src/encoding/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception - -mod text_decoder; -mod text_encoder; - -use js::gc::scope::Scope; -use js::Object; - -pub fn add_to_global(scope: &Scope<'_>, global: Object<'_>) { - text_encoder::TextEncoder::add_to_global(scope, global); - text_decoder::TextDecoder::add_to_global(scope, global); -} diff --git a/crates/builtins/web-globals/src/lib.rs b/crates/builtins/web-globals/src/lib.rs index 0033c2ab..da8ca738 100644 --- a/crates/builtins/web-globals/src/lib.rs +++ b/crates/builtins/web-globals/src/lib.rs @@ -12,7 +12,6 @@ pub mod base64; pub mod dom_exception; -pub mod encoding; pub mod events; pub mod navigator; pub mod performance; @@ -22,7 +21,6 @@ pub mod worker_location; pub fn add_to_global(scope: &js::prelude::Scope<'_>, global: js::Object<'_>) { dom_exception::DOMException::add_to_global(scope, global); - encoding::add_to_global(scope, global); events::add_to_global(scope, global); performance::add_to_global(scope, global); signals::add_to_global(scope, global); diff --git a/crates/builtins/web-globals/tests/encoding.rs b/crates/builtins/web-globals/tests/encoding.rs deleted file mode 100644 index d99ce8d4..00000000 --- a/crates/builtins/web-globals/tests/encoding.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception - -//! A `TextDecoder` check that the enabled WPT `encoding/` tests don't make. - -// This file contains nothing platform-specific, so skip it on wasm32. -#![cfg(not(target_arch = "wasm32"))] - -use core_runtime::test_util::eval_with_setup; - -fn setup() { - core_runtime::runtime::register_global_initializer(|scope, global| { - web_globals::add_to_global(scope, global); - }); -} - -fn eval(code: &str) -> String { - eval_with_setup(setup, code) -} - -/// windows-1252 input without bytes in 0x80..=0x9F decodes to the same code points, and input -/// with them maps those bytes through the encoding's table, including when a streaming decode -/// alternates between the two. -#[test] -fn text_decoder_windows_1252_maps_every_byte() { - assert_eq!( - eval( - "const d = new TextDecoder('latin1'); - const codes = s => Array.from(s, c => c.charCodeAt(0)).join(','); - [d.decode(new Uint8Array([0x41, 0xe9, 0xff, 0xa0])), - d.decode(new Uint8Array([0x41, 0x80, 0x9f, 0xe9])), - d.decode(new Uint8Array([0xe9]), { stream: true }) + - d.decode(new Uint8Array([0x80]), { stream: true }) + d.decode(), - d.decode(new Uint8Array([]))].map(codes).join(' | ')" - ), - "65,233,255,160 | 65,8364,376,233 | 233,8364 | " - ); -} diff --git a/crates/builtins/web-streams/src/lib.rs b/crates/builtins/web-streams/src/lib.rs index a0fb19e7..efb2cf5b 100644 --- a/crates/builtins/web-streams/src/lib.rs +++ b/crates/builtins/web-streams/src/lib.rs @@ -5,6 +5,7 @@ pub mod queuing; pub mod readable; pub(crate) mod support; pub mod transform; +pub use transform::{TransformStream, TransformStreamImpl}; pub mod writable; pub use support::AlgorithmArg; diff --git a/crates/builtins/web-streams/src/transform/mod.rs b/crates/builtins/web-streams/src/transform/mod.rs index 4fd32db9..f72c259a 100644 --- a/crates/builtins/web-streams/src/transform/mod.rs +++ b/crates/builtins/web-streams/src/transform/mod.rs @@ -2,7 +2,7 @@ //! -pub(crate) mod algorithms; +mod algorithms; pub mod readable_writable_pair; pub mod transform_stream; @@ -10,6 +10,6 @@ pub mod transform_stream_default_controller; pub mod transformer; pub use readable_writable_pair::ReadableWritablePair; -pub use transform_stream::TransformStream; +pub use transform_stream::{TransformStream, TransformStreamImpl}; pub use transform_stream_default_controller::TransformStreamDefaultController; pub use transformer::Transformer; diff --git a/crates/builtins/web-streams/src/transform/transform_stream.rs b/crates/builtins/web-streams/src/transform/transform_stream.rs index 8bedbaac..1652ade5 100644 --- a/crates/builtins/web-streams/src/transform/transform_stream.rs +++ b/crates/builtins/web-streams/src/transform/transform_stream.rs @@ -171,15 +171,29 @@ impl TransformStream { /// #[getter] - fn readable<'r>(&self, scope: &'r Scope<'_>) -> ReadableStream<'r> { + pub fn readable<'r>(&self, scope: &'r Scope<'_>) -> ReadableStream<'r> { // Step 1: Return `this`.`[[readable]]`. self.data().readable.get(scope) } /// #[getter] - fn writable<'r>(&self, scope: &'r Scope<'_>) -> WritableStream<'r> { + pub fn writable<'r>(&self, scope: &'r Scope<'_>) -> WritableStream<'r> { // Step 1: Return `this`.`[[writable]]`. self.data().writable.get(scope) } } + +/// Helpers for the encoding crate — not part of the WebIDL interface. +impl TransformStream<'_> { + /// Enqueue a chunk into the readable side of this TransformStream. + pub fn enqueue(&self, scope: &Scope<'_>, chunk: HandleValue<'_>) -> Result<(), ExnThrown> { + let ctrl = self + .data() + .controller + .as_ref() + .expect("controller must be set") + .get(scope); + algorithms::transform_stream_default_controller_enqueue(scope, &ctrl, chunk) + } +} diff --git a/crates/libstarling/Cargo.toml b/crates/libstarling/Cargo.toml index 464387f9..6c925adc 100644 --- a/crates/libstarling/Cargo.toml +++ b/crates/libstarling/Cargo.toml @@ -21,6 +21,7 @@ event-listener.workspace = true futures-lite.workspace = true futures-util.workspace = true http.workspace = true +encoding.workspace = true js.workspace = true platform.workspace = true fetch-event.workspace = true diff --git a/crates/libstarling/src/lib.rs b/crates/libstarling/src/lib.rs index e6d0744f..33d71edd 100644 --- a/crates/libstarling/src/lib.rs +++ b/crates/libstarling/src/lib.rs @@ -25,6 +25,7 @@ pub mod serve_wasm; pub fn register_builtins() { runtime::register_global_initializer(web_globals::add_to_global); runtime::register_global_initializer(web_streams::add_to_global); + runtime::register_global_initializer(encoding::add_to_global); runtime::register_global_initializer(web_url::add_to_global); runtime::register_global_initializer(web_fetch::add_to_global); runtime::register_global_initializer(fetch_event::add_to_global); diff --git a/tests/wpt-harness/expectations/encoding/idlharness.any.js.json b/tests/wpt-harness/expectations/encoding/idlharness.any.js.json index 00b7eabd..52474344 100644 --- a/tests/wpt-harness/expectations/encoding/idlharness.any.js.json +++ b/tests/wpt-harness/expectations/encoding/idlharness.any.js.json @@ -123,51 +123,51 @@ "status": "PASS" }, "TextDecoderStream interface: existence and properties of interface object": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface object length": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface object name": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: existence and properties of interface prototype object": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: existence and properties of interface prototype object's \"constructor\" property": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: existence and properties of interface prototype object's @@unscopables property": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: attribute encoding": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: attribute fatal": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: attribute ignoreBOM": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface: existence and properties of interface object": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface object length": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface object name": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface: existence and properties of interface prototype object": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface: existence and properties of interface prototype object's \"constructor\" property": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface: existence and properties of interface prototype object's @@unscopables property": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface: attribute encoding": { - "status": "FAIL" + "status": "PASS" } -} \ No newline at end of file +} diff --git a/tests/wpt-harness/expectations/encoding/streams/backpressure.any.js.json b/tests/wpt-harness/expectations/encoding/streams/backpressure.any.js.json new file mode 100644 index 00000000..f2c2435c --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/backpressure.any.js.json @@ -0,0 +1,14 @@ +{ + "write() should not complete until read relieves backpressure for TextDecoderStream": { + "status": "PASS" + }, + "additional writes should wait for backpressure to be relieved for class TextDecoderStream": { + "status": "PASS" + }, + "write() should not complete until read relieves backpressure for TextEncoderStream": { + "status": "PASS" + }, + "additional writes should wait for backpressure to be relieved for class TextEncoderStream": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-attributes.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-attributes.any.js.json new file mode 100644 index 00000000..f3cb0937 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-attributes.any.js.json @@ -0,0 +1,86 @@ +{ + "encoding attribute should have correct value for 'unicode-1-1-utf-8'": { + "status": "PASS" + }, + "encoding attribute should have correct value for 'iso-8859-2'": { + "status": "PASS" + }, + "encoding attribute should have correct value for 'ascii'": { + "status": "PASS" + }, + "encoding attribute should have correct value for 'utf-16'": { + "status": "PASS" + }, + "setting fatal to 'false' should set the attribute to false": { + "status": "PASS" + }, + "setting ignoreBOM to 'false' should set the attribute to false": { + "status": "PASS" + }, + "setting fatal to '0' should set the attribute to false": { + "status": "PASS" + }, + "setting ignoreBOM to '0' should set the attribute to false": { + "status": "PASS" + }, + "setting fatal to '' should set the attribute to false": { + "status": "PASS" + }, + "setting ignoreBOM to '' should set the attribute to false": { + "status": "PASS" + }, + "setting fatal to 'undefined' should set the attribute to false": { + "status": "PASS" + }, + "setting ignoreBOM to 'undefined' should set the attribute to false": { + "status": "PASS" + }, + "setting fatal to 'null' should set the attribute to false": { + "status": "PASS" + }, + "setting ignoreBOM to 'null' should set the attribute to false": { + "status": "PASS" + }, + "setting fatal to 'true' should set the attribute to true": { + "status": "PASS" + }, + "setting ignoreBOM to 'true' should set the attribute to true": { + "status": "PASS" + }, + "setting fatal to '1' should set the attribute to true": { + "status": "PASS" + }, + "setting ignoreBOM to '1' should set the attribute to true": { + "status": "PASS" + }, + "setting fatal to '[object Object]' should set the attribute to true": { + "status": "PASS" + }, + "setting ignoreBOM to '[object Object]' should set the attribute to true": { + "status": "PASS" + }, + "setting fatal to '' should set the attribute to true": { + "status": "PASS" + }, + "setting ignoreBOM to '' should set the attribute to true": { + "status": "PASS" + }, + "setting fatal to 'yes' should set the attribute to true": { + "status": "PASS" + }, + "setting ignoreBOM to 'yes' should set the attribute to true": { + "status": "PASS" + }, + "constructing with an invalid encoding should throw": { + "status": "PASS" + }, + "constructing with a non-stringifiable encoding should throw": { + "status": "PASS" + }, + "a throwing fatal member should cause the constructor to throw": { + "status": "PASS" + }, + "a throwing ignoreBOM member should cause the constructor to throw": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-bad-chunks.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-bad-chunks.any.js.json new file mode 100644 index 00000000..3bac5477 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-bad-chunks.any.js.json @@ -0,0 +1,17 @@ +{ + "chunk of type undefined should error the stream": { + "status": "PASS" + }, + "chunk of type null should error the stream": { + "status": "PASS" + }, + "chunk of type numeric should error the stream": { + "status": "PASS" + }, + "chunk of type object, not BufferSource should error the stream": { + "status": "PASS" + }, + "chunk of type array should error the stream": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-ignore-bom.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-ignore-bom.any.js.json new file mode 100644 index 00000000..0b138212 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-ignore-bom.any.js.json @@ -0,0 +1,38 @@ +{ + "ignoreBOM should work for encoding utf-8, split at character 0": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-8, split at character 1": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-8, split at character 2": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-8, split at character 3": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16le, split at character 0": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16le, split at character 1": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16le, split at character 2": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16le, split at character 3": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16be, split at character 0": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16be, split at character 1": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16be, split at character 2": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16be, split at character 3": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-incomplete-input.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-incomplete-input.any.js.json new file mode 100644 index 00000000..c97336eb --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-incomplete-input.any.js.json @@ -0,0 +1,8 @@ +{ + "incomplete input with error mode \"replacement\" should end with a replacement character": { + "status": "PASS" + }, + "incomplete input with error mode \"fatal\" should error the stream": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-non-utf8.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-non-utf8.any.js.json new file mode 100644 index 00000000..2df45e96 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-non-utf8.any.js.json @@ -0,0 +1,41 @@ +{ + "TextDecoderStream should be able to decode UTF-16BE": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode invalid sequences in UTF-16BE": { + "status": "PASS" + }, + "TextDecoderStream should be able to reject invalid sequences in UTF-16BE": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode UTF-16LE": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode invalid sequences in UTF-16LE": { + "status": "PASS" + }, + "TextDecoderStream should be able to reject invalid sequences in UTF-16LE": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode Shift_JIS": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode invalid sequences in Shift_JIS": { + "status": "PASS" + }, + "TextDecoderStream should be able to reject invalid sequences in Shift_JIS": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode ISO-2022-JP": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode invalid sequences in ISO-2022-JP": { + "status": "PASS" + }, + "TextDecoderStream should be able to reject invalid sequences in ISO-2022-JP": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode ISO-8859-14": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-split-character.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-split-character.any.js.json new file mode 100644 index 00000000..eab2671e --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-split-character.any.js.json @@ -0,0 +1,35 @@ +{ + "a code point split between chunks should not be emitted until all bytes are available; split point = 2": { + "status": "PASS" + }, + "a code point split between chunks should not be emitted until all bytes are available; split point = 3": { + "status": "PASS" + }, + "a code point split between chunks should not be emitted until all bytes are available; split point = 4": { + "status": "PASS" + }, + "a code point split between chunks should not be emitted until all bytes are available; split point = 5": { + "status": "PASS" + }, + "a code point should be emitted as soon as all bytes are available": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 1": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 2": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 3": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 4": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 5": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 6": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/encode-bad-chunks.any.js.json b/tests/wpt-harness/expectations/encoding/streams/encode-bad-chunks.any.js.json new file mode 100644 index 00000000..09ebc7e8 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/encode-bad-chunks.any.js.json @@ -0,0 +1,20 @@ +{ + "a chunk that cannot be converted to a string should error the streams": { + "status": "PASS" + }, + "input of type undefined should be converted correctly to string": { + "status": "PASS" + }, + "input of type null should be converted correctly to string": { + "status": "PASS" + }, + "input of type numeric should be converted correctly to string": { + "status": "PASS" + }, + "input of type object should be converted correctly to string": { + "status": "PASS" + }, + "input of type array should be converted correctly to string": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/encode-utf8.any.js.json b/tests/wpt-harness/expectations/encoding/streams/encode-utf8.any.js.json new file mode 100644 index 00000000..037dcc6d --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/encode-utf8.any.js.json @@ -0,0 +1,59 @@ +{ + "encoding one string of UTF-8 should give one complete chunk": { + "status": "PASS" + }, + "a character split between chunks should be correctly encoded": { + "status": "PASS" + }, + "a character following one split between chunks should be correctly encoded": { + "status": "PASS" + }, + "two consecutive astral characters each split down the middle should be correctly reassembled": { + "status": "PASS" + }, + "two consecutive astral characters each split down the middle with an invalid surrogate in the middle should be correctly encoded": { + "status": "PASS" + }, + "a stream ending in a leading surrogate should emit a replacement character as a final chunk": { + "status": "PASS" + }, + "an unmatched surrogate at the end of a chunk followed by an astral character in the next chunk should be replaced with the replacement character at the start of the next output chunk": { + "status": "PASS" + }, + "an unmatched surrogate at the end of a chunk followed by an ascii character in the next chunk should be replaced with the replacement character at the start of the next output chunk": { + "status": "PASS" + }, + "an unmatched surrogate at the end of a chunk followed by a plane 1 character split into two chunks should result in the encoded plane 1 character appearing in the last output chunk": { + "status": "PASS" + }, + "two leading chunks should result in two replacement characters": { + "status": "PASS" + }, + "a non-terminal unpaired leading surrogate should immediately be replaced": { + "status": "PASS" + }, + "a terminal unpaired trailing surrogate should immediately be replaced": { + "status": "PASS" + }, + "a leading surrogate chunk should be carried past empty chunks": { + "status": "PASS" + }, + "a leading surrogate chunk should error when it is clear it didn't form a pair": { + "status": "PASS" + }, + "an empty string should result in no output chunk": { + "status": "PASS" + }, + "a leading empty chunk should be ignored": { + "status": "PASS" + }, + "a trailing empty chunk should be ignored": { + "status": "PASS" + }, + "a plain ASCII chunk should be converted": { + "status": "PASS" + }, + "characters in the ISO-8859-1 range should be encoded correctly": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/readable-writable-properties.any.js.json b/tests/wpt-harness/expectations/encoding/streams/readable-writable-properties.any.js.json new file mode 100644 index 00000000..925c4992 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/readable-writable-properties.any.js.json @@ -0,0 +1,8 @@ +{ + "TextEncoderStream readable and writable properties must pass brand checks": { + "status": "PASS" + }, + "TextDecoderStream readable and writable properties must pass brand checks": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/tests.json b/tests/wpt-harness/tests.json index 72d29455..669f69e5 100644 --- a/tests/wpt-harness/tests.json +++ b/tests/wpt-harness/tests.json @@ -140,6 +140,17 @@ "SLOW encoding/idlharness.any.js", "encoding/iso-2022-jp-decoder.any.js", "encoding/replacement-encodings.any.js", + "encoding/streams/backpressure.any.js", + "encoding/streams/decode-attributes.any.js", + "encoding/streams/decode-bad-chunks.any.js", + "encoding/streams/decode-ignore-bom.any.js", + "encoding/streams/decode-incomplete-input.any.js", + "encoding/streams/decode-non-utf8.any.js", + "encoding/streams/decode-split-character.any.js", + "SKIP(Requires SharedArrayBuffer and MessageChannel) encoding/streams/decode-utf8.any.js", + "encoding/streams/encode-bad-chunks.any.js", + "encoding/streams/encode-utf8.any.js", + "encoding/streams/readable-writable-properties.any.js", "encoding/textdecoder-arguments.any.js", "encoding/textdecoder-byte-order-marks.any.js", "encoding/textdecoder-copy.any.js", From 7654fe8cfe14317c45e455ad49acc1d28af2c336 Mon Sep 17 00:00:00 2001 From: Karthik Ganeshram Date: Tue, 20 Oct 2026 17:55:50 +0530 Subject: [PATCH 2/5] address review comments Signed-off-by: Karthik Ganeshram --- .../builtins/encoding/src/decoder_common.rs | 1 - .../encoding/src/text_decoder_stream.rs | 301 ++++++------------ .../encoding/src/text_encoder_stream.rs | 256 ++++++--------- .../src/transform/transform_stream.rs | 52 +++ 4 files changed, 234 insertions(+), 376 deletions(-) diff --git a/crates/builtins/encoding/src/decoder_common.rs b/crates/builtins/encoding/src/decoder_common.rs index 927153bc..fc1d6226 100644 --- a/crates/builtins/encoding/src/decoder_common.rs +++ b/crates/builtins/encoding/src/decoder_common.rs @@ -16,7 +16,6 @@ pub fn new_decoder( } } - /// Options dictionary for TextDecoder and TextDecoderStream constructors. #[core_runtime::webidl_dictionary] #[derive(Default)] diff --git a/crates/builtins/encoding/src/text_decoder_stream.rs b/crates/builtins/encoding/src/text_decoder_stream.rs index b6592345..75ae45a1 100644 --- a/crates/builtins/encoding/src/text_decoder_stream.rs +++ b/crates/builtins/encoding/src/text_decoder_stream.rs @@ -7,35 +7,35 @@ //! incomplete multi-byte sequences at chunk boundaries. use core_runtime::{webidl_interface, webidl_methods}; +use js::conversion::FromJSVal; use js::error::ExnThrown; +use js::function::cast_payload; use js::gc::handle::Heap; use js::gc::scope::Scope; use js::native::Value; use js::prelude::{CallbackArgs, HandleValue}; -use js::{ArrayBuffer, ArrayBufferView, Function, Object, value}; +use js::{value, JSString}; use web_streams::readable::ReadableStream; -use web_streams::{TransformStream, TransformStreamImpl}; use web_streams::writable::WritableStream; +use web_streams::{AlgorithmArg, TransformStream, TransformStreamImpl}; use crate::decoder_common::{new_decoder, TextDecoderOptions}; +use crate::text_decoder::{decode_bytes_to_js_string, ArrayBufferViewOrArrayBuffer}; /// #[webidl_interface] pub struct TextDecoderStream { - /// The encoding_rs encoding (always `Some` after construction). + /// + /// (always `Some` after construction). #[no_trace] encoding: Option<&'static encoding_rs::Encoding>, - /// The encoding name, ASCII lowercased. - encoding_name: String, - /// /// true = "fatal" mode (throw on error), false = "replacement" mode. fatal: bool, /// - #[no_trace(name = "ignoreBOM")] ignore_bom: bool, // Note: The spec defines an I/O queue (https://encoding.spec.whatwg.org/#textdecodercommon-i-o-queue) @@ -46,7 +46,6 @@ pub struct TextDecoderStream { // `OutputFull` and treat `Malformed` as a fatal error, the result is always `InputEmpty` // with `read == src.len()`. Passing each chunk's bytes directly to the decoder is // equivalent to the spec's queue-then-process pattern. - /// /// The decoder instance, preserving state between chunks. #[no_trace] @@ -72,71 +71,45 @@ impl TextDecoderStream { options: Option, ) -> Result<(), ExnThrown> { // Step 1: Let encoding be the result of getting an encoding from label. + // Step 2: If encoding is failure or replacement, then throw a RangeError. let label = label.as_deref().unwrap_or("utf-8"); - let encoding = - encoding_rs::Encoding::for_label_no_replacement(label.as_bytes()).ok_or_else(|| { + let encoding = encoding_rs::Encoding::for_label_no_replacement(label.as_bytes()) + .ok_or_else(|| { js::error::throw_range_error(scope, c"The encoding label provided is invalid.") })?; - // Step 2: If encoding is failure or replacement, then throw a RangeError. - if encoding == encoding_rs::REPLACEMENT { - return Err(js::error::throw_range_error( - scope, - c"The replacement or failure encoding cannot be used.", - )); - } - // Step 3: Let errorMode be "fatal" if options["fatal"] is true; otherwise "replacement". let options = options.unwrap_or_default(); let fatal = options.fatal; let ignore_bom = options.ignore_bom; - // Step 4: Set up a text decoder stream with this, encoding, errorMode, and options["ignoreBOM"]. - // Step 4.1: Assert: encoding is not replacement. (checked above when we throw) - // Step 4.2 Set stream's encoding to encoding. + // Step 4: Set up a text decoder stream with this, encoding, errorMode, and + // options["ignoreBOM"]. + // Step 4.1: Assert: encoding is not replacement. + // Step 4.2: Set stream's encoding to encoding. self.data_mut().encoding = Some(encoding); - self.data_mut().encoding_name = encoding.name().to_ascii_lowercase(); - // Step 4.3 Set stream's error mode to errorMode. + // Step 4.3: Set stream's error mode to errorMode. self.data_mut().fatal = fatal; - // Step 4.4 Set stream's ignore BOM to options["ignoreBOM"]. + // Step 4.4: Set stream's ignore BOM to ignoreBOM. self.data_mut().ignore_bom = ignore_bom; - // Step 4.5 Set stream’s decoder to a new instance of encoding’s decoder. + // Step 4.5: Set stream’s decoder to a new instance of encoding’s decoder. self.data_mut().decoder = Some(new_decoder(encoding, ignore_bom)); // Step 4.6: Set stream's I/O queue to a new I/O queue. - // (Omitted — encoding_rs's Decoder buffers partial sequences internally; - // see the comment on the struct definition.) - - let self_payload = scope.root_value(self.as_value()); - // Step 4.7: Let transformAlgorithm be an algorithm which takes a chunk argument and runs the decode and enqueue a chunk algorithm with stream and chunk. - let transform_cb = Function::new_callback( - scope, - c"textDecoderTransform", - 1, - decoder_transform_cb, - self_payload, - )?; - - // Step 4.8: Let flushAlgorithm be an algorithm which takes no arguments and runs the flush and enqueue algorithm with stream. - let flush_cb = Function::new_callback( - scope, - c"textDecoderFlush", - 0, - decoder_flush_cb, - self_payload, - )?; - + // Implementation note: this is not stored separately because `encoding_rs::Decoder` + // buffers incomplete sequences internally; see the comment on the struct definition. + + // Step 4.7: Let transformAlgorithm be an algorithm which takes a chunk argument and runs + // the decode and enqueue a chunk algorithm with stream and chunk. + let transform = AlgorithmArg::native(scope, decoder_transform_cb, self)?; + // Step 4.8: Let flushAlgorithm be an algorithm which takes no arguments and runs the + // flush and enqueue algorithm with stream. + let flush = AlgorithmArg::native(scope, decoder_flush_cb, self)?; // Step 4.9: Let transformStream be a new TransformStream. - // Step 4.10: Set up transformStream with transformAlgorithm set to transformAlgorithm and flushAlgorithm set to flushAlgorithm. - // We pass both algorithms as properties on a plain transformer object so that the - // TransformStream constructor sees a real transformer and does not set the - // identity-transform shortcut — otherwise `pipeTo` would bypass decoding for native - // byte sources. - let transformer = Object::new_plain(scope)?; - transformer.set_property(scope, c"transform", transform_cb)?; - transformer.set_property(scope, c"flush", flush_cb)?; - let transform_stream = TransformStream::new(scope, Some(scope.root_value(transformer.as_value())), None, None)?; - - // Step 4.11: Set stream’s transform to transformStream. + // Step 4.10: Set up transformStream with transformAlgorithm set to transformAlgorithm and + // flushAlgorithm set to flushAlgorithm. + let transform_stream = + TransformStream::new_native(scope, transform, flush, AlgorithmArg::None)?; + // Step 4.11: Set stream’s [[TransformStream]] to transformStream. self.data_mut().transform.set(transform_stream); Ok(()) @@ -145,7 +118,9 @@ impl TextDecoderStream { /// #[getter] fn encoding(&self) -> String { - self.data().encoding_name.clone() + // Step 1: Return this’s encoding’s name, ASCII lowercased. + let encoding = self.data().encoding.expect("set by the constructor"); + encoding.name().to_ascii_lowercase() } /// @@ -179,74 +154,46 @@ fn decoder_transform_cb( args: CallbackArgs<'_>, payload: HandleValue<'_>, ) -> Result { - let stream: TextDecoderStream<'_> = Object::from_value(scope, *payload) - .expect("payload is an object") - .cast::>() - .expect("payload is a TextDecoderStream"); + let stream = cast_payload::(scope, payload); // Step 1: Let bufferSource be the result of converting chunk to an AllowSharedBufferSource. - let chunk = args.get(0); - let bytes = buffer_source_to_bytes(scope, chunk)?; - - // Step 2: Push a copy of bufferSource to decoder's I/O queue. - // Step 3: Let output be the I/O queue of scalar values « end-of-queue ». - // Step 4: Process I/O queue. - // (Steps 2-4 are collapsed: per the encoding_rs Decoder contract - // (https://docs.rs/encoding_rs/0.8/encoding_rs/struct.Decoder.html), `src` is fully - // consumed when the result is `InputEmpty`, which is guaranteed here by pre-allocating - // via `max_utf8_buffer_length`. The Decoder tracks partial multi-byte sequences - // internally, so we pass the chunk directly without queuing.) - let mut output = String::new(); - - if !bytes.is_empty() { + let buffer_source = ArrayBufferViewOrArrayBuffer::from_jsval_throwing(scope, args.get(0), ())?; + let output = { + // Step 2: Push a copy of bufferSource to decoder’s I/O queue. + // Step 3: Let output be the I/O queue of scalar values « end-of-queue ». + // Step 4: While true: + // Step 4.1: Let item be the result of reading from decoder’s I/O queue. + // Step 4.2: If item is end-of-queue: + // Step 4.2.1: Let outputChunk be the result of running serialize I/O queue with decoder + // and output. + // Step 4.2.2: If outputChunk is not the empty string, then enqueue outputChunk in + // decoder’s TransformStream. + // Step 4.2.3: Return. + // Step 4.3: Let result be the result of processing an item with item, decoder’s decoder, + // decoder’s I/O queue, output, and decoder’s error mode. + // Step 4.4: If result is error, then throw a TypeError. + // Implementation note: the Encoding Standard permits avoiding the copy when input cannot + // affect future decode calls. The slice is consumed synchronously here and + // `encoding_rs::Decoder` retains only its own state. `decode_bytes_to_js_string` performs + // steps 3-4 with `last = false`, preserving decoder state for the next chunk. + // SAFETY: `buffer_source` is rooted, and the slice is only borrowed for the decode call + // below, which matches `TextDecoder::decode`. + // TODO: `decode_bytes_to_js_string`'s windows-1252 fast path creates the JS string + // directly from `bytes`. That allocation can GC and move a small buffer's inline data + // while the slice is still in use. Copy the bytes before allocating in that fast path. + let bytes = match buffer_source { + ArrayBufferViewOrArrayBuffer::View(view) => unsafe { view.bytes() }, + ArrayBufferViewOrArrayBuffer::Buffer(buffer) => unsafe { buffer.bytes() }, + }; let mut data = stream.data_mut(); + let encoding = data.encoding.expect("encoding must be initialized"); let fatal = data.fatal; let decoder = data.decoder.as_mut().expect("decoder must be initialized"); + decode_bytes_to_js_string(scope, encoding, decoder, fatal, bytes, false, false)? + }; - if fatal { - // Step 4.3: Process through fatal decoder. - if let Some(needed) = decoder.max_utf8_buffer_length_without_replacement(bytes.len()) { - output.reserve(needed); - } - let (result, _read) = - decoder.decode_to_string_without_replacement(&bytes, &mut output, false); - - match result { - encoding_rs::DecoderResult::InputEmpty => {} - // Step 4.4: If error, throw a TypeError. - encoding_rs::DecoderResult::Malformed(_, _) => { - return Err(js::error::throw_type_error( - scope, - c"The encoded data was not valid.", - )); - } - encoding_rs::DecoderResult::OutputFull => { - unreachable!("Output buffer was too small despite max_utf8_buffer_length_without_replacement allocation"); - } - } - } else { - // Step 4.3: Process through replacement decoder. - if let Some(needed) = decoder.max_utf8_buffer_length(bytes.len()) { - output.reserve(needed); - } - match decoder.decode_to_string(&bytes, &mut output, false) { - (encoding_rs::CoderResult::InputEmpty, _, _) => {} - (encoding_rs::CoderResult::OutputFull, _, _) => { - unreachable!("Output buffer was too small despite max_utf8_buffer_length allocation"); - } - } - } - } - - // Step 4.2.1 & 4.2.2: If outputChunk is not the empty string, then enqueue outputChunk. if !output.is_empty() { - let chunk_val = js::JSString::from_str(scope, &output) - .map_err(|_| js::error::throw_type_error(scope, c"unable to create string"))?; - stream - .data() - .transform - .get(scope) - .enqueue(scope, scope.root_value(chunk_val.as_value()))?; + enqueue_to_transform(scope, &stream, output)?; } // Step 4.2.3: Return. @@ -259,107 +206,43 @@ fn decoder_flush_cb( _args: CallbackArgs<'_>, payload: HandleValue<'_>, ) -> Result { - let stream: TextDecoderStream<'_> = Object::from_value(scope, *payload) - .expect("payload is an object") - .cast::>() - .expect("payload is a TextDecoderStream"); + let stream = cast_payload::(scope, payload); // Step 1: Let output be the I/O queue of scalar values « end-of-queue ». - // (No I/O queue to drain — per the encoding_rs Decoder contract - // (https://docs.rs/encoding_rs/0.8/encoding_rs/struct.Decoder.html), partial sequences - // are held in the Decoder's internal state. We pass an empty slice with `last=true` - // to flush that state.) - let mut output = String::new(); - - { + // Step 2: While true: + // Step 2.1: Let item be the result of reading from decoder’s I/O queue. + // Step 2.2: Let result be the result of processing an item with item, decoder’s decoder, + // decoder’s I/O queue, output, and decoder’s error mode. + // Step 2.3: If result is finished: + // Step 2.3.1: Let outputChunk be the result of running serialize I/O queue with decoder and + // output. + // Step 2.3.2: If outputChunk is not the empty string, then enqueue outputChunk in decoder’s + // TransformStream. + // Step 2.3.3: Return. + // Step 2.4: Otherwise, if result is error, throw a TypeError. + // Implementation note: `encoding_rs::Decoder` holds incomplete input internally, so an empty + // slice with `last = true` supplies end-of-queue and processes steps 2.1-2.4 to completion. + let output = { let mut data = stream.data_mut(); + let encoding = data.encoding.expect("encoding must be initialized"); let fatal = data.fatal; let decoder = data.decoder.as_mut().expect("decoder must be initialized"); + decode_bytes_to_js_string(scope, encoding, decoder, fatal, &[], true, false)? + }; - // Step 2: While true: - // (Instead of a manual byte-by-byte loop, we pass `last = true` to encoding_rs. - // This instructs the native decoder to finalize, process any dangling bytes, - // and reset its shift or multibyte state. This acts as our loop termination). - if fatal { - // Pre-reserve using the without-replacement bound. - if let Some(needed) = decoder.max_utf8_buffer_length_without_replacement(0) { - output.reserve(needed); - } - // Step 2.2: Let result be processing an item with item, decoder, output, and error mode. - let (result, _read) = - decoder.decode_to_string_without_replacement(&[], &mut output, true); - - match result { - // Step 2.3: If result is finished - encoding_rs::DecoderResult::InputEmpty => {} - // Step 2.4: If result is error, throw a TypeError. - encoding_rs::DecoderResult::Malformed(_, _) => { - return Err(js::error::throw_type_error( - scope, - c"The encoded data was not valid.", - )); - } - encoding_rs::DecoderResult::OutputFull => { - unreachable!("Output buffer was too small during fatal flush"); - } - } - } else { - // Pre-reserve using the with-replacement bound (accounts for pending U+FFFD output). - if let Some(needed) = decoder.max_utf8_buffer_length(0) { - output.reserve(needed); - } - // Step 2.2: Let result be processing an item with item, decoder, output, and error mode. - match decoder.decode_to_string(&[], &mut output, true) { - (encoding_rs::CoderResult::InputEmpty, _, _) => {} - (encoding_rs::CoderResult::OutputFull, _, _) => { - unreachable!("Output buffer was too small during replacement flush"); - } - } - } - } - - // Step 2.3.1: Let outputChunk be the result of running serialize I/O queue with decoder and output.(handled by encoding_rs) - // Step 2.3.2: If outputChunk is not the empty string, then enqueue outputChunk in decoder's transform. if !output.is_empty() { - let chunk_val = js::JSString::from_str(scope, &output) - .map_err(|_| js::error::throw_type_error(scope, c"unable to create string"))?; - stream - .data() - .transform - .get(scope) - .enqueue(scope, scope.root_value(chunk_val.as_value()))?; + enqueue_to_transform(scope, &stream, output)?; } // Step 2.3.3: Return. Ok(value::undefined()) } -fn buffer_source_to_bytes( +fn enqueue_to_transform( scope: &Scope<'_>, - value: HandleValue<'_>, -) -> Result, ExnThrown> { - let obj = Object::from_value(scope, *value).map_err(|_| { - js::error::throw_type_error(scope, c"argument is not an ArrayBuffer or ArrayBufferView") - })?; - - // Try ArrayBufferView first (typed arrays, DataView). - if let Ok(view) = obj.cast::>() { - // SAFETY: `view` is rooted; we copy the bytes immediately so the - // borrow doesn't outlive any GC-triggering operation. The copy also - // satisfies the spec requirement for SharedArrayBuffer-backed views. - return Ok(unsafe { view.bytes() }.to_vec()); - } - - // Try ArrayBuffer. - if let Ok(buf) = obj.cast::>() { - // SAFETY: `buf` is rooted; same constraints as above. - // Note: If the ArrayBuffer is detached, bytes() returns an empty slice. - return Ok(unsafe { buf.bytes() }.to_vec()); - } - - // Not a valid buffer source. - Err(js::error::throw_type_error( - scope, - c"argument is not an ArrayBuffer or ArrayBufferView", - )) + stream: &TextDecoderStream<'_>, + chunk: JSString<'_>, +) -> Result<(), ExnThrown> { + let transform = stream.data().transform.get(scope); + transform.enqueue(scope, scope.root_value(chunk.as_value())) } diff --git a/crates/builtins/encoding/src/text_encoder_stream.rs b/crates/builtins/encoding/src/text_encoder_stream.rs index 2eabc5b3..4139347e 100644 --- a/crates/builtins/encoding/src/text_encoder_stream.rs +++ b/crates/builtins/encoding/src/text_encoder_stream.rs @@ -4,33 +4,16 @@ use core_runtime::{webidl_interface, webidl_methods}; use js::error::ExnThrown; +use js::function::cast_payload; use js::gc::handle::Heap; use js::gc::scope::Scope; use js::native::Value; use js::prelude::{CallbackArgs, HandleValue}; -use js::{value, Function, JSString, Object, Uint8Array}; +use js::{value, JSString, Uint8Array}; -use std::collections::VecDeque; use web_streams::readable::ReadableStream; -use web_streams::{TransformStream, TransformStreamImpl}; use web_streams::writable::WritableStream; - -/// Extract raw UTF-16 code units from a JS string, preserving surrogates. -/// -/// step 1: -/// "Let input be the result of converting chunk to a DOMString." -fn js_string_to_utf16(scope: &Scope<'_>, js_string: &JSString) -> Vec { - let length = js_string.len(); - if length == 0 { - return Vec::new(); - } - - let mut result = Vec::with_capacity(length); - for i in 0..length { - result.push(js_string.char_at(scope, i).unwrap_or(0xFFFD)); - } - result -} +use web_streams::{AlgorithmArg, TransformStream, TransformStreamImpl}; /// #[webidl_interface] @@ -51,55 +34,28 @@ impl TextEncoderStream { #[constructor] fn new(&self, scope: &Scope<'_>) -> Result<(), ExnThrown> { // Step 1: Set this’s [[encoder]] to an instance of the UTF-8 encoder. - // (Note: The UTF-8 encoder is stateless, so no explicit assignment is required). - - let self_payload = scope.root_value(self.as_value()); - - // Step 2: Let transformAlgorithm be an algorithm which takes a chunk argument - // and runs the encode and enqueue a chunk algorithm with this and chunk. - let transform_cb = Function::new_callback( - scope, - c"textEncoderTransform", - 1, - encoder_transform_cb, - self_payload, - )?; - - // Step 3: Let flushAlgorithm be an algorithm which takes no arguments - // and runs the encode and flush algorithm with this. - let flush_cb = Function::new_callback( - scope, - c"textEncoderFlush", - 0, - encoder_flush_cb, - self_payload, - )?; - + // The UTF-8 encoder is stateless, so no explicit assignment is required. + // Step 2: Let transformAlgorithm be an algorithm which takes a chunk argument and runs + // the encode and enqueue a chunk algorithm with this and chunk. + let transform = AlgorithmArg::native(scope, encoder_transform_cb, self)?; + // Step 3: Let flushAlgorithm be an algorithm which runs the encode and flush algorithm + // with this. + let flush = AlgorithmArg::native(scope, encoder_flush_cb, self)?; // Step 4: Let transformStream be a new TransformStream. - // Step 5: Set up transformStream with transformAlgorithm set to transformAlgorithm - // and flushAlgorithm set to flushAlgorithm. - // We pass both algorithms as properties on a plain transformer object so that the - // TransformStream constructor sees a real transformer and does not set the - // identity-transform shortcut — otherwise `pipeTo` would bypass encoding for native - // byte sources. - let transformer = Object::new_plain(scope)?; - transformer.set_property(scope, c"transform", transform_cb)?; - transformer.set_property(scope, c"flush", flush_cb)?; - let transform_stream = TransformStream::new(scope, Some(scope.root_value(transformer.as_value())), None, None)?; - + // Step 5: Set up transformStream with transformAlgorithm set to transformAlgorithm and + // flushAlgorithm set to flushAlgorithm. + let transform_stream = + TransformStream::new_native(scope, transform, flush, AlgorithmArg::None)?; // Step 6: Set this’s [[TransformStream]] to transformStream. self.data_mut().transform.set(transform_stream); - // Step 7: Set this’s [[pending high surrogate]] to null. - self.data_mut().leading_surrogate = None; - Ok(()) } /// #[getter] - fn encoding(&self) -> String { - "utf-8".into() + fn encoding(&self) -> &'static str { + "utf-8" } /// @@ -121,69 +77,96 @@ fn encoder_transform_cb( args: CallbackArgs<'_>, payload: HandleValue<'_>, ) -> Result { - let stream: TextEncoderStream<'_> = Object::from_value(scope, *payload) - .expect("payload is an object") - .cast::>() - .expect("payload is a TextEncoderStream"); + let stream = cast_payload::(scope, payload); // Step 1: Let input be the result of converting chunk to a DOMString. let js_string = JSString::from_value(scope, args.get(0))?; - - // Step 2: Convert input into an I/O queue of code units. - let mut input = VecDeque::from(js_string_to_utf16(scope, &js_string)); - + // Step 2: Convert input to an I/O queue of code units. // Step 3: Let output be the I/O queue of bytes « end-of-queue ». - let mut output: Vec = Vec::new(); - // Step 4: While true: - loop { - // Step 4.1: Let item be the result of reading from input. - let item = input.pop_front(); - - // Step 4.2: If item is end-of-queue: - let Some(item) = item else { - // Step 4.2.1: Convert output into a byte sequence. - // Step 4.2.2: If output is not empty, then enqueue Uint8Array(output) in encoder’s [[TransformStream]]. - if !output.is_empty() { - let u8array = Uint8Array::with_data(scope, &output)?; - enqueue_to_transform(scope, &stream, u8array)?; - } - // Step 4.2.3: Return. - break; - }; - - // Step 4.3: Let result be the result of running convert code unit to scalar value with encoder, item, and input. - // Scope the mutable borrow so it is dropped before any JS reentry (enqueue_to_transform). - let result = { - let mut data = stream.data_mut(); - convert_code_unit_to_scalar_value(&mut data, item, &mut input) + // Step 4.1: Let item be the result of reading from input. + // Step 4.2: If item is end-of-queue: + // Step 4.2.1: Convert output into a byte sequence. + // Step 4.2.2: If output is not empty: + // Step 4.2.2.1: Let chunk be the result of creating a Uint8Array object given output and + // encoder’s relevant realm. + // Step 4.2.2.2: Enqueue chunk into encoder’s TransformStream. + // Step 4.2.3: Return. + // Step 4.3: Let result be the result of executing the convert code unit to scalar value + // algorithm with encoder, item, and input. + // Step 4.4: If result is not continue, then process an item with result, encoder’s encoder, + // input, output, and "fatal". + // Implementation note: a surrogate pair can only be split at the chunk's edges, so only the + // first and last code units need the [[pending high surrogate]] handling. Everything between + // is encoded in bulk by `to_utf8`, which replaces lone surrogates with U+FFFD exactly as + // steps 4.3 and 4.4 do. + let mut start = 0; + let mut end = js_string.len(); + if end == 0 { + return Ok(value::undefined()); + } + let mut output = Vec::new(); + let pending = stream.data_mut().leading_surrogate.take(); + if let Some(leading) = pending { + let first = js_string.char_at(scope, 0)?; + if is_trailing_surrogate(first) { + let c = 0x10000 + ((u32::from(leading) - 0xD800) << 10) + (u32::from(first) - 0xDC00); + push_scalar_value(&mut output, c); + start = 1; + } else { + push_scalar_value(&mut output, 0xFFFD); + } + } + if start < end { + let last = js_string.char_at(scope, end - 1)?; + if is_leading_surrogate(last) { + stream.data_mut().leading_surrogate = Some(last); + end -= 1; + } + } + if start < end { + let middle = if start == 0 && end == js_string.len() { + js_string + } else { + js_string.substring(scope, start, end - start)? }; - - // Step 4.4: If result is not continue, then process an item with result, encoder’s encoder, input, output, and "fatal". - // (Note: For UTF-8, "processing an item" is encoding the scalar value to its byte sequence and appending to output) - match result { - EncodeResult::Continue => { - // Do nothing, continue the loop. - } - EncodeResult::Scalar(sv) => { - encode_scalar_value(&mut output, sv); - } + let utf8 = middle.to_utf8(scope)?.into_bytes(); + if output.is_empty() { + output = utf8; + } else { + output.extend_from_slice(&utf8); } } + if !output.is_empty() { + let u8array = Uint8Array::with_data(scope, &output)?; + enqueue_to_transform(scope, &stream, u8array)?; + } + Ok(value::undefined()) } +fn is_leading_surrogate(unit: u16) -> bool { + (0xD800..=0xDBFF).contains(&unit) +} + +fn is_trailing_surrogate(unit: u16) -> bool { + (0xDC00..=0xDFFF).contains(&unit) +} + +/// Append the UTF-8 encoding of the scalar value `sv` to `output`. +fn push_scalar_value(output: &mut Vec, sv: u32) { + let c = char::from_u32(sv).expect("a scalar value"); + output.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()); +} + /// The algorithm. fn encoder_flush_cb( scope: &Scope<'_>, _args: CallbackArgs<'_>, payload: HandleValue<'_>, ) -> Result { - let stream: TextEncoderStream<'_> = Object::from_value(scope, *payload) - .expect("payload is an object") - .cast::>() - .expect("payload is a TextEncoderStream"); + let stream = cast_payload::(scope, payload); // Step 1: If encoder’s [[pending high surrogate]] is non-null: // Scope the mutable borrow so it is dropped before any JS reentry. @@ -201,66 +184,6 @@ fn encoder_flush_cb( Ok(value::undefined()) } -/// Result of the convert code unit to scalar value algorithm. -enum EncodeResult { - /// Return `continue` per spec step 2. - Continue, - /// A scalar value to encode. - Scalar(u32), -} - -/// The algorithm. -fn convert_code_unit_to_scalar_value( - data: &mut TextEncoderStreamImpl, - item: u16, - input: &mut VecDeque, -) -> EncodeResult { - // Step 1: If encoder’s leading surrogate is non-null: - if let Some(leading) = data.leading_surrogate.take() { - // Step 1.1: Let high be encoder’s leading surrogate. - // Step 1.2: Set encoder’s leading surrogate to null. (Done above by take()) - - // Step 1.3: If item is a trailing surrogate (U+DC00 to U+DFFF), then return - // the scalar value from surrogates given leadingSurrogate and item. - if (0xDC00..=0xDFFF).contains(&item) { - // To obtain a scalar value from surrogates, given a leading surrogate leading and a trailing surrogate trailing, - // return 0x10000 + ((leading − 0xD800) << 10) + (trailing − 0xDC00). - let sv = 0x10000 + ((leading as u32 - 0xD800) << 10) + (item as u32 - 0xDC00); - return EncodeResult::Scalar(sv); - } - - // Step 1.4: Restore item to input. - input.push_front(item); - - // Step 1.5: Return U+FFFD. - return EncodeResult::Scalar(0xFFFD); - } - - // Step 2: If item is a leading surrogate (U+D800 to U+DBFF): - if (0xD800..=0xDBFF).contains(&item) { - // Step 2.1: Set encoder’s [[pending high surrogate]] to item. - data.leading_surrogate = Some(item); - // Step 2.2: Return continue. - return EncodeResult::Continue; - } - - // Step 3: If item is a trailing surrogate (U+DC00 to U+DFFF), then return U+FFFD. - if (0xDC00..=0xDFFF).contains(&item) { - return EncodeResult::Scalar(0xFFFD); - } - - // Step 4: Return the scalar value item. - EncodeResult::Scalar(item as u32) -} - -/// Encode a scalar value as UTF-8 and append to output. -fn encode_scalar_value(output: &mut Vec, sv: u32) { - let mut buf = [0u8; 4]; - let c = char::from_u32(sv).unwrap_or('\u{FFFD}'); - let len = c.encode_utf8(&mut buf).len(); - output.extend_from_slice(&buf[..len]); -} - /// Enqueue a chunk into the TextEncoderStream's TransformStream. fn enqueue_to_transform( scope: &Scope<'_>, @@ -268,5 +191,6 @@ fn enqueue_to_transform( chunk: Uint8Array<'_>, ) -> Result<(), ExnThrown> { let chunk_val = scope.root_value(chunk.as_value()); - stream.data().transform.get(scope).enqueue(scope, chunk_val) + let transform = stream.data().transform.get(scope); + transform.enqueue(scope, chunk_val) } diff --git a/crates/builtins/web-streams/src/transform/transform_stream.rs b/crates/builtins/web-streams/src/transform/transform_stream.rs index 1652ade5..c1613672 100644 --- a/crates/builtins/web-streams/src/transform/transform_stream.rs +++ b/crates/builtins/web-streams/src/transform/transform_stream.rs @@ -11,6 +11,7 @@ use crate::algorithms::extract_size_algorithm; use crate::queuing::QueuingStrategy; use crate::readable::readable_stream::ReadableStreamImpl; use crate::readable::ReadableStream; +use crate::support::AlgorithmArg; use crate::writable::writable_stream::WritableStreamImpl; use crate::writable::WritableStream; use core_runtime::{webidl_interface, webidl_methods}; @@ -186,6 +187,57 @@ impl TransformStream { /// Helpers for the encoding crate — not part of the WebIDL interface. impl TransformStream<'_> { + /// + /// + /// Create a TransformStream driven by native algorithms, without exposing a transformer + /// object to JavaScript. + pub fn new_native<'r>( + scope: &'r Scope<'_>, + transform: AlgorithmArg<'r>, + flush: AlgorithmArg<'r>, + cancel: AlgorithmArg<'r>, + ) -> Result, ExnThrown> { + let stream = js::class::create_instance_with::(scope, |_| { + TransformStreamImpl::default() + })?; + // Step 1: Let writableHighWaterMark be 1. + // Step 2: Let writableSizeAlgorithm be an algorithm that returns 1. + // Step 3: Let readableHighWaterMark be 0. + // Step 4: Let readableSizeAlgorithm be an algorithm that returns 1. + // Steps 5-7: Let transformAlgorithmWrapper, flushAlgorithmWrapper, and + // cancelAlgorithmWrapper run the given algorithms, converting their results + // and exceptions to promises. + // (The controller's algorithm invocation performs this wrapping.) + // Step 8: Let startPromise be a promise resolved with undefined. + let start_promise = Promise::new_resolved_with_value(scope, HandleValue::undefined())?; + // Step 9: Perform ! InitializeTransformStream(stream, startPromise, + // writableHighWaterMark, writableSizeAlgorithm, readableHighWaterMark, + // readableSizeAlgorithm). + algorithms::initialize_transform_stream( + scope, + &stream, + &start_promise, + 1.0, + HandleValue::undefined(), + 0.0, + HandleValue::undefined(), + )?; + // Step 10: Let controller be a new TransformStreamDefaultController. + let controller = TransformStreamDefaultController::new(scope, stream)?; + // Step 11: Perform ! SetUpTransformStreamDefaultController(stream, controller, + // transformAlgorithmWrapper, flushAlgorithmWrapper, cancelAlgorithmWrapper). + algorithms::set_up_transform_stream_default_controller( + scope, + &stream, + &controller, + transform, + flush, + cancel, + HandleValue::undefined(), + ); + Ok(stream) + } + /// Enqueue a chunk into the readable side of this TransformStream. pub fn enqueue(&self, scope: &Scope<'_>, chunk: HandleValue<'_>) -> Result<(), ExnThrown> { let ctrl = self From 2b30753ff6039d9ddb16671f07cac659aa95b02c Mon Sep 17 00:00:00 2001 From: Karthik Ganeshram Date: Tue, 20 Oct 2026 18:04:53 +0530 Subject: [PATCH 3/5] address GC safety in fast path Signed-off-by: Karthik Ganeshram --- .../encoding/src/text_decoder_stream.rs | 7 +- crates/builtins/encoding/tests/encoding.rs | 69 +++++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/crates/builtins/encoding/src/text_decoder_stream.rs b/crates/builtins/encoding/src/text_decoder_stream.rs index 75ae45a1..c199c9ca 100644 --- a/crates/builtins/encoding/src/text_decoder_stream.rs +++ b/crates/builtins/encoding/src/text_decoder_stream.rs @@ -176,11 +176,8 @@ fn decoder_transform_cb( // affect future decode calls. The slice is consumed synchronously here and // `encoding_rs::Decoder` retains only its own state. `decode_bytes_to_js_string` performs // steps 3-4 with `last = false`, preserving decoder state for the next chunk. - // SAFETY: `buffer_source` is rooted, and the slice is only borrowed for the decode call - // below, which matches `TextDecoder::decode`. - // TODO: `decode_bytes_to_js_string`'s windows-1252 fast path creates the JS string - // directly from `bytes`. That allocation can GC and move a small buffer's inline data - // while the slice is still in use. Copy the bytes before allocating in that fast path. + // SAFETY: `buffer_source` is rooted, and the helper consumes the slice or copies it into + // Rust-owned output before any JS allocation can GC and move the input buffer. let bytes = match buffer_source { ArrayBufferViewOrArrayBuffer::View(view) => unsafe { view.bytes() }, ArrayBufferViewOrArrayBuffer::Buffer(buffer) => unsafe { buffer.bytes() }, diff --git a/crates/builtins/encoding/tests/encoding.rs b/crates/builtins/encoding/tests/encoding.rs index 16ec7f8f..39de5833 100644 --- a/crates/builtins/encoding/tests/encoding.rs +++ b/crates/builtins/encoding/tests/encoding.rs @@ -392,6 +392,42 @@ fn text_decoder_windows_1252() { ); } +#[test] +fn text_decoder_latin1_fast_paths_gc_safe() { + assert_eq!( + eval( + r#" + for (const encoding of ["utf-8", "windows-1252"]) { + for (const fatal of [false, true]) { + const decoder = new TextDecoder(encoding, { fatal }); + for (const length of [0, 1, 24, 25, 64, 256]) { + const bytes = new Uint8Array(length); + for (let i = 0; i < length; i++) { + bytes[i] = encoding === "utf-8" ? 65 + i % 26 : 0xA0 + i % 96; + } + const padded = new Uint8Array(length + 4); + padded.set(bytes, 2); + const expected = String.fromCharCode(...bytes); + for (const input of [ + bytes, + bytes.buffer, + padded.subarray(2, length + 2), + new DataView(padded.buffer, 2, length), + ]) { + if (decoder.decode(input) !== expected) { + throw new Error(`Incorrect ${encoding} fast-path output`); + } + } + } + } + } + true + "#, + ), + "true" + ); +} + // ── WebIDL interface compliance ── #[test] @@ -705,3 +741,36 @@ fn text_decoder_stream_windows_1252() { )); assert_eq!(out, "é"); } + +#[test] +fn text_decoder_stream_windows_1252_gc_safe() { + let out = run_async(&format!( + r#"{DRAIN_STR} + globalThis.__out = "pending"; + const inputs = []; + const expected = []; + for (const length of [1, 24, 25, 64]) {{ + const bytes = new Uint8Array(length).fill(0xE9); + const padded = new Uint8Array(length + 4); + padded.set(bytes, 2); + inputs.push( + bytes, + bytes.buffer, + padded.subarray(2, length + 2), + new DataView(padded.buffer, 2, length), + ); + expected.push("\u00E9".repeat(length * 4)); + }} + const rs = new ReadableStream({{ + start(c) {{ + for (const input of inputs) c.enqueue(input); + c.close(); + }} + }}); + drainStr(rs.pipeThrough(new TextDecoderStream("windows-1252"))) + .then(a => {{ globalThis.__out = String(a.join("") === expected.join("")); }}) + .catch(e => {{ globalThis.__out = String(e); }}); + "# + )); + assert_eq!(out, "true"); +} From aa766f608122f25d29b861f6512759b54bdca8fc Mon Sep 17 00:00:00 2001 From: Karthik Ganeshram Date: Tue, 3 Nov 2026 10:19:04 +0100 Subject: [PATCH 4/5] address review comments Signed-off-by: Karthik Ganeshram --- Cargo.lock | 25 +- Cargo.toml | 2 +- crates/builtins/encoding/Cargo.toml | 8 +- .../builtins/encoding/src/decoder_common.rs | 147 ++++ crates/builtins/encoding/src/text_decoder.rs | 138 +--- .../encoding/src/text_decoder_stream.rs | 37 +- .../encoding/src/text_encoder_stream.rs | 5 +- crates/builtins/encoding/tests/encoding.rs | 759 ++---------------- crates/libstarling/Cargo.toml | 2 +- crates/libstarling/src/lib.rs | 2 +- scripts/test-gc-zeal.sh | 2 +- 11 files changed, 263 insertions(+), 864 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a282825f..f287f667 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -532,18 +532,6 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" -[[package]] -name = "encoding" -version = "0.3.0" -dependencies = [ - "core-runtime", - "encoding_rs", - "js", - "libstarling", - "web-globals", - "web-streams", -] - [[package]] name = "encoding_c" version = "0.9.8" @@ -1520,7 +1508,6 @@ dependencies = [ "bytes", "core-runtime", "cpp-builtins", - "encoding", "event-listener", "fetch-event", "futures-channel", @@ -1537,6 +1524,7 @@ dependencies = [ "tempfile", "tokio", "url", + "web-encoding", "web-fetch", "web-globals", "web-streams", @@ -3091,6 +3079,17 @@ dependencies = [ "semver", ] +[[package]] +name = "web-encoding" +version = "0.3.0" +dependencies = [ + "core-runtime", + "encoding_rs", + "js", + "web-globals", + "web-streams", +] + [[package]] name = "web-fetch" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 3a8834a6..5585210e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,7 +67,7 @@ web-globals = { path = "crates/builtins/web-globals" } web-streams = { path = "crates/builtins/web-streams" } web-url = { path = "crates/builtins/web-url" } cpp-builtins = { path = "crates/builtins/cpp-builtins" } -encoding = { path = "crates/builtins/encoding" } +web-encoding = { path = "crates/builtins/encoding" } [profile.release] lto = "thin" diff --git a/crates/builtins/encoding/Cargo.toml b/crates/builtins/encoding/Cargo.toml index 22e2d62d..d96b635b 100644 --- a/crates/builtins/encoding/Cargo.toml +++ b/crates/builtins/encoding/Cargo.toml @@ -1,11 +1,14 @@ [package] -name = "encoding" +name = "web-encoding" version.workspace = true repository.workspace = true license.workspace = true authors.workspace = true edition.workspace = true +[features] +debugmozjs = ["js/debugmozjs"] + [dependencies] core-runtime.workspace = true encoding_rs.workspace = true @@ -13,10 +16,7 @@ js.workspace = true web-streams.workspace = true [dev-dependencies] -js.workspace = true -libstarling.workspace = true web-globals.workspace = true -web-streams.workspace = true [lints] workspace = true diff --git a/crates/builtins/encoding/src/decoder_common.rs b/crates/builtins/encoding/src/decoder_common.rs index fc1d6226..83bff1d3 100644 --- a/crates/builtins/encoding/src/decoder_common.rs +++ b/crates/builtins/encoding/src/decoder_common.rs @@ -1,5 +1,21 @@ // SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception +use core_runtime::webidl_union; +use js::error::{ExnThrown, OutOfMemory, ThrowException}; +use js::gc::scope::Scope; +use js::{ArrayBuffer, ArrayBufferView, JSString}; + +pub(crate) fn get_encoding( + label: Option<&str>, +) -> Result<&'static encoding_rs::Encoding, js::error::RangeError> { + let label = label.unwrap_or("utf-8"); + encoding_rs::Encoding::for_label_no_replacement(label.as_bytes()).ok_or_else(|| { + js::error::RangeError(format!( + "The encoding label provided ('{label}') is invalid." + )) + }) +} + /// Creates a new streaming decoder for the given encoding, with or without BOM handling. /// /// Per the Encoding spec: ignoreBOM=true means "ignore the BOM's special meaning" @@ -16,6 +32,137 @@ pub fn new_decoder( } } +pub(crate) fn decode_bytes( + encoding: &'static encoding_rs::Encoding, + decoder: &mut encoding_rs::Decoder, + fatal: bool, + bytes: &[u8], + last: bool, + fresh: bool, +) -> Decoded { + // windows-1252 decodes every byte outside 0x80..=0x9F to the code point of the same + // value, and its decoder keeps no state between calls, so such input is its own + // Latin-1 output. + if encoding == encoding_rs::WINDOWS_1252 && !bytes.iter().any(|b| (0x80..=0x9F).contains(b)) { + return Decoded::Latin1(js::typedarray::try_copy(bytes)); + } + // A fresh decoder that decodes its whole input at once turns ASCII into the same + // characters in every ASCII-compatible encoding. + if fresh + && last + && encoding.is_ascii_compatible() + && encoding_rs::Encoding::ascii_valid_up_to(bytes) == bytes.len() + { + return Decoded::Latin1(js::typedarray::try_copy(bytes)); + } + + // Step 4: Let _output_ be the `I/O queue` of scalar values « `end-of-queue` ». + // Step 5: While true: + // Step 5.1: Let _item_ be the result of `reading` from `this`'s `I/O queue`. + // Step 5.2: If _item_ is `end-of-queue` and `this`'s `do not flush` is true, then + // return the result of running `serialize I/O queue` with `this` and + // _output_. + // Step 5.3: Otherwise: + // Step 5.3.1: Let _result_ be the result of `processing an item` with _item_, + // `this`'s `decoder`, `this`'s `I/O queue`, _output_, and `this`'s `error + // mode`. + // Step 5.3.2: If _result_ is `finished`, then return the result of running + // `serialize I/O queue` with `this` and _output_. + // Step 5.3.3: Otherwise, if _result_ is `error`, `throw` a ``TypeError``. + // + // (One `encoding_rs` call processes every item. `last` is true when `end-of-queue` + // is processed rather than returned at step 5.2. The decoder's BOM removal performs + // `serialize I/O queue`'s BOM handling. We decode to UTF-16, which SpiderMonkey + // copies as is, or narrows to Latin-1 if every code unit fits.) + Decoded::Utf16(decode_to_utf16(decoder, bytes, last, fatal)) +} + +/// The owned output of decoding bytes, ready to convert to a JS string. +pub(crate) enum Decoded { + /// A copy of the input, which is its own Latin-1 output. `None` means that allocating the + /// copy failed. + Latin1(Option>), + /// The UTF-16 output. + Utf16(Result, DecodeError>), +} + +impl Decoded { + pub(crate) fn into_js_string<'r>( + self, + scope: &'r Scope<'_>, + ) -> Result, ExnThrown> { + match self { + Self::Latin1(Some(chars)) => JSString::from_latin1(scope, &chars), + Self::Latin1(None) => Err(OutOfMemory.throw(scope)), + Self::Utf16(output) => { + let output = output.map_err(|error| error.throw(scope))?; + JSString::from_utf16(scope, &output) + } + } + } +} + +/// Why [`decode_to_utf16`] failed. +pub(crate) enum DecodeError { + /// The input is malformed, in the "fatal" error mode. + Malformed, + /// The output can't be allocated. + OutOfMemory, +} + +impl DecodeError { + /// Throw the `TypeError` of malformed input, or report the failed allocation. + fn throw(self, scope: &Scope<'_>) -> ExnThrown { + match self { + DecodeError::Malformed => { + js::error::throw_type_error(scope, c"The encoded data was not valid.") + } + DecodeError::OutOfMemory => OutOfMemory.throw(scope), + } + } +} + +/// Decode `bytes` with `decoder` to UTF-16. `last` is true for the end of the input, which the +/// decoder then flushes. With `fatal`, malformed input fails, and otherwise it becomes U+FFFD. +fn decode_to_utf16( + decoder: &mut encoding_rs::Decoder, + bytes: &[u8], + last: bool, + fatal: bool, +) -> Result, DecodeError> { + let max_len = decoder + .max_utf16_buffer_length(bytes.len()) + .ok_or(DecodeError::OutOfMemory)?; + let mut output = Vec::new(); + output + .try_reserve_exact(max_len) + .map_err(|_| DecodeError::OutOfMemory)?; + output.resize(max_len, 0); + let written = if fatal { + let (result, _read, written) = + decoder.decode_to_utf16_without_replacement(bytes, &mut output, last); + match result { + encoding_rs::DecoderResult::InputEmpty => written, + encoding_rs::DecoderResult::Malformed(_, _) => return Err(DecodeError::Malformed), + encoding_rs::DecoderResult::OutputFull => { + unreachable!("output buffer was pre-allocated to max size"); + } + } + } else { + let (_result, _read, written, _had_errors) = + decoder.decode_to_utf16(bytes, &mut output, last); + written + }; + output.truncate(written); + Ok(output) +} + +#[webidl_union] +pub enum ArrayBufferViewOrArrayBuffer<'a> { + View(ArrayBufferView<'a>), + Buffer(ArrayBuffer<'a>), +} + /// Options dictionary for TextDecoder and TextDecoderStream constructors. #[core_runtime::webidl_dictionary] #[derive(Default)] diff --git a/crates/builtins/encoding/src/text_decoder.rs b/crates/builtins/encoding/src/text_decoder.rs index cf74d91c..a1f9fdbc 100644 --- a/crates/builtins/encoding/src/text_decoder.rs +++ b/crates/builtins/encoding/src/text_decoder.rs @@ -2,13 +2,14 @@ //! -use core_runtime::{webidl_dictionary, webidl_interface, webidl_methods, webidl_union}; -use js::error::{ExnThrown, OutOfMemory, RangeError, ThrowException}; +use core_runtime::{webidl_dictionary, webidl_interface, webidl_methods}; +use js::error::{ExnThrown, RangeError}; use js::gc::scope::Scope; -use js::{ArrayBuffer, ArrayBufferView, JSString}; +use js::JSString; -use crate::decoder_common::new_decoder; -use crate::decoder_common::TextDecoderOptions; +use crate::decoder_common::{ + decode_bytes, get_encoding, new_decoder, ArrayBufferViewOrArrayBuffer, TextDecoderOptions, +}; /// #[webidl_interface] @@ -38,17 +39,11 @@ impl TextDecoder<'_> { label: Option, options: Option, ) -> Result<(), RangeError> { - let label = label.as_deref().unwrap_or("utf-8"); let options = options.unwrap_or_default(); // Step 1: Let _encoding_ be the result of `getting an encoding` from _label_. // Step 2: If _encoding_ is failure or `replacement`, then `throw` a ``RangeError``. - let encoding = encoding_rs::Encoding::for_label_no_replacement(label.as_bytes()) - .ok_or_else(|| { - RangeError(format!( - "The encoding label provided ('{label}') is invalid." - )) - })?; + let encoding = get_encoding(label.as_deref())?; let mut data = self.data_mut(); // Step 3: Set `this`'s `encoding` to _encoding_. @@ -110,132 +105,17 @@ impl TextDecoder<'_> { // Step 3: If _input_ is given, then `push` a `copy of` _input_ to `this`'s `I/O queue`. // (We decode directly from _input_'s bytes. Nothing retains them after this call.) - let mut decode = |bytes: &[u8]| { - // windows-1252 decodes every byte outside 0x80..=0x9F to the code point of the same - // value, and its decoder keeps no state between calls, so such input is its own - // Latin-1 output. - if encoding == encoding_rs::WINDOWS_1252 - && !bytes.iter().any(|b| (0x80..=0x9F).contains(b)) - { - return Decoded::Latin1(js::typedarray::try_copy(bytes)); - } - // A fresh decoder that decodes its whole input at once turns ASCII into the same - // characters in every ASCII-compatible encoding. - if fresh - && last - && encoding.is_ascii_compatible() - && encoding_rs::Encoding::ascii_valid_up_to(bytes) == bytes.len() - { - return Decoded::Latin1(js::typedarray::try_copy(bytes)); - } - - // Step 4: Let _output_ be the `I/O queue` of scalar values « `end-of-queue` ». - // Step 5: While true: - // Step 5.1: Let _item_ be the result of `reading` from `this`'s `I/O queue`. - // Step 5.2: If _item_ is `end-of-queue` and `this`'s `do not flush` is true, then - // return the result of running `serialize I/O queue` with `this` and - // _output_. - // Step 5.3: Otherwise: - // Step 5.3.1: Let _result_ be the result of `processing an item` with _item_, - // `this`'s `decoder`, `this`'s `I/O queue`, _output_, and `this`'s `error - // mode`. - // Step 5.3.2: If _result_ is `finished`, then return the result of running - // `serialize I/O queue` with `this` and _output_. - // Step 5.3.3: Otherwise, if _result_ is `error`, `throw` a ``TypeError``. - // - // (One `encoding_rs` call processes every item. `last` is true when `end-of-queue` - // is processed rather than returned at step 5.2. The decoder's BOM removal performs - // `serialize I/O queue`'s BOM handling. We decode to UTF-16, which SpiderMonkey - // copies as is, or narrows to Latin-1 if every code unit fits.) - Decoded::Utf16(decode_to_utf16(decoder, bytes, last, fatal)) - }; + let mut decode = |bytes: &[u8]| decode_bytes(encoding, decoder, fatal, bytes, last, fresh); let decoded = match input { Some(ArrayBufferViewOrArrayBuffer::Buffer(input)) => input.with_bytes(decode), Some(ArrayBufferViewOrArrayBuffer::View(input)) => input.with_bytes(decode), None => decode(&[]), }; drop(data); - match decoded { - Decoded::Latin1(Some(chars)) => JSString::from_latin1(scope, &chars), - Decoded::Latin1(None) => Err(OutOfMemory.throw(scope)), - Decoded::Utf16(output) => { - let output = output.map_err(|error| error.throw(scope))?; - JSString::from_utf16(scope, &output) - } - } + decoded.into_js_string(scope) } } -/// The result of decoding the input of `TextDecoder.decode`. -enum Decoded { - /// A copy of the input, which is its own Latin-1 output. `None` means that allocating the - /// copy failed. - Latin1(Option>), - /// The UTF-16 output. - Utf16(Result, DecodeError>), -} - -/// Why [`decode_to_utf16`] failed. -enum DecodeError { - /// The input is malformed, in the "fatal" error mode. - Malformed, - /// The output can't be allocated. - OutOfMemory, -} - -impl DecodeError { - /// Throw the `TypeError` of malformed input, or report the failed allocation. - fn throw(self, scope: &Scope<'_>) -> ExnThrown { - match self { - DecodeError::Malformed => { - js::error::throw_type_error(scope, c"The encoded data was not valid.") - } - DecodeError::OutOfMemory => OutOfMemory.throw(scope), - } - } -} - -/// Decode `bytes` with `decoder` to UTF-16. `last` is true for the end of the input, which the -/// decoder then flushes. With `fatal`, malformed input fails, and otherwise it becomes U+FFFD. -fn decode_to_utf16( - decoder: &mut encoding_rs::Decoder, - bytes: &[u8], - last: bool, - fatal: bool, -) -> Result, DecodeError> { - let max_len = decoder - .max_utf16_buffer_length(bytes.len()) - .ok_or(DecodeError::OutOfMemory)?; - let mut output = Vec::new(); - output - .try_reserve_exact(max_len) - .map_err(|_| DecodeError::OutOfMemory)?; - output.resize(max_len, 0); - let written = if fatal { - let (result, _read, written) = - decoder.decode_to_utf16_without_replacement(bytes, &mut output, last); - match result { - encoding_rs::DecoderResult::InputEmpty => written, - encoding_rs::DecoderResult::Malformed(_, _) => return Err(DecodeError::Malformed), - encoding_rs::DecoderResult::OutputFull => { - unreachable!("output buffer was pre-allocated to max size"); - } - } - } else { - let (_result, _read, written, _had_errors) = - decoder.decode_to_utf16(bytes, &mut output, last); - written - }; - output.truncate(written); - Ok(output) -} - -#[webidl_union] -pub enum ArrayBufferViewOrArrayBuffer<'a> { - View(ArrayBufferView<'a>), - Buffer(ArrayBuffer<'a>), -} - #[webidl_dictionary] pub struct TextDecodeOptions { #[webidl(default = false)] diff --git a/crates/builtins/encoding/src/text_decoder_stream.rs b/crates/builtins/encoding/src/text_decoder_stream.rs index c199c9ca..6960e68b 100644 --- a/crates/builtins/encoding/src/text_decoder_stream.rs +++ b/crates/builtins/encoding/src/text_decoder_stream.rs @@ -8,7 +8,7 @@ use core_runtime::{webidl_interface, webidl_methods}; use js::conversion::FromJSVal; -use js::error::ExnThrown; +use js::error::{ExnThrown, ThrowException}; use js::function::cast_payload; use js::gc::handle::Heap; use js::gc::scope::Scope; @@ -20,8 +20,9 @@ use web_streams::readable::ReadableStream; use web_streams::writable::WritableStream; use web_streams::{AlgorithmArg, TransformStream, TransformStreamImpl}; -use crate::decoder_common::{new_decoder, TextDecoderOptions}; -use crate::text_decoder::{decode_bytes_to_js_string, ArrayBufferViewOrArrayBuffer}; +use crate::decoder_common::{ + decode_bytes, get_encoding, new_decoder, ArrayBufferViewOrArrayBuffer, TextDecoderOptions, +}; /// #[webidl_interface] @@ -72,11 +73,7 @@ impl TextDecoderStream { ) -> Result<(), ExnThrown> { // Step 1: Let encoding be the result of getting an encoding from label. // Step 2: If encoding is failure or replacement, then throw a RangeError. - let label = label.as_deref().unwrap_or("utf-8"); - let encoding = encoding_rs::Encoding::for_label_no_replacement(label.as_bytes()) - .ok_or_else(|| { - js::error::throw_range_error(scope, c"The encoding label provided is invalid.") - })?; + let encoding = get_encoding(label.as_deref()).map_err(|error| error.throw(scope))?; // Step 3: Let errorMode be "fatal" if options["fatal"] is true; otherwise "replacement". let options = options.unwrap_or_default(); @@ -172,22 +169,17 @@ fn decoder_transform_cb( // Step 4.3: Let result be the result of processing an item with item, decoder’s decoder, // decoder’s I/O queue, output, and decoder’s error mode. // Step 4.4: If result is error, then throw a TypeError. - // Implementation note: the Encoding Standard permits avoiding the copy when input cannot - // affect future decode calls. The slice is consumed synchronously here and - // `encoding_rs::Decoder` retains only its own state. `decode_bytes_to_js_string` performs - // steps 3-4 with `last = false`, preserving decoder state for the next chunk. - // SAFETY: `buffer_source` is rooted, and the helper consumes the slice or copies it into - // Rust-owned output before any JS allocation can GC and move the input buffer. - let bytes = match buffer_source { - ArrayBufferViewOrArrayBuffer::View(view) => unsafe { view.bytes() }, - ArrayBufferViewOrArrayBuffer::Buffer(buffer) => unsafe { buffer.bytes() }, - }; let mut data = stream.data_mut(); let encoding = data.encoding.expect("encoding must be initialized"); let fatal = data.fatal; let decoder = data.decoder.as_mut().expect("decoder must be initialized"); - decode_bytes_to_js_string(scope, encoding, decoder, fatal, bytes, false, false)? - }; + let decode = |bytes: &[u8]| decode_bytes(encoding, decoder, fatal, bytes, false, false); + match buffer_source { + ArrayBufferViewOrArrayBuffer::View(view) => view.with_bytes(decode), + ArrayBufferViewOrArrayBuffer::Buffer(buffer) => buffer.with_bytes(decode), + } + } + .into_js_string(scope)?; if !output.is_empty() { enqueue_to_transform(scope, &stream, output)?; @@ -224,8 +216,9 @@ fn decoder_flush_cb( let encoding = data.encoding.expect("encoding must be initialized"); let fatal = data.fatal; let decoder = data.decoder.as_mut().expect("decoder must be initialized"); - decode_bytes_to_js_string(scope, encoding, decoder, fatal, &[], true, false)? - }; + decode_bytes(encoding, decoder, fatal, &[], true, false) + } + .into_js_string(scope)?; if !output.is_empty() { enqueue_to_transform(scope, &stream, output)?; diff --git a/crates/builtins/encoding/src/text_encoder_stream.rs b/crates/builtins/encoding/src/text_encoder_stream.rs index 4139347e..af8d9e71 100644 --- a/crates/builtins/encoding/src/text_encoder_stream.rs +++ b/crates/builtins/encoding/src/text_encoder_stream.rs @@ -3,7 +3,7 @@ //! use core_runtime::{webidl_interface, webidl_methods}; -use js::error::ExnThrown; +use js::error::{ExnThrown, OutOfMemory, ThrowException}; use js::function::cast_payload; use js::gc::handle::Heap; use js::gc::scope::Scope; @@ -134,6 +134,9 @@ fn encoder_transform_cb( if output.is_empty() { output = utf8; } else { + output + .try_reserve(utf8.len()) + .map_err(|_| OutOfMemory.throw(scope))?; output.extend_from_slice(&utf8); } } diff --git a/crates/builtins/encoding/tests/encoding.rs b/crates/builtins/encoding/tests/encoding.rs index 39de5833..0bac3bd8 100644 --- a/crates/builtins/encoding/tests/encoding.rs +++ b/crates/builtins/encoding/tests/encoding.rs @@ -1,397 +1,79 @@ // SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception -//! Integration tests for the Encoding API (TextEncoder, TextDecoder, -//! TextEncoderStream, TextDecoderStream). +//! Implementation-specific regressions not covered by the encoding WPT suite. + +#![cfg(not(target_arch = "wasm32"))] use core_runtime::config::RuntimeConfig; use core_runtime::event_loop::run_microtasks; use core_runtime::runtime::{clear_global_initializers, register_global_initializer, Runtime}; -use core_runtime::test_util::{eval_with_setup, throws_with_setup}; +use core_runtime::test_util::eval_with_setup; use js::conversion::FromJSVal; fn setup() { clear_global_initializers(); - register_global_initializer(|scope, global| { - web_globals::add_to_global(scope, global); - }); - register_global_initializer(|scope, global| { - encoding::add_to_global(scope, global); - }); - register_global_initializer(|scope, global| { - web_streams::add_to_global(scope, global); - }); + register_global_initializer(web_globals::add_to_global); + register_global_initializer(web_encoding::add_to_global); + register_global_initializer(web_streams::add_to_global); } fn eval(code: &str) -> String { eval_with_setup(setup, code) } -fn throws(code: &str) -> bool { - throws_with_setup(setup, code) -} - -/// Runs async JS code and returns the value of `globalThis.__out` after -/// microtasks have settled. fn run_async(code: &str) -> String { - clear_global_initializers(); - register_global_initializer(|scope, global| web_globals::add_to_global(scope, global)); - register_global_initializer(|scope, global| encoding::add_to_global(scope, global)); - register_global_initializer(|scope, global| web_streams::add_to_global(scope, global)); - let rt = Runtime::init(&RuntimeConfig::default()).expect("initialize test runtime"); - let scope = rt.default_global(); - js::compile::evaluate_with_filename(&scope, code, "test.js", 1).ok(); + setup(); + let runtime = Runtime::init(&RuntimeConfig::default()).expect("initialize test runtime"); + let scope = runtime.default_global(); + js::compile::evaluate_with_filename(&scope, code, "test.js", 1).unwrap(); run_microtasks(&scope); - let out = js::compile::evaluate_with_filename(&scope, "globalThis.__out", "out.js", 1).unwrap(); - String::from_jsval(&scope, out, ()).unwrap() -} - -/// Drains a ReadableStringStream into an array of string chunks. -const DRAIN_STR: &str = r#" - async function drainStr(stream) { - const r = stream.getReader(); - const acc = []; - for (;;) { - const {value, done} = await r.read(); - if (done) break; - acc.push(value); - } - return acc; - } -"#; - -/// Drains a ReadableStream into a flat array of byte numbers. -const DRAIN_BYTES: &str = r#" - async function drainBytes(stream) { - const r = stream.getReader(); - const acc = []; - for (;;) { - const {value, done} = await r.read(); - if (done) break; - for (const b of value) acc.push(b); - } - return acc; - } -"#; - -// ── TextEncoder existence ── - -#[test] -fn text_encoder_exists() { - assert_eq!(eval("typeof TextEncoder"), "function"); -} - -#[test] -fn text_encoder_construct() { - assert_eq!(eval("new TextEncoder() instanceof TextEncoder"), "true"); -} - -// ── TextEncoder.encoding ── - -#[test] -fn text_encoder_encoding() { - assert_eq!(eval("new TextEncoder().encoding"), "utf-8"); -} - -// ── TextEncoder.encode() ── - -#[test] -fn text_encoder_encode_empty() { - assert_eq!(eval("new TextEncoder().encode('').length"), "0"); -} - -#[test] -fn text_encoder_encode_ascii() { - assert_eq!( - eval("Array.from(new TextEncoder().encode('abc')).join(',')"), - "97,98,99" - ); -} - -#[test] -fn text_encoder_encode_returns_uint8array() { - assert_eq!( - eval("new TextEncoder().encode('a') instanceof Uint8Array"), - "true" - ); -} - -#[test] -fn text_encoder_encode_no_args() { - // encode() with no arguments encodes empty string - assert_eq!(eval("new TextEncoder().encode().length"), "0"); + let output = + js::compile::evaluate_with_filename(&scope, "globalThis.__out", "out.js", 1).unwrap(); + String::from_jsval(&scope, output, ()).unwrap() } #[test] -fn text_encoder_encode_multibyte() { - // "€" is U+20AC, encoded as [0xE2, 0x82, 0xAC] in UTF-8 - assert_eq!( - eval(r#"Array.from(new TextEncoder().encode('€')).join(',')"#), - "226,130,172" - ); -} - -#[test] -fn text_encoder_encode_emoji() { - // "😀" is U+1F600, encoded as [0xF0, 0x9F, 0x98, 0x80] - assert_eq!( - eval(r#"Array.from(new TextEncoder().encode('😀')).join(',')"#), - "240,159,152,128" - ); -} - -// ── TextEncoder.encodeInto() ── - -#[test] -fn text_encoder_encode_into_basic() { +fn text_decoder_windows_1252_maps_every_byte() { assert_eq!( eval( - r#" - const enc = new TextEncoder(); - const buf = new Uint8Array(10); - const result = enc.encodeInto('abc', buf); - result.read + ',' + result.written - "# + "const decoder = new TextDecoder('latin1'); + const codes = text => Array.from(text, char => char.charCodeAt(0)).join(','); + [decoder.decode(new Uint8Array([0x41, 0xe9, 0xff, 0xa0])), + decoder.decode(new Uint8Array([0x41, 0x80, 0x9f, 0xe9])), + decoder.decode(new Uint8Array([0xe9]), { stream: true }) + + decoder.decode(new Uint8Array([0x80]), { stream: true }) + decoder.decode(), + decoder.decode(new Uint8Array([]))].map(codes).join(' | ')" ), - "3,3" + "65,233,255,160 | 65,8364,376,233 | 233,8364 | " ); } #[test] -fn text_encoder_encode_into_writes_bytes() { +fn text_decoder_label_errors_match() { assert_eq!( eval( r#" - const enc = new TextEncoder(); - const buf = new Uint8Array(10); - enc.encodeInto('abc', buf); - buf[0] + ',' + buf[1] + ',' + buf[2] - "# - ), - "97,98,99" - ); -} - -#[test] -fn text_encoder_encode_into_truncation() { - // Buffer too small to fit all characters - assert_eq!( - eval( - r#" - const enc = new TextEncoder(); - const buf = new Uint8Array(2); - const result = enc.encodeInto('abcdef', buf); - result.read + ',' + result.written - "# - ), - "2,2" - ); -} - -#[test] -fn text_encoder_encode_into_multibyte_truncation() { - // "€" needs 3 bytes; buffer of 2 can't fit it - assert_eq!( - eval( - r#" - const enc = new TextEncoder(); - const buf = new Uint8Array(2); - const result = enc.encodeInto('€', buf); - result.read + ',' + result.written - "# - ), - "0,0" - ); -} - -#[test] -fn text_encoder_encode_into_surrogate_pair_count() { - // "😀" is U+1F600, takes 4 UTF-8 bytes and 2 UTF-16 code units - assert_eq!( - eval( - r#" - const enc = new TextEncoder(); - const buf = new Uint8Array(10); - const result = enc.encodeInto('😀', buf); - result.read + ',' + result.written - "# - ), - "2,4" - ); -} - -// ── TextDecoder existence ── - -#[test] -fn text_decoder_exists() { - assert_eq!(eval("typeof TextDecoder"), "function"); -} - -#[test] -fn text_decoder_construct() { - assert_eq!(eval("new TextDecoder() instanceof TextDecoder"), "true"); -} - -// ── TextDecoder.encoding ── - -#[test] -fn text_decoder_encoding_default() { - assert_eq!(eval("new TextDecoder().encoding"), "utf-8"); -} - -#[test] -fn text_decoder_encoding_explicit_utf8() { - assert_eq!(eval("new TextDecoder('utf-8').encoding"), "utf-8"); -} - -#[test] -fn text_decoder_encoding_label_normalization() { - // "UTF-8" should be normalized to "utf-8" - assert_eq!(eval("new TextDecoder('UTF-8').encoding"), "utf-8"); -} - -#[test] -fn text_decoder_encoding_windows_1252() { - assert_eq!( - eval("new TextDecoder('windows-1252').encoding"), - "windows-1252" - ); -} - -// ── TextDecoder.fatal ── - -#[test] -fn text_decoder_fatal_default() { - assert_eq!(eval("new TextDecoder().fatal"), "false"); -} - -#[test] -fn text_decoder_fatal_true() { - assert_eq!( - eval("new TextDecoder('utf-8', { fatal: true }).fatal"), - "true" - ); -} - -// ── TextDecoder.ignoreBOM ── - -#[test] -fn text_decoder_ignore_bom_default() { - assert_eq!(eval("new TextDecoder().ignoreBOM"), "false"); -} - -#[test] -fn text_decoder_ignore_bom_true() { - assert_eq!( - eval("new TextDecoder('utf-8', { ignoreBOM: true }).ignoreBOM"), - "true" - ); -} - -// ── TextDecoder.decode() ── - -#[test] -fn text_decoder_decode_utf8() { - assert_eq!( - eval(r#"new TextDecoder().decode(new Uint8Array([97, 98, 99]))"#), - "abc" - ); -} - -#[test] -fn text_decoder_decode_no_args() { - // decode() with no arguments returns empty string - assert_eq!(eval("new TextDecoder().decode()"), ""); -} - -#[test] -fn text_decoder_decode_empty_array() { - assert_eq!(eval("new TextDecoder().decode(new Uint8Array([]))"), ""); -} - -#[test] -fn text_decoder_decode_multibyte_utf8() { - // "€" = [0xE2, 0x82, 0xAC] - assert_eq!( - eval("new TextDecoder().decode(new Uint8Array([0xE2, 0x82, 0xAC]))"), - "€" - ); -} - -#[test] -fn text_decoder_decode_emoji() { - // "😀" = [0xF0, 0x9F, 0x98, 0x80] - assert_eq!( - eval("new TextDecoder().decode(new Uint8Array([0xF0, 0x9F, 0x98, 0x80]))"), - "😀" - ); -} - -// ── TextDecoder error handling ── - -#[test] -fn text_decoder_invalid_label_throws() { - assert!(throws("new TextDecoder('invalid-encoding')")); -} - -#[test] -fn text_decoder_replacement_encoding_throws() { - // "replacement" is not a valid label for TextDecoder (excluded by spec) - assert!(throws("new TextDecoder('replacement')")); -} - -#[test] -fn text_decoder_fatal_invalid_bytes() { - // Invalid UTF-8 sequence with fatal: true should throw - assert!(throws( - "new TextDecoder('utf-8', { fatal: true }).decode(new Uint8Array([0xFF, 0xFE]))" - )); -} - -#[test] -fn text_decoder_replacement_mode_invalid_bytes() { - // Invalid UTF-8 in replacement mode should produce U+FFFD - assert_eq!( - eval( - r#" - const decoder = new TextDecoder('utf-8'); - const result = decoder.decode(new Uint8Array([0xFF])); - result === '\uFFFD' - "# + for (const label of ["invalid-encoding", "replacement", ""]) { + const errors = [TextDecoder, TextDecoderStream].map(Constructor => { + try { + new Constructor(label); + throw new Error("Expected invalid encoding label to throw"); + } catch (error) { + if (!(error instanceof RangeError)) throw error; + return error.message; + } + }); + if (errors[0] !== errors[1] || !errors[0].includes(label)) { + throw new Error("Decoder constructor errors differ"); + } + } + true + "# ), "true" ); } -// ── TextDecoder streaming ── - -#[test] -fn text_decoder_streaming() { - // Multi-byte character split across two decode() calls - // "€" = [0xE2, 0x82, 0xAC] - assert_eq!( - eval( - r#" - const decoder = new TextDecoder(); - let result = decoder.decode(new Uint8Array([0xE2, 0x82]), { stream: true }); - result += decoder.decode(new Uint8Array([0xAC])); - result - "# - ), - "€" - ); -} - -// ── TextDecoder with non-UTF-8 encodings ── - -#[test] -fn text_decoder_windows_1252() { - // 0xE9 in windows-1252 is "é" - assert_eq!( - eval(r#"new TextDecoder('windows-1252').decode(new Uint8Array([0xE9]))"#), - "é" - ); -} - #[test] fn text_decoder_latin1_fast_paths_gc_safe() { assert_eq!( @@ -402,8 +84,9 @@ fn text_decoder_latin1_fast_paths_gc_safe() { const decoder = new TextDecoder(encoding, { fatal }); for (const length of [0, 1, 24, 25, 64, 256]) { const bytes = new Uint8Array(length); - for (let i = 0; i < length; i++) { - bytes[i] = encoding === "utf-8" ? 65 + i % 26 : 0xA0 + i % 96; + for (let index = 0; index < length; index++) { + bytes[index] = encoding === "utf-8" + ? 65 + index % 26 : 0xA0 + index % 96; } const padded = new Uint8Array(length + 4); padded.set(bytes, 2); @@ -422,334 +105,20 @@ fn text_decoder_latin1_fast_paths_gc_safe() { } } true - "#, + "#, ), "true" ); } -// ── WebIDL interface compliance ── - -#[test] -fn text_encoder_to_string_tag() { - assert_eq!( - eval("Object.prototype.toString.call(new TextEncoder())"), - "[object TextEncoder]" - ); -} - -#[test] -fn text_decoder_to_string_tag() { - assert_eq!( - eval("Object.prototype.toString.call(new TextDecoder())"), - "[object TextDecoder]" - ); -} - -// ── TextEncoderStream ── - -#[test] -fn text_encoder_stream_exists() { - assert_eq!(eval("typeof TextEncoderStream"), "function"); -} - -#[test] -fn text_encoder_stream_encoding() { - assert_eq!(eval("new TextEncoderStream().encoding"), "utf-8"); -} - -#[test] -fn text_encoder_stream_readable_writable() { - assert_eq!( - eval( - r#" - const s = new TextEncoderStream(); - (s.readable instanceof ReadableStream) + ',' + (s.writable instanceof WritableStream) - "# - ), - "true,true" - ); -} - -#[test] -fn text_encoder_stream_ascii() { - let out = run_async(&format!( - r#"{DRAIN_BYTES} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue("abc"); c.close(); }} }}); - const os = rs.pipeThrough(new TextEncoderStream()); - drainBytes(os).then(a => {{ globalThis.__out = a.join(","); }}); - "# - )); - assert_eq!(out, "97,98,99"); -} - -#[test] -fn text_encoder_stream_multibyte() { - // "€" = [0xE2, 0x82, 0xAC] - let out = run_async(&format!( - r#"{DRAIN_BYTES} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue("€"); c.close(); }} }}); - const os = rs.pipeThrough(new TextEncoderStream()); - drainBytes(os).then(a => {{ globalThis.__out = a.join(","); }}); - "# - )); - assert_eq!(out, "226,130,172"); -} - -#[test] -fn text_encoder_stream_unpaired_surrogate() { - // Lone leading surrogate produces replacement character [0xEF, 0xBF, 0xBD] - let out = run_async(&format!( - r#"{DRAIN_BYTES} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue("\uD83D"); c.close(); }} }}); - const os = rs.pipeThrough(new TextEncoderStream()); - drainBytes(os).then(a => {{ globalThis.__out = a.join(","); }}); - "# - )); - assert_eq!(out, "239,191,189"); -} - -#[test] -fn text_encoder_stream_surrogate_pair_split() { - // Surrogates in separate chunks should still produce the correct encoding - let out = run_async(&format!( - r#"{DRAIN_BYTES} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue("\uD83D"); c.enqueue("\uDE00"); c.close(); }} }}); - const os = rs.pipeThrough(new TextEncoderStream()); - drainBytes(os).then(a => {{ globalThis.__out = a.join(","); }}); - "# - )); - assert_eq!(out, "240,159,152,128"); -} - -// ── TextDecoderStream ── - -#[test] -fn text_decoder_stream_exists() { - assert_eq!(eval("typeof TextDecoderStream"), "function"); -} - -#[test] -fn text_decoder_stream_encoding() { - assert_eq!(eval("new TextDecoderStream().encoding"), "utf-8"); -} - -#[test] -fn text_decoder_stream_fatal() { - assert_eq!(eval("new TextDecoderStream().fatal"), "false"); -} - -#[test] -fn text_decoder_stream_ignore_bom() { - assert_eq!(eval("new TextDecoderStream().ignoreBOM"), "false"); -} - -#[test] -fn text_decoder_stream_readable_writable() { - assert_eq!( - eval( - r#" - const s = new TextDecoderStream(); - (s.readable instanceof ReadableStream) + ',' + (s.writable instanceof WritableStream) - "# - ), - "true,true" - ); -} - -#[test] -fn text_decoder_stream_invalid_label_throws() { - assert!(throws("new TextDecoderStream('invalid-encoding')")); -} - -#[test] -fn text_decoder_stream_ascii() { - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([97, 98, 99])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream()); - drainStr(os).then(a => {{ globalThis.__out = a.join("|"); }}); - "# - )); - assert_eq!(out, "abc"); -} - -#[test] -fn text_decoder_stream_multibyte() { - // "€" = [0xE2, 0x82, 0xAC] - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xE2, 0x82, 0xAC])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream()); - drainStr(os).then(a => {{ globalThis.__out = a.join("|"); }}); - "# - )); - assert_eq!(out, "€"); -} - -#[test] -fn text_decoder_stream_split_multibyte() { - // "€" = [0xE2, 0x82, 0xAC], split across two chunks - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xE2, 0x82])); c.enqueue(new Uint8Array([0xAC])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream()); - drainStr(os).then(a => {{ globalThis.__out = a.join("|"); }}); - "# - )); - assert_eq!(out, "€"); -} - -#[test] -fn text_decoder_stream_split_4byte() { - // "😀" = [0xF0, 0x9F, 0x98, 0x80], split after first byte - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xF0])); c.enqueue(new Uint8Array([0x9F, 0x98, 0x80])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream()); - drainStr(os).then(a => {{ globalThis.__out = a.join("|"); }}); - "# - )); - assert_eq!(out, "😀"); -} - -#[test] -fn text_decoder_stream_incomplete_at_eof() { - // Incomplete 4-byte sequence at EOF produces replacement character(s) - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xF0, 0x9F, 0x98])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream()); - drainStr(os).then(a => {{ globalThis.__out = a.join("").length; }}); - "# - )); - assert_eq!(out, "1"); -} - -#[test] -fn text_decoder_stream_fatal_invalid_bytes() { - // fatal: true should reject the readable stream on invalid bytes - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xFF, 0xFE])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream("utf-8", {{ fatal: true }})); - drainStr(os).then( - () => {{ globalThis.__out = "fulfilled"; }}, - e => {{ globalThis.__out = "rejected:" + e.name; }} - ); - "# - )); - assert_eq!(out, "rejected:TypeError"); -} - -// -- BOM handling -- - -#[test] -fn text_decoder_stream_bom_default() { - // ignoreBOM: false (default) → BOM is removed - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xEF, 0xBB, 0xBF, 97])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream()); - drainStr(os).then(a => {{ globalThis.__out = a.join(""); }}); - "# - )); - assert_eq!(out, "a"); -} - -#[test] -fn text_decoder_stream_bom_ignored() { - // ignoreBOM: true → BOM is kept as U+FEFF - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xEF, 0xBB, 0xBF, 97])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream("utf-8", {{ ignoreBOM: true }})); - drainStr(os).then(a => {{ globalThis.__out = a[0].charCodeAt(0) + "," + a[0][1]; }}); - "# - )); - assert_eq!(out, "65279,a"); -} - -#[test] -fn text_decoder_stream_bom_split_across_chunks() { - // BOM bytes split across two chunks, ignoreBOM: true → BOM kept as U+FEFF - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ - c.enqueue(new Uint8Array([0xEF, 0xBB])); - c.enqueue(new Uint8Array([0xBF, 97])); - c.close(); - }} }}); - const os = rs.pipeThrough(new TextDecoderStream("utf-8", {{ ignoreBOM: true }})); - drainStr(os).then(a => {{ globalThis.__out = a[0].charCodeAt(0) + "," + a[0][1]; }}); - "# - )); - assert_eq!(out, "65279,a"); -} - -#[test] -fn text_decoder_stream_bom_only() { - // BOM only, ignoreBOM: true → emits U+FEFF - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xEF, 0xBB, 0xBF])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream("utf-8", {{ ignoreBOM: true }})); - drainStr(os).then(a => {{ globalThis.__out = a[0].charCodeAt(0); }}); - "# - )); - assert_eq!(out, "65279"); -} - -#[test] -fn text_decoder_stream_bom_only_default() { - // BOM only, ignoreBOM: false (default) → no output - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xEF, 0xBB, 0xBF])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream()); - drainStr(os).then(a => {{ globalThis.__out = a.length; }}); - "# - )); - assert_eq!(out, "0"); -} - -#[test] -fn text_decoder_stream_windows_1252() { - // 0xE9 in windows-1252 is "é" - let out = run_async(&format!( - r#"{DRAIN_STR} - globalThis.__out = "pending"; - const rs = new ReadableStream({{ start(c) {{ c.enqueue(new Uint8Array([0xE9])); c.close(); }} }}); - const os = rs.pipeThrough(new TextDecoderStream("windows-1252")); - drainStr(os).then(a => {{ globalThis.__out = a.join("|"); }}); - "# - )); - assert_eq!(out, "é"); -} - #[test] fn text_decoder_stream_windows_1252_gc_safe() { - let out = run_async(&format!( - r#"{DRAIN_STR} + let output = run_async( + r#" globalThis.__out = "pending"; const inputs = []; const expected = []; - for (const length of [1, 24, 25, 64]) {{ + for (const length of [1, 24, 25, 64]) { const bytes = new Uint8Array(length).fill(0xE9); const padded = new Uint8Array(length + 4); padded.set(bytes, 2); @@ -760,17 +129,25 @@ fn text_decoder_stream_windows_1252_gc_safe() { new DataView(padded.buffer, 2, length), ); expected.push("\u00E9".repeat(length * 4)); - }} - const rs = new ReadableStream({{ - start(c) {{ - for (const input of inputs) c.enqueue(input); - c.close(); - }} - }}); - drainStr(rs.pipeThrough(new TextDecoderStream("windows-1252"))) - .then(a => {{ globalThis.__out = String(a.join("") === expected.join("")); }}) - .catch(e => {{ globalThis.__out = String(e); }}); - "# - )); - assert_eq!(out, "true"); + } + const source = new ReadableStream({ + start(controller) { + for (const input of inputs) controller.enqueue(input); + controller.close(); + } + }); + (async () => { + const reader = source.pipeThrough(new TextDecoderStream("windows-1252")) + .getReader(); + const chunks = []; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + chunks.push(value); + } + globalThis.__out = String(chunks.join("") === expected.join("")); + })().catch(error => { globalThis.__out = String(error); }); + "#, + ); + assert_eq!(output, "true"); } diff --git a/crates/libstarling/Cargo.toml b/crates/libstarling/Cargo.toml index 6c925adc..5d45e20b 100644 --- a/crates/libstarling/Cargo.toml +++ b/crates/libstarling/Cargo.toml @@ -21,7 +21,7 @@ event-listener.workspace = true futures-lite.workspace = true futures-util.workspace = true http.workspace = true -encoding.workspace = true +web-encoding.workspace = true js.workspace = true platform.workspace = true fetch-event.workspace = true diff --git a/crates/libstarling/src/lib.rs b/crates/libstarling/src/lib.rs index 33d71edd..7665d058 100644 --- a/crates/libstarling/src/lib.rs +++ b/crates/libstarling/src/lib.rs @@ -25,7 +25,7 @@ pub mod serve_wasm; pub fn register_builtins() { runtime::register_global_initializer(web_globals::add_to_global); runtime::register_global_initializer(web_streams::add_to_global); - runtime::register_global_initializer(encoding::add_to_global); + runtime::register_global_initializer(web_encoding::add_to_global); runtime::register_global_initializer(web_url::add_to_global); runtime::register_global_initializer(web_fetch::add_to_global); runtime::register_global_initializer(fetch_event::add_to_global); diff --git a/scripts/test-gc-zeal.sh b/scripts/test-gc-zeal.sh index 2587097f..bb54329b 100755 --- a/scripts/test-gc-zeal.sh +++ b/scripts/test-gc-zeal.sh @@ -89,7 +89,7 @@ done if [[ "$PACKAGE_SELECTED" == false ]]; then CARGO_ARGS=( -p js -p core-runtime -p wit-bridge - -p cpp-builtins -p fetch-event -p web-fetch -p web-globals -p web-streams -p web-url + -p cpp-builtins -p fetch-event -p web-encoding -p web-fetch -p web-globals -p web-streams -p web-url ${CARGO_ARGS[@]+"${CARGO_ARGS[@]}"} ) fi From a1174e09605560ecc53818efaa2a4dc7a137c29b Mon Sep 17 00:00:00 2001 From: Karthik Ganeshram Date: Tue, 3 Nov 2026 14:44:30 +0100 Subject: [PATCH 5/5] address more review comments Signed-off-by: Karthik Ganeshram --- Cargo.toml | 2 +- .../{encoding => web-encoding}/Cargo.toml | 0 .../src/decoder_common.rs | 0 .../{encoding => web-encoding}/src/lib.rs | 0 .../src/text_decoder.rs | 0 .../src/text_decoder_stream.rs | 20 +-- .../src/text_encoder.rs | 0 .../src/text_encoder_stream.rs | 121 ++++++++++-------- .../tests/encoding.rs | 21 +-- .../src/transform/transform_stream.rs | 2 +- 10 files changed, 89 insertions(+), 77 deletions(-) rename crates/builtins/{encoding => web-encoding}/Cargo.toml (100%) rename crates/builtins/{encoding => web-encoding}/src/decoder_common.rs (100%) rename crates/builtins/{encoding => web-encoding}/src/lib.rs (100%) rename crates/builtins/{encoding => web-encoding}/src/text_decoder.rs (100%) rename crates/builtins/{encoding => web-encoding}/src/text_decoder_stream.rs (94%) rename crates/builtins/{encoding => web-encoding}/src/text_encoder.rs (100%) rename crates/builtins/{encoding => web-encoding}/src/text_encoder_stream.rs (63%) rename crates/builtins/{encoding => web-encoding}/tests/encoding.rs (87%) diff --git a/Cargo.toml b/Cargo.toml index 5585210e..4180ccb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,7 +67,7 @@ web-globals = { path = "crates/builtins/web-globals" } web-streams = { path = "crates/builtins/web-streams" } web-url = { path = "crates/builtins/web-url" } cpp-builtins = { path = "crates/builtins/cpp-builtins" } -web-encoding = { path = "crates/builtins/encoding" } +web-encoding = { path = "crates/builtins/web-encoding" } [profile.release] lto = "thin" diff --git a/crates/builtins/encoding/Cargo.toml b/crates/builtins/web-encoding/Cargo.toml similarity index 100% rename from crates/builtins/encoding/Cargo.toml rename to crates/builtins/web-encoding/Cargo.toml diff --git a/crates/builtins/encoding/src/decoder_common.rs b/crates/builtins/web-encoding/src/decoder_common.rs similarity index 100% rename from crates/builtins/encoding/src/decoder_common.rs rename to crates/builtins/web-encoding/src/decoder_common.rs diff --git a/crates/builtins/encoding/src/lib.rs b/crates/builtins/web-encoding/src/lib.rs similarity index 100% rename from crates/builtins/encoding/src/lib.rs rename to crates/builtins/web-encoding/src/lib.rs diff --git a/crates/builtins/encoding/src/text_decoder.rs b/crates/builtins/web-encoding/src/text_decoder.rs similarity index 100% rename from crates/builtins/encoding/src/text_decoder.rs rename to crates/builtins/web-encoding/src/text_decoder.rs diff --git a/crates/builtins/encoding/src/text_decoder_stream.rs b/crates/builtins/web-encoding/src/text_decoder_stream.rs similarity index 94% rename from crates/builtins/encoding/src/text_decoder_stream.rs rename to crates/builtins/web-encoding/src/text_decoder_stream.rs index 6960e68b..e6b15490 100644 --- a/crates/builtins/encoding/src/text_decoder_stream.rs +++ b/crates/builtins/web-encoding/src/text_decoder_stream.rs @@ -83,14 +83,18 @@ impl TextDecoderStream { // Step 4: Set up a text decoder stream with this, encoding, errorMode, and // options["ignoreBOM"]. // Step 4.1: Assert: encoding is not replacement. - // Step 4.2: Set stream's encoding to encoding. - self.data_mut().encoding = Some(encoding); - // Step 4.3: Set stream's error mode to errorMode. - self.data_mut().fatal = fatal; - // Step 4.4: Set stream's ignore BOM to ignoreBOM. - self.data_mut().ignore_bom = ignore_bom; - // Step 4.5: Set stream’s decoder to a new instance of encoding’s decoder. - self.data_mut().decoder = Some(new_decoder(encoding, ignore_bom)); + // Release the mutable borrow before JS allocations below can trigger GC and trace self. + { + let mut data = self.data_mut(); + // Step 4.2: Set stream's encoding to encoding. + data.encoding = Some(encoding); + // Step 4.3: Set stream's error mode to errorMode. + data.fatal = fatal; + // Step 4.4: Set stream's ignore BOM to ignoreBOM. + data.ignore_bom = ignore_bom; + // Step 4.5: Set stream’s decoder to a new instance of encoding’s decoder. + data.decoder = Some(new_decoder(encoding, ignore_bom)); + } // Step 4.6: Set stream's I/O queue to a new I/O queue. // Implementation note: this is not stored separately because `encoding_rs::Decoder` // buffers incomplete sequences internally; see the comment on the struct definition. diff --git a/crates/builtins/encoding/src/text_encoder.rs b/crates/builtins/web-encoding/src/text_encoder.rs similarity index 100% rename from crates/builtins/encoding/src/text_encoder.rs rename to crates/builtins/web-encoding/src/text_encoder.rs diff --git a/crates/builtins/encoding/src/text_encoder_stream.rs b/crates/builtins/web-encoding/src/text_encoder_stream.rs similarity index 63% rename from crates/builtins/encoding/src/text_encoder_stream.rs rename to crates/builtins/web-encoding/src/text_encoder_stream.rs index af8d9e71..6a90141c 100644 --- a/crates/builtins/encoding/src/text_encoder_stream.rs +++ b/crates/builtins/web-encoding/src/text_encoder_stream.rs @@ -9,6 +9,7 @@ use js::gc::handle::Heap; use js::gc::scope::Scope; use js::native::Value; use js::prelude::{CallbackArgs, HandleValue}; +use js::string::StrChars; use js::{value, JSString, Uint8Array}; use web_streams::readable::ReadableStream; @@ -78,9 +79,8 @@ fn encoder_transform_cb( payload: HandleValue<'_>, ) -> Result { let stream = cast_payload::(scope, payload); - // Step 1: Let input be the result of converting chunk to a DOMString. - let js_string = JSString::from_value(scope, args.get(0))?; + let input = JSString::from_value(scope, args.get(0))?; // Step 2: Convert input to an I/O queue of code units. // Step 3: Let output be the I/O queue of bytes « end-of-queue ». // Step 4: While true: @@ -96,57 +96,84 @@ fn encoder_transform_cb( // algorithm with encoder, item, and input. // Step 4.4: If result is not continue, then process an item with result, encoder’s encoder, // input, output, and "fatal". - // Implementation note: a surrogate pair can only be split at the chunk's edges, so only the - // first and last code units need the [[pending high surrogate]] handling. Everything between - // is encoded in bulk by `to_utf8`, which replaces lone surrogates with U+FFFD exactly as - // steps 4.3 and 4.4 do. - let mut start = 0; - let mut end = js_string.len(); - if end == 0 { + // + // (`encoding_rs` runs this loop over the whole chunk, and replaces unpaired surrogates with + // U+FFFD. It doesn't keep a high surrogate between calls, so we encode the pending one + // together with the chunk's first code unit, and hold back a high surrogate that ends the + // chunk.) + if input.is_empty() { return Ok(value::undefined()); } - let mut output = Vec::new(); let pending = stream.data_mut().leading_surrogate.take(); - if let Some(leading) = pending { - let first = js_string.char_at(scope, 0)?; - if is_trailing_surrogate(first) { - let c = 0x10000 + ((u32::from(leading) - 0xD800) << 10) + (u32::from(first) - 0xDC00); - push_scalar_value(&mut output, c); - start = 1; - } else { - push_scalar_value(&mut output, 0xFFFD); - } + // SAFETY: the closure doesn't call into SpiderMonkey. + let (output, pending) = unsafe { + input.with_chars(scope, |chars| match chars { + StrChars::Latin1(chars) => (encode_latin1(pending, chars), None), + StrChars::TwoByte(chars) => encode_utf16(pending, chars), + }) + }?; + stream.data_mut().leading_surrogate = pending; + let output = output.ok_or_else(|| OutOfMemory.throw(scope))?; + if !output.is_empty() { + let chunk = Uint8Array::with_data(scope, &output)?; + enqueue_to_transform(scope, &stream, chunk)?; } - if start < end { - let last = js_string.char_at(scope, end - 1)?; - if is_leading_surrogate(last) { - stream.data_mut().leading_surrogate = Some(last); - end -= 1; + Ok(value::undefined()) +} +/// Returns the UTF-8 encoding of `chars`, preceded by U+FFFD if `pending` is set, or `None` if +/// allocating it fails. Latin-1 characters aren't surrogates, so `pending` is always unpaired. +fn encode_latin1(pending: Option, chars: &[u8]) -> Option> { + let head = pending.as_slice(); + let mut output = zeroed_buffer(head.len() * 3 + chars.len() * 2)?; + let mut written = encoding_rs::mem::convert_utf16_to_utf8(head, &mut output); + written += encoding_rs::mem::convert_latin1_to_utf8(chars, &mut output[written..]); + output.truncate(written); + Some(output) +} +/// Returns the UTF-8 encoding of `chars` preceded by the high surrogate `pending`, or `None` if +/// allocating it fails. A high surrogate at the end of `chars` isn't encoded. It is returned as +/// the new pending high surrogate. +fn encode_utf16(pending: Option, chars: &[u16]) -> (Option>, Option) { + let mut head = [0; 2]; + let mut head_len = 0; + let mut rest = chars; + if let Some(high) = pending { + head[0] = high; + head_len = 1; + if let Some((&first, tail)) = rest.split_first() { + if is_trailing_surrogate(first) { + head[1] = first; + head_len = 2; + rest = tail; + } } } - if start < end { - let middle = if start == 0 && end == js_string.len() { - js_string - } else { - js_string.substring(scope, start, end - start)? - }; - let utf8 = middle.to_utf8(scope)?.into_bytes(); - if output.is_empty() { - output = utf8; - } else { - output - .try_reserve(utf8.len()) - .map_err(|_| OutOfMemory.throw(scope))?; - output.extend_from_slice(&utf8); + let mut new_pending = None; + if let Some((&last, init)) = rest.split_last() { + if is_leading_surrogate(last) { + new_pending = Some(last); + rest = init; } } - - if !output.is_empty() { - let u8array = Uint8Array::with_data(scope, &output)?; - enqueue_to_transform(scope, &stream, u8array)?; + let Some(mut output) = zeroed_buffer((head_len + rest.len()) * 3) else { + return (None, new_pending); + }; + let mut written = encoding_rs::mem::convert_utf16_to_utf8(&head[..head_len], &mut output); + written += encoding_rs::mem::convert_utf16_to_utf8(rest, &mut output[written..]); + output.truncate(written); + (Some(output), new_pending) +} +/// Returns `len` zero bytes, or `None` if allocating them fails. +fn zeroed_buffer(len: usize) -> Option> { + if len == 0 { + return Some(Vec::new()); } - - Ok(value::undefined()) + let layout = std::alloc::Layout::array::(len).ok()?; + // SAFETY: `layout` has a non-zero size. + let ptr = unsafe { std::alloc::alloc_zeroed(layout) }; + // SAFETY: `ptr` was allocated by the global allocator with `layout`, and its `len` bytes are + // initialized to zero. + (!ptr.is_null()).then(|| unsafe { Vec::from_raw_parts(ptr, len, len) }) } fn is_leading_surrogate(unit: u16) -> bool { @@ -157,12 +184,6 @@ fn is_trailing_surrogate(unit: u16) -> bool { (0xDC00..=0xDFFF).contains(&unit) } -/// Append the UTF-8 encoding of the scalar value `sv` to `output`. -fn push_scalar_value(output: &mut Vec, sv: u32) { - let c = char::from_u32(sv).expect("a scalar value"); - output.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()); -} - /// The algorithm. fn encoder_flush_cb( scope: &Scope<'_>, diff --git a/crates/builtins/encoding/tests/encoding.rs b/crates/builtins/web-encoding/tests/encoding.rs similarity index 87% rename from crates/builtins/encoding/tests/encoding.rs rename to crates/builtins/web-encoding/tests/encoding.rs index 0bac3bd8..ab7a6981 100644 --- a/crates/builtins/encoding/tests/encoding.rs +++ b/crates/builtins/web-encoding/tests/encoding.rs @@ -4,11 +4,8 @@ #![cfg(not(target_arch = "wasm32"))] -use core_runtime::config::RuntimeConfig; -use core_runtime::event_loop::run_microtasks; -use core_runtime::runtime::{clear_global_initializers, register_global_initializer, Runtime}; -use core_runtime::test_util::eval_with_setup; -use js::conversion::FromJSVal; +use core_runtime::runtime::{clear_global_initializers, register_global_initializer}; +use core_runtime::test_util::{eval_out_with_setup, eval_with_setup}; fn setup() { clear_global_initializers(); @@ -21,17 +18,6 @@ fn eval(code: &str) -> String { eval_with_setup(setup, code) } -fn run_async(code: &str) -> String { - setup(); - let runtime = Runtime::init(&RuntimeConfig::default()).expect("initialize test runtime"); - let scope = runtime.default_global(); - js::compile::evaluate_with_filename(&scope, code, "test.js", 1).unwrap(); - run_microtasks(&scope); - let output = - js::compile::evaluate_with_filename(&scope, "globalThis.__out", "out.js", 1).unwrap(); - String::from_jsval(&scope, output, ()).unwrap() -} - #[test] fn text_decoder_windows_1252_maps_every_byte() { assert_eq!( @@ -113,7 +99,8 @@ fn text_decoder_latin1_fast_paths_gc_safe() { #[test] fn text_decoder_stream_windows_1252_gc_safe() { - let output = run_async( + let output = eval_out_with_setup( + setup, r#" globalThis.__out = "pending"; const inputs = []; diff --git a/crates/builtins/web-streams/src/transform/transform_stream.rs b/crates/builtins/web-streams/src/transform/transform_stream.rs index c1613672..0624ed14 100644 --- a/crates/builtins/web-streams/src/transform/transform_stream.rs +++ b/crates/builtins/web-streams/src/transform/transform_stream.rs @@ -185,7 +185,7 @@ impl TransformStream { } } -/// Helpers for the encoding crate — not part of the WebIDL interface. +/// Helpers for native consumers of TransformStreams. impl TransformStream<'_> { /// ///