diff --git a/Cargo.lock b/Cargo.lock index e01c821c..f287f667 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1524,6 +1524,7 @@ dependencies = [ "tempfile", "tokio", "url", + "web-encoding", "web-fetch", "web-globals", "web-streams", @@ -3078,6 +3079,17 @@ dependencies = [ "semver", ] +[[package]] +name = "web-encoding" +version = "0.3.0" +dependencies = [ + "core-runtime", + "encoding_rs", + "js", + "web-globals", + "web-streams", +] + [[package]] name = "web-fetch" version = "0.3.0" @@ -3103,7 +3115,6 @@ dependencies = [ "bitflags", "core-runtime", "data-url", - "encoding_rs", "js", "libstarling", "platform", diff --git a/Cargo.toml b/Cargo.toml index 8dfddadb..5585210e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ web-globals = { path = "crates/builtins/web-globals" } web-streams = { path = "crates/builtins/web-streams" } web-url = { path = "crates/builtins/web-url" } cpp-builtins = { path = "crates/builtins/cpp-builtins" } +web-encoding = { path = "crates/builtins/encoding" } [profile.release] lto = "thin" diff --git a/crates/builtins/encoding/Cargo.toml b/crates/builtins/encoding/Cargo.toml new file mode 100644 index 00000000..d96b635b --- /dev/null +++ b/crates/builtins/encoding/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "web-encoding" +version.workspace = true +repository.workspace = true +license.workspace = true +authors.workspace = true +edition.workspace = true + +[features] +debugmozjs = ["js/debugmozjs"] + +[dependencies] +core-runtime.workspace = true +encoding_rs.workspace = true +js.workspace = true +web-streams.workspace = true + +[dev-dependencies] +web-globals.workspace = true + +[lints] +workspace = true diff --git a/crates/builtins/encoding/src/decoder_common.rs b/crates/builtins/encoding/src/decoder_common.rs new file mode 100644 index 00000000..83bff1d3 --- /dev/null +++ b/crates/builtins/encoding/src/decoder_common.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +use core_runtime::webidl_union; +use js::error::{ExnThrown, OutOfMemory, ThrowException}; +use js::gc::scope::Scope; +use js::{ArrayBuffer, ArrayBufferView, JSString}; + +pub(crate) fn get_encoding( + label: Option<&str>, +) -> Result<&'static encoding_rs::Encoding, js::error::RangeError> { + let label = label.unwrap_or("utf-8"); + encoding_rs::Encoding::for_label_no_replacement(label.as_bytes()).ok_or_else(|| { + js::error::RangeError(format!( + "The encoding label provided ('{label}') is invalid." + )) + }) +} + +/// Creates a new streaming decoder for the given encoding, with or without BOM handling. +/// +/// Per the Encoding spec: ignoreBOM=true means "ignore the BOM's special meaning" +/// (treat it as a regular U+FEFF character). ignoreBOM=false (default) means the +/// BOM is detected and removed. +pub fn new_decoder( + encoding: &'static encoding_rs::Encoding, + ignore_bom: bool, +) -> encoding_rs::Decoder { + if ignore_bom { + encoding.new_decoder_without_bom_handling() + } else { + encoding.new_decoder_with_bom_removal() + } +} + +pub(crate) fn decode_bytes( + encoding: &'static encoding_rs::Encoding, + decoder: &mut encoding_rs::Decoder, + fatal: bool, + bytes: &[u8], + last: bool, + fresh: bool, +) -> Decoded { + // windows-1252 decodes every byte outside 0x80..=0x9F to the code point of the same + // value, and its decoder keeps no state between calls, so such input is its own + // Latin-1 output. + if encoding == encoding_rs::WINDOWS_1252 && !bytes.iter().any(|b| (0x80..=0x9F).contains(b)) { + return Decoded::Latin1(js::typedarray::try_copy(bytes)); + } + // A fresh decoder that decodes its whole input at once turns ASCII into the same + // characters in every ASCII-compatible encoding. + if fresh + && last + && encoding.is_ascii_compatible() + && encoding_rs::Encoding::ascii_valid_up_to(bytes) == bytes.len() + { + return Decoded::Latin1(js::typedarray::try_copy(bytes)); + } + + // Step 4: Let _output_ be the `I/O queue` of scalar values « `end-of-queue` ». + // Step 5: While true: + // Step 5.1: Let _item_ be the result of `reading` from `this`'s `I/O queue`. + // Step 5.2: If _item_ is `end-of-queue` and `this`'s `do not flush` is true, then + // return the result of running `serialize I/O queue` with `this` and + // _output_. + // Step 5.3: Otherwise: + // Step 5.3.1: Let _result_ be the result of `processing an item` with _item_, + // `this`'s `decoder`, `this`'s `I/O queue`, _output_, and `this`'s `error + // mode`. + // Step 5.3.2: If _result_ is `finished`, then return the result of running + // `serialize I/O queue` with `this` and _output_. + // Step 5.3.3: Otherwise, if _result_ is `error`, `throw` a ``TypeError``. + // + // (One `encoding_rs` call processes every item. `last` is true when `end-of-queue` + // is processed rather than returned at step 5.2. The decoder's BOM removal performs + // `serialize I/O queue`'s BOM handling. We decode to UTF-16, which SpiderMonkey + // copies as is, or narrows to Latin-1 if every code unit fits.) + Decoded::Utf16(decode_to_utf16(decoder, bytes, last, fatal)) +} + +/// The owned output of decoding bytes, ready to convert to a JS string. +pub(crate) enum Decoded { + /// A copy of the input, which is its own Latin-1 output. `None` means that allocating the + /// copy failed. + Latin1(Option>), + /// The UTF-16 output. + Utf16(Result, DecodeError>), +} + +impl Decoded { + pub(crate) fn into_js_string<'r>( + self, + scope: &'r Scope<'_>, + ) -> Result, ExnThrown> { + match self { + Self::Latin1(Some(chars)) => JSString::from_latin1(scope, &chars), + Self::Latin1(None) => Err(OutOfMemory.throw(scope)), + Self::Utf16(output) => { + let output = output.map_err(|error| error.throw(scope))?; + JSString::from_utf16(scope, &output) + } + } + } +} + +/// Why [`decode_to_utf16`] failed. +pub(crate) enum DecodeError { + /// The input is malformed, in the "fatal" error mode. + Malformed, + /// The output can't be allocated. + OutOfMemory, +} + +impl DecodeError { + /// Throw the `TypeError` of malformed input, or report the failed allocation. + fn throw(self, scope: &Scope<'_>) -> ExnThrown { + match self { + DecodeError::Malformed => { + js::error::throw_type_error(scope, c"The encoded data was not valid.") + } + DecodeError::OutOfMemory => OutOfMemory.throw(scope), + } + } +} + +/// Decode `bytes` with `decoder` to UTF-16. `last` is true for the end of the input, which the +/// decoder then flushes. With `fatal`, malformed input fails, and otherwise it becomes U+FFFD. +fn decode_to_utf16( + decoder: &mut encoding_rs::Decoder, + bytes: &[u8], + last: bool, + fatal: bool, +) -> Result, DecodeError> { + let max_len = decoder + .max_utf16_buffer_length(bytes.len()) + .ok_or(DecodeError::OutOfMemory)?; + let mut output = Vec::new(); + output + .try_reserve_exact(max_len) + .map_err(|_| DecodeError::OutOfMemory)?; + output.resize(max_len, 0); + let written = if fatal { + let (result, _read, written) = + decoder.decode_to_utf16_without_replacement(bytes, &mut output, last); + match result { + encoding_rs::DecoderResult::InputEmpty => written, + encoding_rs::DecoderResult::Malformed(_, _) => return Err(DecodeError::Malformed), + encoding_rs::DecoderResult::OutputFull => { + unreachable!("output buffer was pre-allocated to max size"); + } + } + } else { + let (_result, _read, written, _had_errors) = + decoder.decode_to_utf16(bytes, &mut output, last); + written + }; + output.truncate(written); + Ok(output) +} + +#[webidl_union] +pub enum ArrayBufferViewOrArrayBuffer<'a> { + View(ArrayBufferView<'a>), + Buffer(ArrayBuffer<'a>), +} + +/// Options dictionary for TextDecoder and TextDecoderStream constructors. +#[core_runtime::webidl_dictionary] +#[derive(Default)] +pub struct TextDecoderOptions { + /// + /// When true, sets the error mode to "fatal", causing the decoder to throw + /// a TypeError on any malformed input. When false (default), the error mode + /// is "replacement", replacing malformed sequences with U+FFFD. + #[webidl(default = false)] + pub fatal: bool, + + /// + /// When true, the decoder ignores the byte order mark (BOM). When false + /// (default), the decoder detects and removes the BOM. For UTF-16 encodings, + /// the BOM also determines byte order. + #[webidl(default = false, name = "ignoreBOM")] + pub ignore_bom: bool, +} diff --git a/crates/builtins/encoding/src/lib.rs b/crates/builtins/encoding/src/lib.rs new file mode 100644 index 00000000..d1e9d812 --- /dev/null +++ b/crates/builtins/encoding/src/lib.rs @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +//! WHATWG Encoding Standard +//! +//! +//! Provides `TextEncoder`, `TextDecoder`, `TextEncoderStream`, and `TextDecoderStream`. + +mod decoder_common; +mod text_decoder; +mod text_decoder_stream; +mod text_encoder; +mod text_encoder_stream; + +use js::gc::scope::Scope; +use js::Object; + +pub fn add_to_global(scope: &Scope<'_>, global: Object<'_>) { + text_encoder::TextEncoder::add_to_global(scope, global); + text_decoder::TextDecoder::add_to_global(scope, global); + text_decoder_stream::TextDecoderStream::add_to_global(scope, global); + text_encoder_stream::TextEncoderStream::add_to_global(scope, global); +} diff --git a/crates/builtins/encoding/src/text_decoder.rs b/crates/builtins/encoding/src/text_decoder.rs new file mode 100644 index 00000000..a1f9fdbc --- /dev/null +++ b/crates/builtins/encoding/src/text_decoder.rs @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +//! + +use core_runtime::{webidl_dictionary, webidl_interface, webidl_methods}; +use js::error::{ExnThrown, RangeError}; +use js::gc::scope::Scope; +use js::JSString; + +use crate::decoder_common::{ + decode_bytes, get_encoding, new_decoder, ArrayBufferViewOrArrayBuffer, TextDecoderOptions, +}; + +/// +#[webidl_interface] +pub struct TextDecoder { + /// + /// (always `Some` after construction). + #[no_trace] + encoding: Option<&'static encoding_rs::Encoding>, + /// , which also holds the I/O queue + /// and BOM seen state. `None` until the first `decode()` call. + #[no_trace] + decoder: Option, + /// : whether it is "fatal". + fatal: bool, + /// + ignore_bom: bool, + /// + do_not_flush: bool, +} + +#[webidl_methods] +impl TextDecoder<'_> { + /// + #[constructor] + fn new( + &self, + label: Option, + options: Option, + ) -> Result<(), RangeError> { + let options = options.unwrap_or_default(); + + // Step 1: Let _encoding_ be the result of `getting an encoding` from _label_. + // Step 2: If _encoding_ is failure or `replacement`, then `throw` a ``RangeError``. + let encoding = get_encoding(label.as_deref())?; + + let mut data = self.data_mut(); + // Step 3: Set `this`'s `encoding` to _encoding_. + data.encoding = Some(encoding); + // Step 4: If _options_["``fatal``"] is true, then set `this`'s `error mode` to + // "`fatal`". + data.fatal = options.fatal; + // Step 5: Set `this`'s `ignore BOM` to _options_["``ignoreBOM``"]. + data.ignore_bom = options.ignore_bom; + + Ok(()) + } + + /// + #[getter] + fn encoding(&self) -> String { + // Step 1: Return `this`'s `encoding`'s `name`, `ASCII lowercased`. + let encoding = self.data().encoding.expect("set by the constructor"); + encoding.name().to_ascii_lowercase() + } + + /// + #[getter(name = "fatal")] + fn get_fatal(&self) -> bool { + // Step 1: Return true if `this`'s `error mode` is "`fatal`"; otherwise false. + self.data().fatal + } + + /// + #[getter(name = "ignoreBOM")] + fn ignore_bom(&self) -> bool { + // Step 1: Return `this`'s `ignore BOM`. + self.data().ignore_bom + } + + /// + #[method] + fn decode<'r>( + &self, + scope: &'r Scope<'_>, + input: Option>, + options: Option, + ) -> Result, ExnThrown> { + let mut data = self.data_mut(); + let encoding = data.encoding.expect("set by the constructor"); + // Step 1: If `this`'s `do not flush` is false, then set `this`'s `decoder` to a new + // instance of `this`'s `encoding`'s `decoder`, `this`'s `I/O queue` to the `I/O + // queue` of bytes « `end-of-queue` », and `this`'s `BOM seen` to false. + // (The `encoding_rs` decoder holds the I/O queue and BOM seen state.) + let fresh = !data.do_not_flush; + if fresh { + data.decoder = Some(new_decoder(encoding, data.ignore_bom)); + } + // Step 2: Set `this`'s `do not flush` to _options_["``stream``"]. + data.do_not_flush = options.is_some_and(|o| o.stream); + let last = !data.do_not_flush; + let fatal = data.fatal; + let decoder = data.decoder.as_mut().expect("set by step 1"); + + // Step 3: If _input_ is given, then `push` a `copy of` _input_ to `this`'s `I/O queue`. + // (We decode directly from _input_'s bytes. Nothing retains them after this call.) + let mut decode = |bytes: &[u8]| decode_bytes(encoding, decoder, fatal, bytes, last, fresh); + let decoded = match input { + Some(ArrayBufferViewOrArrayBuffer::Buffer(input)) => input.with_bytes(decode), + Some(ArrayBufferViewOrArrayBuffer::View(input)) => input.with_bytes(decode), + None => decode(&[]), + }; + drop(data); + decoded.into_js_string(scope) + } +} + +#[webidl_dictionary] +pub struct TextDecodeOptions { + #[webidl(default = false)] + pub stream: bool, +} diff --git a/crates/builtins/encoding/src/text_decoder_stream.rs b/crates/builtins/encoding/src/text_decoder_stream.rs new file mode 100644 index 00000000..6960e68b --- /dev/null +++ b/crates/builtins/encoding/src/text_decoder_stream.rs @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +//! +//! +//! TextDecoderStream is a TransformStream that decodes bytes to strings using +//! a configurable encoding. It maintains decoder state across chunks to handle +//! incomplete multi-byte sequences at chunk boundaries. + +use core_runtime::{webidl_interface, webidl_methods}; +use js::conversion::FromJSVal; +use js::error::{ExnThrown, ThrowException}; +use js::function::cast_payload; +use js::gc::handle::Heap; +use js::gc::scope::Scope; +use js::native::Value; +use js::prelude::{CallbackArgs, HandleValue}; +use js::{value, JSString}; + +use web_streams::readable::ReadableStream; +use web_streams::writable::WritableStream; +use web_streams::{AlgorithmArg, TransformStream, TransformStreamImpl}; + +use crate::decoder_common::{ + decode_bytes, get_encoding, new_decoder, ArrayBufferViewOrArrayBuffer, TextDecoderOptions, +}; + +/// +#[webidl_interface] +pub struct TextDecoderStream { + /// + /// (always `Some` after construction). + #[no_trace] + encoding: Option<&'static encoding_rs::Encoding>, + + /// + /// true = "fatal" mode (throw on error), false = "replacement" mode. + fatal: bool, + + /// + ignore_bom: bool, + + // Note: The spec defines an I/O queue (https://encoding.spec.whatwg.org/#textdecodercommon-i-o-queue) + // to buffer incomplete multi-byte sequences across chunks. We omit it because encoding_rs's + // Decoder is stateful and tracks partial sequences internally. Per the Decoder contract + // (https://docs.rs/encoding_rs/0.8/encoding_rs/struct.Decoder.html), `src` is only + // partially consumed on `OutputFull` or `Malformed`; since we pre-allocate to prevent + // `OutputFull` and treat `Malformed` as a fatal error, the result is always `InputEmpty` + // with `read == src.len()`. Passing each chunk's bytes directly to the decoder is + // equivalent to the spec's queue-then-process pattern. + /// + /// The decoder instance, preserving state between chunks. + #[no_trace] + decoder: Option, + + /// The underlying TransformStream. + transform: Heap, +} + +#[webidl_methods] +impl TextDecoderStream { + /// + /// 4. Set up a text decoder stream with this, encoding, errorMode, and + /// options["ignoreBOM"]. + /// + /// "Set up a text decoder stream" initializes all internal slots and creates + /// the TransformStream with the decode and flush algorithms. + #[constructor] + fn new( + &self, + scope: &Scope<'_>, + label: Option, + options: Option, + ) -> Result<(), ExnThrown> { + // Step 1: Let encoding be the result of getting an encoding from label. + // Step 2: If encoding is failure or replacement, then throw a RangeError. + let encoding = get_encoding(label.as_deref()).map_err(|error| error.throw(scope))?; + + // Step 3: Let errorMode be "fatal" if options["fatal"] is true; otherwise "replacement". + let options = options.unwrap_or_default(); + let fatal = options.fatal; + let ignore_bom = options.ignore_bom; + + // Step 4: Set up a text decoder stream with this, encoding, errorMode, and + // options["ignoreBOM"]. + // Step 4.1: Assert: encoding is not replacement. + // Step 4.2: Set stream's encoding to encoding. + self.data_mut().encoding = Some(encoding); + // Step 4.3: Set stream's error mode to errorMode. + self.data_mut().fatal = fatal; + // Step 4.4: Set stream's ignore BOM to ignoreBOM. + self.data_mut().ignore_bom = ignore_bom; + // Step 4.5: Set stream’s decoder to a new instance of encoding’s decoder. + self.data_mut().decoder = Some(new_decoder(encoding, ignore_bom)); + // Step 4.6: Set stream's I/O queue to a new I/O queue. + // Implementation note: this is not stored separately because `encoding_rs::Decoder` + // buffers incomplete sequences internally; see the comment on the struct definition. + + // Step 4.7: Let transformAlgorithm be an algorithm which takes a chunk argument and runs + // the decode and enqueue a chunk algorithm with stream and chunk. + let transform = AlgorithmArg::native(scope, decoder_transform_cb, self)?; + // Step 4.8: Let flushAlgorithm be an algorithm which takes no arguments and runs the + // flush and enqueue algorithm with stream. + let flush = AlgorithmArg::native(scope, decoder_flush_cb, self)?; + // Step 4.9: Let transformStream be a new TransformStream. + // Step 4.10: Set up transformStream with transformAlgorithm set to transformAlgorithm and + // flushAlgorithm set to flushAlgorithm. + let transform_stream = + TransformStream::new_native(scope, transform, flush, AlgorithmArg::None)?; + // Step 4.11: Set stream’s [[TransformStream]] to transformStream. + self.data_mut().transform.set(transform_stream); + + Ok(()) + } + + /// + #[getter] + fn encoding(&self) -> String { + // Step 1: Return this’s encoding’s name, ASCII lowercased. + let encoding = self.data().encoding.expect("set by the constructor"); + encoding.name().to_ascii_lowercase() + } + + /// + #[getter] + fn fatal(&self) -> bool { + self.data().fatal + } + + /// + #[getter(name = "ignoreBOM")] + fn ignore_bom(&self) -> bool { + self.data().ignore_bom + } + + /// + #[getter] + fn readable<'r>(&self, scope: &'r Scope<'_>) -> ReadableStream<'r> { + self.data().transform.get(scope).readable(scope) + } + + /// + #[getter] + fn writable<'r>(&self, scope: &'r Scope<'_>) -> WritableStream<'r> { + self.data().transform.get(scope).writable(scope) + } +} + +/// The algorithm. +fn decoder_transform_cb( + scope: &Scope<'_>, + args: CallbackArgs<'_>, + payload: HandleValue<'_>, +) -> Result { + let stream = cast_payload::(scope, payload); + + // Step 1: Let bufferSource be the result of converting chunk to an AllowSharedBufferSource. + let buffer_source = ArrayBufferViewOrArrayBuffer::from_jsval_throwing(scope, args.get(0), ())?; + let output = { + // Step 2: Push a copy of bufferSource to decoder’s I/O queue. + // Step 3: Let output be the I/O queue of scalar values « end-of-queue ». + // Step 4: While true: + // Step 4.1: Let item be the result of reading from decoder’s I/O queue. + // Step 4.2: If item is end-of-queue: + // Step 4.2.1: Let outputChunk be the result of running serialize I/O queue with decoder + // and output. + // Step 4.2.2: If outputChunk is not the empty string, then enqueue outputChunk in + // decoder’s TransformStream. + // Step 4.2.3: Return. + // Step 4.3: Let result be the result of processing an item with item, decoder’s decoder, + // decoder’s I/O queue, output, and decoder’s error mode. + // Step 4.4: If result is error, then throw a TypeError. + let mut data = stream.data_mut(); + let encoding = data.encoding.expect("encoding must be initialized"); + let fatal = data.fatal; + let decoder = data.decoder.as_mut().expect("decoder must be initialized"); + let decode = |bytes: &[u8]| decode_bytes(encoding, decoder, fatal, bytes, false, false); + match buffer_source { + ArrayBufferViewOrArrayBuffer::View(view) => view.with_bytes(decode), + ArrayBufferViewOrArrayBuffer::Buffer(buffer) => buffer.with_bytes(decode), + } + } + .into_js_string(scope)?; + + if !output.is_empty() { + enqueue_to_transform(scope, &stream, output)?; + } + + // Step 4.2.3: Return. + Ok(value::undefined()) +} + +/// The algorithm. +fn decoder_flush_cb( + scope: &Scope<'_>, + _args: CallbackArgs<'_>, + payload: HandleValue<'_>, +) -> Result { + let stream = cast_payload::(scope, payload); + + // Step 1: Let output be the I/O queue of scalar values « end-of-queue ». + // Step 2: While true: + // Step 2.1: Let item be the result of reading from decoder’s I/O queue. + // Step 2.2: Let result be the result of processing an item with item, decoder’s decoder, + // decoder’s I/O queue, output, and decoder’s error mode. + // Step 2.3: If result is finished: + // Step 2.3.1: Let outputChunk be the result of running serialize I/O queue with decoder and + // output. + // Step 2.3.2: If outputChunk is not the empty string, then enqueue outputChunk in decoder’s + // TransformStream. + // Step 2.3.3: Return. + // Step 2.4: Otherwise, if result is error, throw a TypeError. + // Implementation note: `encoding_rs::Decoder` holds incomplete input internally, so an empty + // slice with `last = true` supplies end-of-queue and processes steps 2.1-2.4 to completion. + let output = { + let mut data = stream.data_mut(); + let encoding = data.encoding.expect("encoding must be initialized"); + let fatal = data.fatal; + let decoder = data.decoder.as_mut().expect("decoder must be initialized"); + decode_bytes(encoding, decoder, fatal, &[], true, false) + } + .into_js_string(scope)?; + + if !output.is_empty() { + enqueue_to_transform(scope, &stream, output)?; + } + + // Step 2.3.3: Return. + Ok(value::undefined()) +} + +fn enqueue_to_transform( + scope: &Scope<'_>, + stream: &TextDecoderStream<'_>, + chunk: JSString<'_>, +) -> Result<(), ExnThrown> { + let transform = stream.data().transform.get(scope); + transform.enqueue(scope, scope.root_value(chunk.as_value())) +} diff --git a/crates/builtins/web-globals/src/encoding/text_encoder.rs b/crates/builtins/encoding/src/text_encoder.rs similarity index 100% rename from crates/builtins/web-globals/src/encoding/text_encoder.rs rename to crates/builtins/encoding/src/text_encoder.rs diff --git a/crates/builtins/encoding/src/text_encoder_stream.rs b/crates/builtins/encoding/src/text_encoder_stream.rs new file mode 100644 index 00000000..af8d9e71 --- /dev/null +++ b/crates/builtins/encoding/src/text_encoder_stream.rs @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +//! + +use core_runtime::{webidl_interface, webidl_methods}; +use js::error::{ExnThrown, OutOfMemory, ThrowException}; +use js::function::cast_payload; +use js::gc::handle::Heap; +use js::gc::scope::Scope; +use js::native::Value; +use js::prelude::{CallbackArgs, HandleValue}; +use js::{value, JSString, Uint8Array}; + +use web_streams::readable::ReadableStream; +use web_streams::writable::WritableStream; +use web_streams::{AlgorithmArg, TransformStream, TransformStreamImpl}; + +/// +#[webidl_interface] +pub struct TextEncoderStream { + /// + /// Represents the slot [[pending high surrogate]]. + leading_surrogate: Option, + + /// The underlying TransformStream that handles the streaming pipeline. + transform: Heap, +} + +#[webidl_methods] +impl TextEncoderStream { + /// + /// + /// The `new TextEncoderStream()` constructor steps: + #[constructor] + fn new(&self, scope: &Scope<'_>) -> Result<(), ExnThrown> { + // Step 1: Set this’s [[encoder]] to an instance of the UTF-8 encoder. + // The UTF-8 encoder is stateless, so no explicit assignment is required. + // Step 2: Let transformAlgorithm be an algorithm which takes a chunk argument and runs + // the encode and enqueue a chunk algorithm with this and chunk. + let transform = AlgorithmArg::native(scope, encoder_transform_cb, self)?; + // Step 3: Let flushAlgorithm be an algorithm which runs the encode and flush algorithm + // with this. + let flush = AlgorithmArg::native(scope, encoder_flush_cb, self)?; + // Step 4: Let transformStream be a new TransformStream. + // Step 5: Set up transformStream with transformAlgorithm set to transformAlgorithm and + // flushAlgorithm set to flushAlgorithm. + let transform_stream = + TransformStream::new_native(scope, transform, flush, AlgorithmArg::None)?; + // Step 6: Set this’s [[TransformStream]] to transformStream. + self.data_mut().transform.set(transform_stream); + + Ok(()) + } + + /// + #[getter] + fn encoding(&self) -> &'static str { + "utf-8" + } + + /// + #[getter] + fn readable<'r>(&self, scope: &'r Scope<'_>) -> ReadableStream<'r> { + self.data().transform.get(scope).readable(scope) + } + + /// + #[getter] + fn writable<'r>(&self, scope: &'r Scope<'_>) -> WritableStream<'r> { + self.data().transform.get(scope).writable(scope) + } +} + +/// The algorithm. +fn encoder_transform_cb( + scope: &Scope<'_>, + args: CallbackArgs<'_>, + payload: HandleValue<'_>, +) -> Result { + let stream = cast_payload::(scope, payload); + + // Step 1: Let input be the result of converting chunk to a DOMString. + let js_string = JSString::from_value(scope, args.get(0))?; + // Step 2: Convert input to an I/O queue of code units. + // Step 3: Let output be the I/O queue of bytes « end-of-queue ». + // Step 4: While true: + // Step 4.1: Let item be the result of reading from input. + // Step 4.2: If item is end-of-queue: + // Step 4.2.1: Convert output into a byte sequence. + // Step 4.2.2: If output is not empty: + // Step 4.2.2.1: Let chunk be the result of creating a Uint8Array object given output and + // encoder’s relevant realm. + // Step 4.2.2.2: Enqueue chunk into encoder’s TransformStream. + // Step 4.2.3: Return. + // Step 4.3: Let result be the result of executing the convert code unit to scalar value + // algorithm with encoder, item, and input. + // Step 4.4: If result is not continue, then process an item with result, encoder’s encoder, + // input, output, and "fatal". + // Implementation note: a surrogate pair can only be split at the chunk's edges, so only the + // first and last code units need the [[pending high surrogate]] handling. Everything between + // is encoded in bulk by `to_utf8`, which replaces lone surrogates with U+FFFD exactly as + // steps 4.3 and 4.4 do. + let mut start = 0; + let mut end = js_string.len(); + if end == 0 { + return Ok(value::undefined()); + } + let mut output = Vec::new(); + let pending = stream.data_mut().leading_surrogate.take(); + if let Some(leading) = pending { + let first = js_string.char_at(scope, 0)?; + if is_trailing_surrogate(first) { + let c = 0x10000 + ((u32::from(leading) - 0xD800) << 10) + (u32::from(first) - 0xDC00); + push_scalar_value(&mut output, c); + start = 1; + } else { + push_scalar_value(&mut output, 0xFFFD); + } + } + if start < end { + let last = js_string.char_at(scope, end - 1)?; + if is_leading_surrogate(last) { + stream.data_mut().leading_surrogate = Some(last); + end -= 1; + } + } + if start < end { + let middle = if start == 0 && end == js_string.len() { + js_string + } else { + js_string.substring(scope, start, end - start)? + }; + let utf8 = middle.to_utf8(scope)?.into_bytes(); + if output.is_empty() { + output = utf8; + } else { + output + .try_reserve(utf8.len()) + .map_err(|_| OutOfMemory.throw(scope))?; + output.extend_from_slice(&utf8); + } + } + + if !output.is_empty() { + let u8array = Uint8Array::with_data(scope, &output)?; + enqueue_to_transform(scope, &stream, u8array)?; + } + + Ok(value::undefined()) +} + +fn is_leading_surrogate(unit: u16) -> bool { + (0xD800..=0xDBFF).contains(&unit) +} + +fn is_trailing_surrogate(unit: u16) -> bool { + (0xDC00..=0xDFFF).contains(&unit) +} + +/// Append the UTF-8 encoding of the scalar value `sv` to `output`. +fn push_scalar_value(output: &mut Vec, sv: u32) { + let c = char::from_u32(sv).expect("a scalar value"); + output.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()); +} + +/// The algorithm. +fn encoder_flush_cb( + scope: &Scope<'_>, + _args: CallbackArgs<'_>, + payload: HandleValue<'_>, +) -> Result { + let stream = cast_payload::(scope, payload); + + // Step 1: If encoder’s [[pending high surrogate]] is non-null: + // Scope the mutable borrow so it is dropped before any JS reentry. + let has_pending = { + let mut data = stream.data_mut(); + data.leading_surrogate.take().is_some() + }; + if has_pending { + // Step 1.1: Let chunk be a Uint8Array object wrapping « 0xEF, 0xBF, 0xBD ». (U+FFFD in UTF-8) + let chunk = Uint8Array::with_data(scope, &[0xEF, 0xBF, 0xBD])?; + // Step 1.2: Enqueue chunk in encoder’s [[TransformStream]]. + enqueue_to_transform(scope, &stream, chunk)?; + } + + Ok(value::undefined()) +} + +/// Enqueue a chunk into the TextEncoderStream's TransformStream. +fn enqueue_to_transform( + scope: &Scope<'_>, + stream: &TextEncoderStream<'_>, + chunk: Uint8Array<'_>, +) -> Result<(), ExnThrown> { + let chunk_val = scope.root_value(chunk.as_value()); + let transform = stream.data().transform.get(scope); + transform.enqueue(scope, chunk_val) +} diff --git a/crates/builtins/encoding/tests/encoding.rs b/crates/builtins/encoding/tests/encoding.rs new file mode 100644 index 00000000..0bac3bd8 --- /dev/null +++ b/crates/builtins/encoding/tests/encoding.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception + +//! Implementation-specific regressions not covered by the encoding WPT suite. + +#![cfg(not(target_arch = "wasm32"))] + +use core_runtime::config::RuntimeConfig; +use core_runtime::event_loop::run_microtasks; +use core_runtime::runtime::{clear_global_initializers, register_global_initializer, Runtime}; +use core_runtime::test_util::eval_with_setup; +use js::conversion::FromJSVal; + +fn setup() { + clear_global_initializers(); + register_global_initializer(web_globals::add_to_global); + register_global_initializer(web_encoding::add_to_global); + register_global_initializer(web_streams::add_to_global); +} + +fn eval(code: &str) -> String { + eval_with_setup(setup, code) +} + +fn run_async(code: &str) -> String { + setup(); + let runtime = Runtime::init(&RuntimeConfig::default()).expect("initialize test runtime"); + let scope = runtime.default_global(); + js::compile::evaluate_with_filename(&scope, code, "test.js", 1).unwrap(); + run_microtasks(&scope); + let output = + js::compile::evaluate_with_filename(&scope, "globalThis.__out", "out.js", 1).unwrap(); + String::from_jsval(&scope, output, ()).unwrap() +} + +#[test] +fn text_decoder_windows_1252_maps_every_byte() { + assert_eq!( + eval( + "const decoder = new TextDecoder('latin1'); + const codes = text => Array.from(text, char => char.charCodeAt(0)).join(','); + [decoder.decode(new Uint8Array([0x41, 0xe9, 0xff, 0xa0])), + decoder.decode(new Uint8Array([0x41, 0x80, 0x9f, 0xe9])), + decoder.decode(new Uint8Array([0xe9]), { stream: true }) + + decoder.decode(new Uint8Array([0x80]), { stream: true }) + decoder.decode(), + decoder.decode(new Uint8Array([]))].map(codes).join(' | ')" + ), + "65,233,255,160 | 65,8364,376,233 | 233,8364 | " + ); +} + +#[test] +fn text_decoder_label_errors_match() { + assert_eq!( + eval( + r#" + for (const label of ["invalid-encoding", "replacement", ""]) { + const errors = [TextDecoder, TextDecoderStream].map(Constructor => { + try { + new Constructor(label); + throw new Error("Expected invalid encoding label to throw"); + } catch (error) { + if (!(error instanceof RangeError)) throw error; + return error.message; + } + }); + if (errors[0] !== errors[1] || !errors[0].includes(label)) { + throw new Error("Decoder constructor errors differ"); + } + } + true + "# + ), + "true" + ); +} + +#[test] +fn text_decoder_latin1_fast_paths_gc_safe() { + assert_eq!( + eval( + r#" + for (const encoding of ["utf-8", "windows-1252"]) { + for (const fatal of [false, true]) { + const decoder = new TextDecoder(encoding, { fatal }); + for (const length of [0, 1, 24, 25, 64, 256]) { + const bytes = new Uint8Array(length); + for (let index = 0; index < length; index++) { + bytes[index] = encoding === "utf-8" + ? 65 + index % 26 : 0xA0 + index % 96; + } + const padded = new Uint8Array(length + 4); + padded.set(bytes, 2); + const expected = String.fromCharCode(...bytes); + for (const input of [ + bytes, + bytes.buffer, + padded.subarray(2, length + 2), + new DataView(padded.buffer, 2, length), + ]) { + if (decoder.decode(input) !== expected) { + throw new Error(`Incorrect ${encoding} fast-path output`); + } + } + } + } + } + true + "#, + ), + "true" + ); +} + +#[test] +fn text_decoder_stream_windows_1252_gc_safe() { + let output = run_async( + r#" + globalThis.__out = "pending"; + const inputs = []; + const expected = []; + for (const length of [1, 24, 25, 64]) { + const bytes = new Uint8Array(length).fill(0xE9); + const padded = new Uint8Array(length + 4); + padded.set(bytes, 2); + inputs.push( + bytes, + bytes.buffer, + padded.subarray(2, length + 2), + new DataView(padded.buffer, 2, length), + ); + expected.push("\u00E9".repeat(length * 4)); + } + const source = new ReadableStream({ + start(controller) { + for (const input of inputs) controller.enqueue(input); + controller.close(); + } + }); + (async () => { + const reader = source.pipeThrough(new TextDecoderStream("windows-1252")) + .getReader(); + const chunks = []; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + chunks.push(value); + } + globalThis.__out = String(chunks.join("") === expected.join("")); + })().catch(error => { globalThis.__out = String(error); }); + "#, + ); + assert_eq!(output, "true"); +} diff --git a/crates/builtins/web-globals/Cargo.toml b/crates/builtins/web-globals/Cargo.toml index 979f8b72..fe96ac38 100644 --- a/crates/builtins/web-globals/Cargo.toml +++ b/crates/builtins/web-globals/Cargo.toml @@ -14,7 +14,6 @@ test = false bitflags.workspace = true core-runtime.workspace = true data-url.workspace = true -encoding_rs.workspace = true js.workspace = true platform.workspace = true smallvec.workspace = true diff --git a/crates/builtins/web-globals/src/encoding/mod.rs b/crates/builtins/web-globals/src/encoding/mod.rs deleted file mode 100644 index b43fd31a..00000000 --- a/crates/builtins/web-globals/src/encoding/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception - -mod text_decoder; -mod text_encoder; - -use js::gc::scope::Scope; -use js::Object; - -pub fn add_to_global(scope: &Scope<'_>, global: Object<'_>) { - text_encoder::TextEncoder::add_to_global(scope, global); - text_decoder::TextDecoder::add_to_global(scope, global); -} diff --git a/crates/builtins/web-globals/src/encoding/text_decoder.rs b/crates/builtins/web-globals/src/encoding/text_decoder.rs deleted file mode 100644 index e7242a44..00000000 --- a/crates/builtins/web-globals/src/encoding/text_decoder.rs +++ /dev/null @@ -1,259 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception - -//! - -use core_runtime::{webidl_dictionary, webidl_interface, webidl_methods, webidl_union}; -use js::error::{ExnThrown, OutOfMemory, RangeError, ThrowException}; -use js::gc::scope::Scope; -use js::{ArrayBuffer, ArrayBufferView, JSString}; - -/// -#[webidl_interface] -pub struct TextDecoder { - /// - /// (always `Some` after construction). - #[no_trace] - encoding: Option<&'static encoding_rs::Encoding>, - /// , which also holds the I/O queue - /// and BOM seen state. `None` until the first `decode()` call. - #[no_trace] - decoder: Option, - /// : whether it is "fatal". - fatal: bool, - /// - ignore_bom: bool, - /// - do_not_flush: bool, -} - -#[webidl_methods] -impl TextDecoder<'_> { - /// - #[constructor] - fn new( - &self, - label: Option, - options: Option, - ) -> Result<(), RangeError> { - let label = label.as_deref().unwrap_or("utf-8"); - let options = options.unwrap_or_default(); - - // Step 1: Let _encoding_ be the result of `getting an encoding` from _label_. - // Step 2: If _encoding_ is failure or `replacement`, then `throw` a ``RangeError``. - let encoding = encoding_rs::Encoding::for_label_no_replacement(label.as_bytes()) - .ok_or_else(|| { - RangeError(format!( - "The encoding label provided ('{label}') is invalid." - )) - })?; - - let mut data = self.data_mut(); - // Step 3: Set `this`'s `encoding` to _encoding_. - data.encoding = Some(encoding); - // Step 4: If _options_["``fatal``"] is true, then set `this`'s `error mode` to - // "`fatal`". - data.fatal = options.fatal; - // Step 5: Set `this`'s `ignore BOM` to _options_["``ignoreBOM``"]. - data.ignore_bom = options.ignore_bom; - - Ok(()) - } - - /// - #[getter] - fn encoding(&self) -> String { - // Step 1: Return `this`'s `encoding`'s `name`, `ASCII lowercased`. - let encoding = self.data().encoding.expect("set by the constructor"); - encoding.name().to_ascii_lowercase() - } - - /// - #[getter(name = "fatal")] - fn get_fatal(&self) -> bool { - // Step 1: Return true if `this`'s `error mode` is "`fatal`"; otherwise false. - self.data().fatal - } - - /// - #[getter(name = "ignoreBOM")] - fn ignore_bom(&self) -> bool { - // Step 1: Return `this`'s `ignore BOM`. - self.data().ignore_bom - } - - /// - #[method] - fn decode<'r>( - &self, - scope: &'r Scope<'_>, - input: Option>, - options: Option, - ) -> Result, ExnThrown> { - let mut data = self.data_mut(); - let encoding = data.encoding.expect("set by the constructor"); - // Step 1: If `this`'s `do not flush` is false, then set `this`'s `decoder` to a new - // instance of `this`'s `encoding`'s `decoder`, `this`'s `I/O queue` to the `I/O - // queue` of bytes « `end-of-queue` », and `this`'s `BOM seen` to false. - // (The `encoding_rs` decoder holds the I/O queue and BOM seen state.) - let fresh = !data.do_not_flush; - if fresh { - data.decoder = Some(new_decoder(encoding, data.ignore_bom)); - } - // Step 2: Set `this`'s `do not flush` to _options_["``stream``"]. - data.do_not_flush = options.is_some_and(|o| o.stream); - let last = !data.do_not_flush; - let fatal = data.fatal; - let decoder = data.decoder.as_mut().expect("set by step 1"); - - // Step 3: If _input_ is given, then `push` a `copy of` _input_ to `this`'s `I/O queue`. - // (We decode directly from _input_'s bytes. Nothing retains them after this call.) - let mut decode = |bytes: &[u8]| { - // windows-1252 decodes every byte outside 0x80..=0x9F to the code point of the same - // value, and its decoder keeps no state between calls, so such input is its own - // Latin-1 output. - if encoding == encoding_rs::WINDOWS_1252 - && !bytes.iter().any(|b| (0x80..=0x9F).contains(b)) - { - return Decoded::Latin1(js::typedarray::try_copy(bytes)); - } - // A fresh decoder that decodes its whole input at once turns ASCII into the same - // characters in every ASCII-compatible encoding. - if fresh - && last - && encoding.is_ascii_compatible() - && encoding_rs::Encoding::ascii_valid_up_to(bytes) == bytes.len() - { - return Decoded::Latin1(js::typedarray::try_copy(bytes)); - } - - // Step 4: Let _output_ be the `I/O queue` of scalar values « `end-of-queue` ». - // Step 5: While true: - // Step 5.1: Let _item_ be the result of `reading` from `this`'s `I/O queue`. - // Step 5.2: If _item_ is `end-of-queue` and `this`'s `do not flush` is true, then - // return the result of running `serialize I/O queue` with `this` and - // _output_. - // Step 5.3: Otherwise: - // Step 5.3.1: Let _result_ be the result of `processing an item` with _item_, - // `this`'s `decoder`, `this`'s `I/O queue`, _output_, and `this`'s `error - // mode`. - // Step 5.3.2: If _result_ is `finished`, then return the result of running - // `serialize I/O queue` with `this` and _output_. - // Step 5.3.3: Otherwise, if _result_ is `error`, `throw` a ``TypeError``. - // - // (One `encoding_rs` call processes every item. `last` is true when `end-of-queue` - // is processed rather than returned at step 5.2. The decoder's BOM removal performs - // `serialize I/O queue`'s BOM handling. We decode to UTF-16, which SpiderMonkey - // copies as is, or narrows to Latin-1 if every code unit fits.) - Decoded::Utf16(decode_to_utf16(decoder, bytes, last, fatal)) - }; - let decoded = match input { - Some(ArrayBufferViewOrArrayBuffer::Buffer(input)) => input.with_bytes(decode), - Some(ArrayBufferViewOrArrayBuffer::View(input)) => input.with_bytes(decode), - None => decode(&[]), - }; - drop(data); - match decoded { - Decoded::Latin1(Some(chars)) => JSString::from_latin1(scope, &chars), - Decoded::Latin1(None) => Err(OutOfMemory.throw(scope)), - Decoded::Utf16(output) => { - let output = output.map_err(|error| error.throw(scope))?; - JSString::from_utf16(scope, &output) - } - } - } -} - -/// The result of decoding the input of `TextDecoder.decode`. -enum Decoded { - /// A copy of the input, which is its own Latin-1 output. `None` means that allocating the - /// copy failed. - Latin1(Option>), - /// The UTF-16 output. - Utf16(Result, DecodeError>), -} - -/// A new decoder of `encoding`, which removes a byte order mark at the start of its input unless -/// `ignore_bom` is true. -fn new_decoder(encoding: &'static encoding_rs::Encoding, ignore_bom: bool) -> encoding_rs::Decoder { - if ignore_bom { - encoding.new_decoder_without_bom_handling() - } else { - encoding.new_decoder_with_bom_removal() - } -} - -/// Why [`decode_to_utf16`] failed. -enum DecodeError { - /// The input is malformed, in the "fatal" error mode. - Malformed, - /// The output can't be allocated. - OutOfMemory, -} - -impl DecodeError { - /// Throw the `TypeError` of malformed input, or report the failed allocation. - fn throw(self, scope: &Scope<'_>) -> ExnThrown { - match self { - DecodeError::Malformed => { - js::error::throw_type_error(scope, c"The encoded data was not valid.") - } - DecodeError::OutOfMemory => OutOfMemory.throw(scope), - } - } -} - -/// Decode `bytes` with `decoder` to UTF-16. `last` is true for the end of the input, which the -/// decoder then flushes. With `fatal`, malformed input fails, and otherwise it becomes U+FFFD. -fn decode_to_utf16( - decoder: &mut encoding_rs::Decoder, - bytes: &[u8], - last: bool, - fatal: bool, -) -> Result, DecodeError> { - let max_len = decoder - .max_utf16_buffer_length(bytes.len()) - .ok_or(DecodeError::OutOfMemory)?; - let mut output = Vec::new(); - output - .try_reserve_exact(max_len) - .map_err(|_| DecodeError::OutOfMemory)?; - output.resize(max_len, 0); - let written = if fatal { - let (result, _read, written) = - decoder.decode_to_utf16_without_replacement(bytes, &mut output, last); - match result { - encoding_rs::DecoderResult::InputEmpty => written, - encoding_rs::DecoderResult::Malformed(_, _) => return Err(DecodeError::Malformed), - encoding_rs::DecoderResult::OutputFull => { - unreachable!("output buffer was pre-allocated to max size"); - } - } - } else { - let (_result, _read, written, _had_errors) = - decoder.decode_to_utf16(bytes, &mut output, last); - written - }; - output.truncate(written); - Ok(output) -} - -#[webidl_union] -pub enum ArrayBufferViewOrArrayBuffer<'a> { - View(ArrayBufferView<'a>), - Buffer(ArrayBuffer<'a>), -} - -#[webidl_dictionary] -pub struct TextDecodeOptions { - #[webidl(default = false)] - pub stream: bool, -} - -#[webidl_dictionary] -#[derive(Default)] -pub struct TextDecoderOptions { - #[webidl(default = false)] - pub fatal: bool, - #[webidl(default = false, name = "ignoreBOM")] - pub ignore_bom: bool, -} diff --git a/crates/builtins/web-globals/src/lib.rs b/crates/builtins/web-globals/src/lib.rs index 0033c2ab..da8ca738 100644 --- a/crates/builtins/web-globals/src/lib.rs +++ b/crates/builtins/web-globals/src/lib.rs @@ -12,7 +12,6 @@ pub mod base64; pub mod dom_exception; -pub mod encoding; pub mod events; pub mod navigator; pub mod performance; @@ -22,7 +21,6 @@ pub mod worker_location; pub fn add_to_global(scope: &js::prelude::Scope<'_>, global: js::Object<'_>) { dom_exception::DOMException::add_to_global(scope, global); - encoding::add_to_global(scope, global); events::add_to_global(scope, global); performance::add_to_global(scope, global); signals::add_to_global(scope, global); diff --git a/crates/builtins/web-globals/tests/encoding.rs b/crates/builtins/web-globals/tests/encoding.rs deleted file mode 100644 index d99ce8d4..00000000 --- a/crates/builtins/web-globals/tests/encoding.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0-WITH-LLVM-exception - -//! A `TextDecoder` check that the enabled WPT `encoding/` tests don't make. - -// This file contains nothing platform-specific, so skip it on wasm32. -#![cfg(not(target_arch = "wasm32"))] - -use core_runtime::test_util::eval_with_setup; - -fn setup() { - core_runtime::runtime::register_global_initializer(|scope, global| { - web_globals::add_to_global(scope, global); - }); -} - -fn eval(code: &str) -> String { - eval_with_setup(setup, code) -} - -/// windows-1252 input without bytes in 0x80..=0x9F decodes to the same code points, and input -/// with them maps those bytes through the encoding's table, including when a streaming decode -/// alternates between the two. -#[test] -fn text_decoder_windows_1252_maps_every_byte() { - assert_eq!( - eval( - "const d = new TextDecoder('latin1'); - const codes = s => Array.from(s, c => c.charCodeAt(0)).join(','); - [d.decode(new Uint8Array([0x41, 0xe9, 0xff, 0xa0])), - d.decode(new Uint8Array([0x41, 0x80, 0x9f, 0xe9])), - d.decode(new Uint8Array([0xe9]), { stream: true }) + - d.decode(new Uint8Array([0x80]), { stream: true }) + d.decode(), - d.decode(new Uint8Array([]))].map(codes).join(' | ')" - ), - "65,233,255,160 | 65,8364,376,233 | 233,8364 | " - ); -} diff --git a/crates/builtins/web-streams/src/lib.rs b/crates/builtins/web-streams/src/lib.rs index a0fb19e7..efb2cf5b 100644 --- a/crates/builtins/web-streams/src/lib.rs +++ b/crates/builtins/web-streams/src/lib.rs @@ -5,6 +5,7 @@ pub mod queuing; pub mod readable; pub(crate) mod support; pub mod transform; +pub use transform::{TransformStream, TransformStreamImpl}; pub mod writable; pub use support::AlgorithmArg; diff --git a/crates/builtins/web-streams/src/transform/mod.rs b/crates/builtins/web-streams/src/transform/mod.rs index 4fd32db9..f72c259a 100644 --- a/crates/builtins/web-streams/src/transform/mod.rs +++ b/crates/builtins/web-streams/src/transform/mod.rs @@ -2,7 +2,7 @@ //! -pub(crate) mod algorithms; +mod algorithms; pub mod readable_writable_pair; pub mod transform_stream; @@ -10,6 +10,6 @@ pub mod transform_stream_default_controller; pub mod transformer; pub use readable_writable_pair::ReadableWritablePair; -pub use transform_stream::TransformStream; +pub use transform_stream::{TransformStream, TransformStreamImpl}; pub use transform_stream_default_controller::TransformStreamDefaultController; pub use transformer::Transformer; diff --git a/crates/builtins/web-streams/src/transform/transform_stream.rs b/crates/builtins/web-streams/src/transform/transform_stream.rs index 8bedbaac..c1613672 100644 --- a/crates/builtins/web-streams/src/transform/transform_stream.rs +++ b/crates/builtins/web-streams/src/transform/transform_stream.rs @@ -11,6 +11,7 @@ use crate::algorithms::extract_size_algorithm; use crate::queuing::QueuingStrategy; use crate::readable::readable_stream::ReadableStreamImpl; use crate::readable::ReadableStream; +use crate::support::AlgorithmArg; use crate::writable::writable_stream::WritableStreamImpl; use crate::writable::WritableStream; use core_runtime::{webidl_interface, webidl_methods}; @@ -171,15 +172,80 @@ impl TransformStream { /// #[getter] - fn readable<'r>(&self, scope: &'r Scope<'_>) -> ReadableStream<'r> { + pub fn readable<'r>(&self, scope: &'r Scope<'_>) -> ReadableStream<'r> { // Step 1: Return `this`.`[[readable]]`. self.data().readable.get(scope) } /// #[getter] - fn writable<'r>(&self, scope: &'r Scope<'_>) -> WritableStream<'r> { + pub fn writable<'r>(&self, scope: &'r Scope<'_>) -> WritableStream<'r> { // Step 1: Return `this`.`[[writable]]`. self.data().writable.get(scope) } } + +/// Helpers for the encoding crate — not part of the WebIDL interface. +impl TransformStream<'_> { + /// + /// + /// Create a TransformStream driven by native algorithms, without exposing a transformer + /// object to JavaScript. + pub fn new_native<'r>( + scope: &'r Scope<'_>, + transform: AlgorithmArg<'r>, + flush: AlgorithmArg<'r>, + cancel: AlgorithmArg<'r>, + ) -> Result, ExnThrown> { + let stream = js::class::create_instance_with::(scope, |_| { + TransformStreamImpl::default() + })?; + // Step 1: Let writableHighWaterMark be 1. + // Step 2: Let writableSizeAlgorithm be an algorithm that returns 1. + // Step 3: Let readableHighWaterMark be 0. + // Step 4: Let readableSizeAlgorithm be an algorithm that returns 1. + // Steps 5-7: Let transformAlgorithmWrapper, flushAlgorithmWrapper, and + // cancelAlgorithmWrapper run the given algorithms, converting their results + // and exceptions to promises. + // (The controller's algorithm invocation performs this wrapping.) + // Step 8: Let startPromise be a promise resolved with undefined. + let start_promise = Promise::new_resolved_with_value(scope, HandleValue::undefined())?; + // Step 9: Perform ! InitializeTransformStream(stream, startPromise, + // writableHighWaterMark, writableSizeAlgorithm, readableHighWaterMark, + // readableSizeAlgorithm). + algorithms::initialize_transform_stream( + scope, + &stream, + &start_promise, + 1.0, + HandleValue::undefined(), + 0.0, + HandleValue::undefined(), + )?; + // Step 10: Let controller be a new TransformStreamDefaultController. + let controller = TransformStreamDefaultController::new(scope, stream)?; + // Step 11: Perform ! SetUpTransformStreamDefaultController(stream, controller, + // transformAlgorithmWrapper, flushAlgorithmWrapper, cancelAlgorithmWrapper). + algorithms::set_up_transform_stream_default_controller( + scope, + &stream, + &controller, + transform, + flush, + cancel, + HandleValue::undefined(), + ); + Ok(stream) + } + + /// Enqueue a chunk into the readable side of this TransformStream. + pub fn enqueue(&self, scope: &Scope<'_>, chunk: HandleValue<'_>) -> Result<(), ExnThrown> { + let ctrl = self + .data() + .controller + .as_ref() + .expect("controller must be set") + .get(scope); + algorithms::transform_stream_default_controller_enqueue(scope, &ctrl, chunk) + } +} diff --git a/crates/libstarling/Cargo.toml b/crates/libstarling/Cargo.toml index 464387f9..5d45e20b 100644 --- a/crates/libstarling/Cargo.toml +++ b/crates/libstarling/Cargo.toml @@ -21,6 +21,7 @@ event-listener.workspace = true futures-lite.workspace = true futures-util.workspace = true http.workspace = true +web-encoding.workspace = true js.workspace = true platform.workspace = true fetch-event.workspace = true diff --git a/crates/libstarling/src/lib.rs b/crates/libstarling/src/lib.rs index e6d0744f..7665d058 100644 --- a/crates/libstarling/src/lib.rs +++ b/crates/libstarling/src/lib.rs @@ -25,6 +25,7 @@ pub mod serve_wasm; pub fn register_builtins() { runtime::register_global_initializer(web_globals::add_to_global); runtime::register_global_initializer(web_streams::add_to_global); + runtime::register_global_initializer(web_encoding::add_to_global); runtime::register_global_initializer(web_url::add_to_global); runtime::register_global_initializer(web_fetch::add_to_global); runtime::register_global_initializer(fetch_event::add_to_global); diff --git a/scripts/test-gc-zeal.sh b/scripts/test-gc-zeal.sh index 2587097f..bb54329b 100755 --- a/scripts/test-gc-zeal.sh +++ b/scripts/test-gc-zeal.sh @@ -89,7 +89,7 @@ done if [[ "$PACKAGE_SELECTED" == false ]]; then CARGO_ARGS=( -p js -p core-runtime -p wit-bridge - -p cpp-builtins -p fetch-event -p web-fetch -p web-globals -p web-streams -p web-url + -p cpp-builtins -p fetch-event -p web-encoding -p web-fetch -p web-globals -p web-streams -p web-url ${CARGO_ARGS[@]+"${CARGO_ARGS[@]}"} ) fi diff --git a/tests/wpt-harness/expectations/encoding/idlharness.any.js.json b/tests/wpt-harness/expectations/encoding/idlharness.any.js.json index 00b7eabd..52474344 100644 --- a/tests/wpt-harness/expectations/encoding/idlharness.any.js.json +++ b/tests/wpt-harness/expectations/encoding/idlharness.any.js.json @@ -123,51 +123,51 @@ "status": "PASS" }, "TextDecoderStream interface: existence and properties of interface object": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface object length": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface object name": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: existence and properties of interface prototype object": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: existence and properties of interface prototype object's \"constructor\" property": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: existence and properties of interface prototype object's @@unscopables property": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: attribute encoding": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: attribute fatal": { - "status": "FAIL" + "status": "PASS" }, "TextDecoderStream interface: attribute ignoreBOM": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface: existence and properties of interface object": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface object length": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface object name": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface: existence and properties of interface prototype object": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface: existence and properties of interface prototype object's \"constructor\" property": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface: existence and properties of interface prototype object's @@unscopables property": { - "status": "FAIL" + "status": "PASS" }, "TextEncoderStream interface: attribute encoding": { - "status": "FAIL" + "status": "PASS" } -} \ No newline at end of file +} diff --git a/tests/wpt-harness/expectations/encoding/streams/backpressure.any.js.json b/tests/wpt-harness/expectations/encoding/streams/backpressure.any.js.json new file mode 100644 index 00000000..f2c2435c --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/backpressure.any.js.json @@ -0,0 +1,14 @@ +{ + "write() should not complete until read relieves backpressure for TextDecoderStream": { + "status": "PASS" + }, + "additional writes should wait for backpressure to be relieved for class TextDecoderStream": { + "status": "PASS" + }, + "write() should not complete until read relieves backpressure for TextEncoderStream": { + "status": "PASS" + }, + "additional writes should wait for backpressure to be relieved for class TextEncoderStream": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-attributes.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-attributes.any.js.json new file mode 100644 index 00000000..f3cb0937 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-attributes.any.js.json @@ -0,0 +1,86 @@ +{ + "encoding attribute should have correct value for 'unicode-1-1-utf-8'": { + "status": "PASS" + }, + "encoding attribute should have correct value for 'iso-8859-2'": { + "status": "PASS" + }, + "encoding attribute should have correct value for 'ascii'": { + "status": "PASS" + }, + "encoding attribute should have correct value for 'utf-16'": { + "status": "PASS" + }, + "setting fatal to 'false' should set the attribute to false": { + "status": "PASS" + }, + "setting ignoreBOM to 'false' should set the attribute to false": { + "status": "PASS" + }, + "setting fatal to '0' should set the attribute to false": { + "status": "PASS" + }, + "setting ignoreBOM to '0' should set the attribute to false": { + "status": "PASS" + }, + "setting fatal to '' should set the attribute to false": { + "status": "PASS" + }, + "setting ignoreBOM to '' should set the attribute to false": { + "status": "PASS" + }, + "setting fatal to 'undefined' should set the attribute to false": { + "status": "PASS" + }, + "setting ignoreBOM to 'undefined' should set the attribute to false": { + "status": "PASS" + }, + "setting fatal to 'null' should set the attribute to false": { + "status": "PASS" + }, + "setting ignoreBOM to 'null' should set the attribute to false": { + "status": "PASS" + }, + "setting fatal to 'true' should set the attribute to true": { + "status": "PASS" + }, + "setting ignoreBOM to 'true' should set the attribute to true": { + "status": "PASS" + }, + "setting fatal to '1' should set the attribute to true": { + "status": "PASS" + }, + "setting ignoreBOM to '1' should set the attribute to true": { + "status": "PASS" + }, + "setting fatal to '[object Object]' should set the attribute to true": { + "status": "PASS" + }, + "setting ignoreBOM to '[object Object]' should set the attribute to true": { + "status": "PASS" + }, + "setting fatal to '' should set the attribute to true": { + "status": "PASS" + }, + "setting ignoreBOM to '' should set the attribute to true": { + "status": "PASS" + }, + "setting fatal to 'yes' should set the attribute to true": { + "status": "PASS" + }, + "setting ignoreBOM to 'yes' should set the attribute to true": { + "status": "PASS" + }, + "constructing with an invalid encoding should throw": { + "status": "PASS" + }, + "constructing with a non-stringifiable encoding should throw": { + "status": "PASS" + }, + "a throwing fatal member should cause the constructor to throw": { + "status": "PASS" + }, + "a throwing ignoreBOM member should cause the constructor to throw": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-bad-chunks.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-bad-chunks.any.js.json new file mode 100644 index 00000000..3bac5477 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-bad-chunks.any.js.json @@ -0,0 +1,17 @@ +{ + "chunk of type undefined should error the stream": { + "status": "PASS" + }, + "chunk of type null should error the stream": { + "status": "PASS" + }, + "chunk of type numeric should error the stream": { + "status": "PASS" + }, + "chunk of type object, not BufferSource should error the stream": { + "status": "PASS" + }, + "chunk of type array should error the stream": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-ignore-bom.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-ignore-bom.any.js.json new file mode 100644 index 00000000..0b138212 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-ignore-bom.any.js.json @@ -0,0 +1,38 @@ +{ + "ignoreBOM should work for encoding utf-8, split at character 0": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-8, split at character 1": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-8, split at character 2": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-8, split at character 3": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16le, split at character 0": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16le, split at character 1": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16le, split at character 2": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16le, split at character 3": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16be, split at character 0": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16be, split at character 1": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16be, split at character 2": { + "status": "PASS" + }, + "ignoreBOM should work for encoding utf-16be, split at character 3": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-incomplete-input.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-incomplete-input.any.js.json new file mode 100644 index 00000000..c97336eb --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-incomplete-input.any.js.json @@ -0,0 +1,8 @@ +{ + "incomplete input with error mode \"replacement\" should end with a replacement character": { + "status": "PASS" + }, + "incomplete input with error mode \"fatal\" should error the stream": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-non-utf8.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-non-utf8.any.js.json new file mode 100644 index 00000000..2df45e96 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-non-utf8.any.js.json @@ -0,0 +1,41 @@ +{ + "TextDecoderStream should be able to decode UTF-16BE": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode invalid sequences in UTF-16BE": { + "status": "PASS" + }, + "TextDecoderStream should be able to reject invalid sequences in UTF-16BE": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode UTF-16LE": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode invalid sequences in UTF-16LE": { + "status": "PASS" + }, + "TextDecoderStream should be able to reject invalid sequences in UTF-16LE": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode Shift_JIS": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode invalid sequences in Shift_JIS": { + "status": "PASS" + }, + "TextDecoderStream should be able to reject invalid sequences in Shift_JIS": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode ISO-2022-JP": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode invalid sequences in ISO-2022-JP": { + "status": "PASS" + }, + "TextDecoderStream should be able to reject invalid sequences in ISO-2022-JP": { + "status": "PASS" + }, + "TextDecoderStream should be able to decode ISO-8859-14": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/decode-split-character.any.js.json b/tests/wpt-harness/expectations/encoding/streams/decode-split-character.any.js.json new file mode 100644 index 00000000..eab2671e --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/decode-split-character.any.js.json @@ -0,0 +1,35 @@ +{ + "a code point split between chunks should not be emitted until all bytes are available; split point = 2": { + "status": "PASS" + }, + "a code point split between chunks should not be emitted until all bytes are available; split point = 3": { + "status": "PASS" + }, + "a code point split between chunks should not be emitted until all bytes are available; split point = 4": { + "status": "PASS" + }, + "a code point split between chunks should not be emitted until all bytes are available; split point = 5": { + "status": "PASS" + }, + "a code point should be emitted as soon as all bytes are available": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 1": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 2": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 3": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 4": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 5": { + "status": "PASS" + }, + "an empty chunk inside a code point split between chunks should not change the output; split point = 6": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/encode-bad-chunks.any.js.json b/tests/wpt-harness/expectations/encoding/streams/encode-bad-chunks.any.js.json new file mode 100644 index 00000000..09ebc7e8 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/encode-bad-chunks.any.js.json @@ -0,0 +1,20 @@ +{ + "a chunk that cannot be converted to a string should error the streams": { + "status": "PASS" + }, + "input of type undefined should be converted correctly to string": { + "status": "PASS" + }, + "input of type null should be converted correctly to string": { + "status": "PASS" + }, + "input of type numeric should be converted correctly to string": { + "status": "PASS" + }, + "input of type object should be converted correctly to string": { + "status": "PASS" + }, + "input of type array should be converted correctly to string": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/encode-utf8.any.js.json b/tests/wpt-harness/expectations/encoding/streams/encode-utf8.any.js.json new file mode 100644 index 00000000..037dcc6d --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/encode-utf8.any.js.json @@ -0,0 +1,59 @@ +{ + "encoding one string of UTF-8 should give one complete chunk": { + "status": "PASS" + }, + "a character split between chunks should be correctly encoded": { + "status": "PASS" + }, + "a character following one split between chunks should be correctly encoded": { + "status": "PASS" + }, + "two consecutive astral characters each split down the middle should be correctly reassembled": { + "status": "PASS" + }, + "two consecutive astral characters each split down the middle with an invalid surrogate in the middle should be correctly encoded": { + "status": "PASS" + }, + "a stream ending in a leading surrogate should emit a replacement character as a final chunk": { + "status": "PASS" + }, + "an unmatched surrogate at the end of a chunk followed by an astral character in the next chunk should be replaced with the replacement character at the start of the next output chunk": { + "status": "PASS" + }, + "an unmatched surrogate at the end of a chunk followed by an ascii character in the next chunk should be replaced with the replacement character at the start of the next output chunk": { + "status": "PASS" + }, + "an unmatched surrogate at the end of a chunk followed by a plane 1 character split into two chunks should result in the encoded plane 1 character appearing in the last output chunk": { + "status": "PASS" + }, + "two leading chunks should result in two replacement characters": { + "status": "PASS" + }, + "a non-terminal unpaired leading surrogate should immediately be replaced": { + "status": "PASS" + }, + "a terminal unpaired trailing surrogate should immediately be replaced": { + "status": "PASS" + }, + "a leading surrogate chunk should be carried past empty chunks": { + "status": "PASS" + }, + "a leading surrogate chunk should error when it is clear it didn't form a pair": { + "status": "PASS" + }, + "an empty string should result in no output chunk": { + "status": "PASS" + }, + "a leading empty chunk should be ignored": { + "status": "PASS" + }, + "a trailing empty chunk should be ignored": { + "status": "PASS" + }, + "a plain ASCII chunk should be converted": { + "status": "PASS" + }, + "characters in the ISO-8859-1 range should be encoded correctly": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/expectations/encoding/streams/readable-writable-properties.any.js.json b/tests/wpt-harness/expectations/encoding/streams/readable-writable-properties.any.js.json new file mode 100644 index 00000000..925c4992 --- /dev/null +++ b/tests/wpt-harness/expectations/encoding/streams/readable-writable-properties.any.js.json @@ -0,0 +1,8 @@ +{ + "TextEncoderStream readable and writable properties must pass brand checks": { + "status": "PASS" + }, + "TextDecoderStream readable and writable properties must pass brand checks": { + "status": "PASS" + } +} diff --git a/tests/wpt-harness/tests.json b/tests/wpt-harness/tests.json index 72d29455..669f69e5 100644 --- a/tests/wpt-harness/tests.json +++ b/tests/wpt-harness/tests.json @@ -140,6 +140,17 @@ "SLOW encoding/idlharness.any.js", "encoding/iso-2022-jp-decoder.any.js", "encoding/replacement-encodings.any.js", + "encoding/streams/backpressure.any.js", + "encoding/streams/decode-attributes.any.js", + "encoding/streams/decode-bad-chunks.any.js", + "encoding/streams/decode-ignore-bom.any.js", + "encoding/streams/decode-incomplete-input.any.js", + "encoding/streams/decode-non-utf8.any.js", + "encoding/streams/decode-split-character.any.js", + "SKIP(Requires SharedArrayBuffer and MessageChannel) encoding/streams/decode-utf8.any.js", + "encoding/streams/encode-bad-chunks.any.js", + "encoding/streams/encode-utf8.any.js", + "encoding/streams/readable-writable-properties.any.js", "encoding/textdecoder-arguments.any.js", "encoding/textdecoder-byte-order-marks.any.js", "encoding/textdecoder-copy.any.js",