diff --git a/Cargo.lock b/Cargo.lock index 6a114150c4d..1f45b835c6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8255,6 +8255,7 @@ dependencies = [ "crossbeam-queue", "deno_core_icudata", "derive_more 0.99.20", + "encoding_rs", "enum-map", "faststr", "flate2", diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 0cd5fcef773..3d0adcff4a4 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -104,6 +104,7 @@ wasmtime-internal-fiber.workspace = true async_cache = "0.3.1" faststr = "0.2.23" core_affinity = "0.8" +encoding_rs = "0.8" [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = {workspace = true} diff --git a/crates/core/src/host/v8/builtins/mod.rs b/crates/core/src/host/v8/builtins/mod.rs index 37e795a38ec..d8c215f510e 100644 --- a/crates/core/src/host/v8/builtins/mod.rs +++ b/crates/core/src/host/v8/builtins/mod.rs @@ -1,7 +1,7 @@ use v8::{FunctionCallbackArguments, Local, PinScope}; use super::de::scratch_buf; -use super::error::{exception_already_thrown, ExcResult, StringTooLongError, Throwable, TypeError}; +use super::error::{exception_already_thrown, ExcResult, RangeError, Throwable, TypeError}; use super::string::{str_from_ident, StringConst}; use super::{FnRet, IntoJsString}; @@ -104,7 +104,14 @@ fn resolve_builtins_module<'scope>( /// This is not public API, since it's not accessible to user modules - only to /// the js builtins in this directory. fn internal_builtins_module<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, v8::Module> { - create_synthetic_module!(scope, "spacetime:internal_builtins", utf8_encode, utf8_decode) + create_synthetic_module!( + scope, + "spacetime:internal_builtins", + utf8_encode, + utf8_decode, + normalize_label, + generic_decode + ) } /// Encode a JS string into UTF-8. @@ -135,22 +142,148 @@ fn utf8_encode<'scope>(scope: &mut PinScope<'scope, '_>, args: FunctionCallbackA /// /// Signature fom ./types.d.ts: /// ```ts -/// export function utf8_decode(s: ArrayBufferView, fatal: boolean): string; +/// export function utf8_decode(buf: AllowSharedBufferSource, ignoreBOM: boolean): string; /// ``` fn utf8_decode<'scope>(scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>) -> FnRet<'scope> { let buf = args.get(0); - let fatal = args.get(1).boolean_value(scope); - if let Ok(buf) = buf.try_cast::() { - let buffer = buf.get_contents(&mut []); - let res = if fatal { - let s = std::str::from_utf8(buffer).map_err(|e| TypeError(e.to_string()).throw(scope))?; - s.into_string(scope) - } else { - v8::String::new_from_utf8(scope, buffer, v8::NewStringType::Normal) - .ok_or_else(|| StringTooLongError::new(&String::from_utf8_lossy(buffer))) - }; - res.map(Into::into).map_err(|e| e.into_range_error().throw(scope)) + let ignore_bom = args.get(1).boolean_value(scope); + let buf = cast_buffer(scope, buf)?; + let mut buffer = buf.get_contents(&mut []); + const BOM_UTF8: &[u8] = &[0xEF, 0xBB, 0xBF]; + if !ignore_bom { + buffer = buffer.strip_prefix(BOM_UTF8).unwrap_or(buffer) + } + let string = v8::String::new_from_utf8(scope, buffer, v8::NewStringType::Normal) + .ok_or_else(|| RangeError("Value too large to decode").throw(scope))?; + Ok(string.into()) +} + +fn cast_buffer<'scope, 'a: 'scope>( + scope: &mut PinScope<'scope, '_>, + buf: Local<'a, v8::Value>, +) -> ExcResult> { + buf.try_cast::() + .or_else(|_| { + buf.try_cast::() + .map(|buf| v8::Uint8Array::new(scope, buf, 0, buf.byte_length()).unwrap().into()) + }) + .or_else(|_| { + buf.try_cast::().map(|buf| { + // pretend that the SAB is a regular AB - rusty_v8 doesn't implement the overload + let arr = unsafe { Local::::cast_unchecked(Local::::from(buf)) }; + v8::Uint8Array::new(scope, arr, 0, buf.byte_length()).unwrap().into() + }) + }) + .map_err(|_| TypeError("argument is not an `ArrayBuffer` or a view on one").throw(scope)) +} + +/// Map a label for an encoding to its canonical form. +/// +/// Signature fom ./types.d.ts: +/// ```ts +/// export function normalize_label(label: string): string | null; +/// ``` +fn normalize_label<'scope>(scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>) -> FnRet<'scope> { + let label = args.get(0); + let label = label + .to_string(scope) + .ok_or_else(exception_already_thrown)? + .to_rust_string_lossy(scope); + match encoding_rs::Encoding::for_label_no_replacement(label.as_bytes()) { + Some(encoding) => Ok(encoding.name().to_ascii_lowercase().into_string(scope).unwrap().into()), + None => Ok(v8::null(scope).into()), + } +} + +/// Decode a UTF-8 string from an `ArrayBuffer` into a JS string. +/// +/// If `fatal` is true, throw an error if the data is not valid UTF-8. +/// +/// Signature fom ./types.d.ts: +/// ```ts +/// export function generic_decode(encoding: string, buf: AllowSharedBufferSource, fatal: boolean, ignoreBOM: boolean): string; +/// ``` +fn generic_decode<'scope>(scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>) -> FnRet<'scope> { + let mut scratch = scratch_buf::<32>(); + // unsafe cast: this will only ever be called by this privileged code in text_encoding.js + let encoding = args.get(0).cast::().to_rust_cow_lossy(scope, &mut scratch); + let encoding = encoding_rs::Encoding::for_label(encoding.as_bytes()).unwrap(); + + let buf = args + .get(1) + .try_cast::() + .map_err(|_| TypeError("argument is not an `ArrayBuffer` or a view on one").throw(scope))?; + let fatal = args.get(2).boolean_value(scope); + let ignore_bom = args.get(3).boolean_value(scope); + + let mut buffer = buf.get_contents(&mut []); + + let mut decoder = if ignore_bom { + encoding.new_decoder_without_bom_handling() } else { - Err(TypeError("argument is not an `ArrayBuffer` or a view on one").throw(scope)) + encoding.new_decoder_with_bom_removal() + }; + + let len = decoder + .max_utf16_buffer_length(buffer.len()) + .ok_or_else(|| RangeError("Value too large to decode").throw(scope))?; + let mut output = vec![0u16; len]; + + let mut total_written = 0; + let mut out = &mut output[..]; + if fatal { + loop { + let (result, read, written) = decoder.decode_to_utf16_without_replacement(buffer, out, true); + total_written += written; + match result { + encoding_rs::DecoderResult::InputEmpty => break, + encoding_rs::DecoderResult::OutputFull => { + buffer = &buffer[read..]; + output.reserve(1); + out = &mut output[total_written..]; + } + encoding_rs::DecoderResult::Malformed(_, _) => { + return Err(TypeError("The encoded data is not valid").throw(scope)) + } + } + } + } else { + loop { + let (result, read, written, _) = decoder.decode_to_utf16(buffer, out, true); + total_written += written; + match result { + encoding_rs::CoderResult::InputEmpty => break, + encoding_rs::CoderResult::OutputFull => { + buffer = &buffer[read..]; + output.reserve(1); + out = &mut output[total_written..]; + } + } + } + } + + output.truncate(total_written); + + v8::String::new_from_two_byte(scope, &output, v8::NewStringType::Normal) + .map(Into::into) + .ok_or_else(|| RangeError("Value too large to decode").throw(scope)) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::host::v8::to_value::test::with_scope; + use crate::host::v8::{catch_exception, eval_user_module}; + + #[test] + fn text_decoder_dependency_construction_and_decoding() { + with_scope(|scope| { + catch_exception(scope, |scope| { + evaluate_builtins(scope)?; + eval_user_module(scope, include_str!("builtins/text_encoding.test.js"))?; + Ok(()) + }) + .expect("TextDecoder regression module should evaluate successfully"); + }); } } diff --git a/crates/core/src/host/v8/builtins/text_encoding.js b/crates/core/src/host/v8/builtins/text_encoding.js index 1a343e04d34..58b4238da0c 100644 --- a/crates/core/src/host/v8/builtins/text_encoding.js +++ b/crates/core/src/host/v8/builtins/text_encoding.js @@ -2,7 +2,12 @@ /// -import { utf8_decode, utf8_encode } from 'spacetime:internal_builtins'; +import { + generic_decode, + normalize_label, + utf8_decode, + utf8_encode, +} from 'spacetime:internal_builtins'; globalThis.TextEncoder = class TextEncoder { constructor() {} @@ -23,19 +28,29 @@ globalThis.TextDecoder = class TextDecoder { /** @type {boolean} */ #fatal; + /** @type {boolean} */ + #ignoreBOM; + + /** @type {boolean} */ + #utf8FastPath; + /** * @argument {string} label * @argument {any} options */ constructor(label = 'utf-8', options = {}) { - if (label !== 'utf-8') { - throw new RangeError('The encoding label provided is invalid'); + if (label === 'utf-8' || label === 'utf8') { + label = 'utf-8'; + } else { + const normalized = normalize_label(label); + if (normalized == null) + throw new RangeError('The encoding label provided is invalid'); + label = normalized; } this.#encoding = label; this.#fatal = !!options.fatal; - if (options.ignoreBOM) { - throw new TypeError("Option 'ignoreBOM' not supported"); - } + this.#ignoreBOM = !!options.ignoreBOM; + this.#utf8FastPath = label === 'utf-8' && !this.#fatal; } get encoding() { @@ -45,20 +60,20 @@ globalThis.TextDecoder = class TextDecoder { return this.#fatal; } get ignoreBOM() { - return false; + return this.#ignoreBOM; } /** - * @argument {any} input + * @argument {AllowSharedBufferSource} input * @argument {any} options */ decode(input, options = {}) { if (options.stream) { throw new TypeError("Option 'stream' not supported"); } - if (input instanceof ArrayBuffer || input instanceof SharedArrayBuffer) { - input = new Uint8Array(input); + if (this.#utf8FastPath) { + return utf8_decode(input, this.#ignoreBOM); } - return utf8_decode(input, this.#fatal); + return generic_decode(this.#encoding, input, this.#fatal, this.#ignoreBOM); } }; diff --git a/crates/core/src/host/v8/builtins/text_encoding.test.js b/crates/core/src/host/v8/builtins/text_encoding.test.js new file mode 100644 index 00000000000..026d1b2f40a --- /dev/null +++ b/crates/core/src/host/v8/builtins/text_encoding.test.js @@ -0,0 +1,151 @@ +// Evaluated as a user module by the V8 host test, after installing the builtins. +function equal(actual, expected) { + if (actual !== expected) { + throw new Error( + `Expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}` + ); + } +} + +function throws(action, type) { + try { + action(); + } catch (error) { + if (error instanceof type) { + return; + } + throw error; + } + throw new Error(`Expected ${type.name}`); +} + +// h3-js's generated browser bundle constructs both of these eagerly during +// module evaluation, including the unused UTF16Decoder (issue #5994). +const UTF8Decoder = new TextDecoder('utf8'); +const UTF16Decoder = new TextDecoder('utf-16le'); +equal(UTF8Decoder.encoding, 'utf-8'); +equal(UTF16Decoder.encoding, 'utf-16le'); +equal(UTF8Decoder.decode(new Uint8Array([0x48, 0xc3, 0xa9])), 'Hé'); +equal(UTF16Decoder.decode(new Uint8Array([0x48, 0, 0xe9, 0])), 'Hé'); + +for (const [encoding, labels] of [ + [ + 'utf-8', + [ + 'unicode-1-1-utf-8', + 'unicode11utf8', + 'unicode20utf8', + 'utf-8', + 'utf8', + 'x-unicode20utf8', + ], + ], + [ + 'utf-16le', + [ + 'csunicode', + 'iso-10646-ucs-2', + 'ucs-2', + 'unicode', + 'unicodefeff', + 'utf-16', + 'utf-16le', + ], + ], +]) { + for (const label of labels) { + equal(new TextDecoder(label).encoding, encoding); + equal( + new TextDecoder(` \t\n\f\r${label.toUpperCase()}\r\f\n\t `).encoding, + encoding + ); + } +} +equal(new TextDecoder().encoding, 'utf-8'); +equal(new TextDecoder({ toString: () => 'UTF8' }).encoding, 'utf-8'); +throws(() => new TextDecoder(Symbol()), TypeError); +for (const label of [ + '', + 'utf-16be', + 'latin1', + 'replacement', + 'utf_8', + '\u00a0utf8', + 'utf8\u000b', + 'utf8\uFEFF', +]) { + throws(() => new TextDecoder(label), RangeError); +} + +for (const decoder of [UTF8Decoder, UTF16Decoder]) { + equal(decoder.fatal, false); + equal(decoder.ignoreBOM, false); + equal(decoder.decode(), ''); + equal(decoder.decode(new Uint8Array()), ''); + throws(() => decoder.decode(null), TypeError); + throws(() => decoder.decode([0x41, 0]), TypeError); + throws(() => decoder.decode(new Uint8Array(), { stream: true }), TypeError); + throws( + () => new TextDecoder(decoder.encoding, { ignoreBOM: true }), + TypeError + ); +} + +// Buffer sources must decode the bytes in the view, including odd offsets. +const buffer = new Uint8Array([0xff, 0x41, 0, 0x3d, 0xd8, 0, 0xde, 0xff]); +equal(UTF16Decoder.decode(buffer.subarray(1, 7)), 'A😀'); +equal(UTF16Decoder.decode(new DataView(buffer.buffer, 1, 6)), 'A😀'); +equal(UTF16Decoder.decode(buffer.slice(1, 7).buffer), 'A😀'); +const words = new Uint16Array(3); +new Uint8Array(words.buffer).set([0x41, 0, 0x3d, 0xd8, 0, 0xde]); +equal(UTF16Decoder.decode(words), 'A😀'); +const shared = new SharedArrayBuffer(6); +new Uint8Array(shared).set([0x41, 0, 0x3d, 0xd8, 0, 0xde]); +equal(UTF16Decoder.decode(shared), 'A😀'); +equal(UTF16Decoder.decode(new DataView(shared, 2, 4)), '😀'); + +equal( + UTF16Decoder.decode(new Uint8Array([0xff, 0xfe, 0x41, 0, 0xff, 0xfe])), + 'A\uFEFF' +); +equal(UTF16Decoder.decode(new Uint8Array([0xff, 0xfe])), ''); +equal(UTF16Decoder.decode(new Uint8Array([0xff, 0xfe, 0x42, 0])), 'B'); +// A big-endian BOM does not change the selected encoding. +equal( + UTF16Decoder.decode(new Uint8Array([0xfe, 0xff, 0, 0x41])), + '\uFFFE\u4100' +); +equal(UTF16Decoder.decode(new Uint8Array([0, 0, 0xff, 0xff])), '\0\uFFFF'); + +const fatal = new TextDecoder('utf-16le', { fatal: true }); +equal(fatal.fatal, true); +equal(fatal.decode(new Uint8Array([0xff, 0xfe, 0x3d, 0xd8, 0, 0xde])), '😀'); +for (const [bytes, expected] of [ + [[0x41], '\uFFFD'], + [[0x41, 0, 0x42], 'A\uFFFD'], + [[0, 0xdc], '\uFFFD'], + [[0, 0xd8], '\uFFFD'], + [[0, 0xd8, 0x41], '\uFFFD'], + [[0, 0xd8, 0x41, 0], '\uFFFDA'], + [[0, 0xd8, 0, 0xd8, 0, 0xdc], '\uFFFD𐀀'], + [[0, 0xdc, 0, 0xdc], '\uFFFD\uFFFD'], + [[0xff, 0xfe, 0, 0xd8], '\uFFFD'], +]) { + const input = new Uint8Array(bytes); + equal(UTF16Decoder.decode(input), expected); + throws(() => fatal.decode(input), TypeError); + // Independent calls reset decoding after a fatal error. + equal(fatal.decode(new Uint8Array([0x41, 0])), 'A'); +} + +equal(UTF8Decoder.decode(new Uint8Array([0xff])), '\uFFFD'); +throws( + () => new TextDecoder('utf8', { fatal: true }).decode(new Uint8Array([0xff])), + TypeError +); +// Large inputs should not hit the argument-count limit of fromCharCode(...). +const large = new Uint8Array(140000); +for (let i = 0; i < large.length; i += 2) { + large[i] = 0x41; +} +equal(UTF16Decoder.decode(large), 'A'.repeat(70000)); \ No newline at end of file diff --git a/crates/core/src/host/v8/builtins/types.d.ts b/crates/core/src/host/v8/builtins/types.d.ts index 570b92447d3..479b86ac572 100644 --- a/crates/core/src/host/v8/builtins/types.d.ts +++ b/crates/core/src/host/v8/builtins/types.d.ts @@ -1,4 +1,14 @@ declare module 'spacetime:internal_builtins' { export function utf8_encode(s: string): Uint8Array; - export function utf8_decode(s: ArrayBufferView, fatal: boolean): string; + export function utf8_decode( + buf: AllowSharedBufferSource, + ignoreBOM: boolean + ): string; + export function normalize_label(label: string): string | null; + export function generic_decode( + encoding: string, + buf: AllowSharedBufferSource, + fatal: boolean, + ignoreBOM: boolean + ): string; }