diff --git a/quest/m1/README.md b/quest/m1/README.md index 78e781cf0..f442f6c3a 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -17,6 +17,5 @@ are folded into the [exporter's](/quest/m1/ffi-export/README.md). The ## Required - [Waiting: publish the UniFFI fork](/quest/m1/uniffi-release.md) - the fork publishes to crates.io under `rust2-*` names, once the maintainer decides it should -- [Strings from UTF-8 bytes](/quest/m1/ts-utf8.md) - `str::from_utf8` translates, so the translator follows moq past its owned `Path` - [Exporter](/quest/m1/ffi-export/README.md) - moq's bindings exported from its crates' `pub` API, replacing the fork - [Land main through quest land](/quest/m1/land.md) - every queued PR lands as one tested batch through the shared gate diff --git a/quest/m1/ts-utf8.md b/quest/m1/ts-utf8.md deleted file mode 100644 index 230e934d4..000000000 --- a/quest/m1/ts-utf8.md +++ /dev/null @@ -1,20 +0,0 @@ -# [S] Strings from UTF-8 bytes - -## Goal - -[rust2-ts](/ts/README.md) translates `std::str::from_utf8` and `Utf8Error` (its -`valid_up_to`, `error_len`, `Display`, and `Debug`), with std's results on invalid input, -proven by differential tests against native Rust. `just ts moq` passes at the moq revision -past [moq-dev/moq#6843](https://github.com/moq-dev/moq/pull/6843), whose decoder calls it. - -## Plan - -Decided with the maintainer on 2026-10-27: the exporter and the translator share one moq -pin, so [the exporter's pin bump](https://github.com/kixelated/rust2/pull/144) merges once -this passes at its revision. - -- `TextDecoder` with `fatal` rejects what std rejects, but the error position must be std's - `valid_up_to` and `error_len`, so check its edge cases (truncated sequences, surrogates, - overlongs) against native. - -Public API: the translator's output. diff --git a/ts/MIGRATE.md b/ts/MIGRATE.md index 563cd4a88..80d157df9 100644 --- a/ts/MIGRATE.md +++ b/ts/MIGRATE.md @@ -2,6 +2,8 @@ ## Unreleased +- `rt.FromUtf8Error` holds its `rt.Utf8Error` as std's does: `e.validUpTo` and + `e.errorLen` are `e.error.validUpTo` and `e.error.errorLen`. - The runtime no longer exports `ownedNth`, `ownedNthBack`, `ownedSkip`, `ownedTake`, or `ownedStepBy`: `iterNth`, `iterNthBack`, `iterSkip`, `iterTake`, and `iterStepBy` take what drops an item last. `rt.Iter`'s `nthItem` and `nthBackItem` are `nth` and diff --git a/ts/fixtures/lang/caller.ts b/ts/fixtures/lang/caller.ts index 6d81f0c79..862f3bb7d 100644 --- a/ts/fixtures/lang/caller.ts +++ b/ts/fixtures/lang/caller.ts @@ -63,4 +63,22 @@ function wide(n: bigint): Uint8Array { return bytes; } -export const cases: Uint8Array[] = [...loops, ...durations]; +/** + * `utf8::decodes` of raw bytes: a byte order mark, which std keeps, and the sequences at + * each edge of what it takes: surrogates, overlongs, past U+10FFFF, and cut short at the + * end or by another byte. + */ +const utf8 = [ + [0xef, 0xbb, 0xbf, 0x61], + [0xed, 0x9f, 0xbf, 0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf], + [0xc0, 0x80, 0xc1, 0xbf, 0xc2, 0x80], + [0xe0, 0x80, 0x80, 0xe0, 0x9f, 0xbf, 0xe0, 0xa0, 0x80], + [0xf0, 0x80, 0x80, 0x80, 0xf0, 0x8f, 0xbf, 0xbf, 0xf0, 0x90, 0x80, 0x80], + [0xf4, 0x8f, 0xbf, 0xbf, 0xf4, 0x90, 0x80, 0x80, 0xf5, 0x80, 0x80, 0x80], + [0xf0, 0x9f, 0x98, 0x80, 0xf0, 0x9f, 0x98, 0x41, 0xe2, 0x82, 0x41, 0xf0, 0x9f, 0x98], + [0xe2, 0x82], + [0xc2], + [0x80, 0xf8, 0xff], +].map((bytes) => Uint8Array.of(57, 0, ...bytes)); + +export const cases: Uint8Array[] = [...loops, ...durations, ...utf8]; diff --git a/ts/fixtures/lang/expected/harness.ts b/ts/fixtures/lang/expected/harness.ts index cb8e4ee7f..6729b1287 100644 --- a/ts/fixtures/lang/expected/harness.ts +++ b/ts/fixtures/lang/expected/harness.ts @@ -18,6 +18,7 @@ import { refs } from "./refs.ts"; import { copies, order, discriminants, loops, patterns, closures, library, flows } from "./shapes.ts"; import { splits } from "./splits.ts"; import { strs, slices } from "./strs.ts"; +import { decodes } from "./utf8.ts"; import { wrappers } from "./wrappers.ts"; /** Run one case. */ @@ -29,7 +30,7 @@ export function run(input: rt.ByteSlice): rt.ByteVec { const _10: [rt.u8, rt.u8] = [input.data[input.start], input.data[input.start + 1]]; const op: rt.u8 = _10[0]; const kind: rt.u8 = _10[1]; - const _14: rt.u8 = rt.u8Rem(op, 57); + const _14: rt.u8 = rt.u8Rem(op, 58); if (_14 === 0) { u8Op(kind, new rt.ByteSlice(input.data, input.start + 2, input.length - 2), out); } else if (_14 === 1) { @@ -159,8 +160,10 @@ export function run(input: rt.ByteSlice): rt.ByteVec { skips(kind, new rt.ByteSlice(input.data, input.start + 2, input.length - 2), out); } else if (_14 === 55) { wrappers(kind, new rt.ByteSlice(input.data, input.start + 2, input.length - 2), out); - } else { + } else if (_14 === 56) { edits(kind, new rt.ByteSlice(input.data, input.start + 2, input.length - 2), out); + } else { + decodes(kind, new rt.ByteSlice(input.data, input.start + 2, input.length - 2), out); } return out; } diff --git a/ts/fixtures/lang/expected/utf8.ts b/ts/fixtures/lang/expected/utf8.ts new file mode 100644 index 000000000..a210d4104 --- /dev/null +++ b/ts/fixtures/lang/expected/utf8.ts @@ -0,0 +1,50 @@ +// Generated by rust2-ts. Do not edit. + +import * as rt from "./rt.ts"; +import { mapOr$Option_usize_u8_consumers_closure8 } from "./core/option.ts"; +import { put } from "./fmt.ts"; + +export function decodes(kind: rt.u8, data: rt.ByteSlice, out: rt.ByteVec): void { + let bytes!: rt.ByteVec; + if (rt.u8Rem(kind, 2) === 0) { + bytes = rt.byteSliceToByteVec(data); + } else { + bytes = rt.iterToByteVec(rt.iterMap(rt.iterSlice(rt.byteSliceView(data)), (_2: rt.u8): rt.u8 => { + const _54: rt.usize = rt.u8Rem(_2, 32) as rt.usize; + if (!(_54 < 32)) { + rt.panic("index out of bounds: the len is " + 32 + " but the index is " + _54); + } + return Uint8Array.of(0, 65, 127, 192, 193, 194, 223, 224, 225, 237, 238, 239, 240, 241, 244, 245, 128, 143, 144, 159, 160, 191, 128, 143, 144, 159, 160, 191, 128, 191, 152, 187)[_54]; + })); + } + const iter: rt.RangeInclusive = new rt.RangeInclusive(0, bytes.length, rt.usizeStep); + while (true) { + const _14: rt.usize | rt.Done = iter.next(); + if (_14 === rt.DONE) { + break; + } + const _18: rt.Result = rt.strFromUtf8(rt.byteSliceRange(rt.byteVecSlice(bytes), _14)); + if (_18.ok) { + const s: string = _18.value; + const args2: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtStr(s, f)]; + put(out, rt.fmtFormat([args2[0]])); + rt.byteVecPush(out, rt.stringLen(s) & 0xff); + } else { + const e: rt.Utf8Error = _18.error; + rt.byteVecPush(out, e.validUpTo & 0xff); + rt.byteVecPush(out, mapOr$Option_usize_u8_consumers_closure8(e.errorLen, 255, (n: rt.usize): rt.u8 => { + return n & 0xff; + })); + const args4: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8ErrorDisplay(e, f), (f: rt.Formatter) => rt.fmtUtf8ErrorDebug(e, f)]; + put(out, rt.fmtFormat([args4[0], "|", rt.fmtSpec({ align: ">", width: 64 }, args4[0]), "|", args4[1], "|", rt.fmtSpec({ alternate: true }, args4[1])])); + } + } + const _61: rt.Result = rt.strFromUtf8(rt.byteVecSlice(bytes)); + if (_61.ok) { + rt.byteVecPush(out, rt.stringLen(_61.value) & 0xff); + } else { + rt.byteVecPush(out, _61.error.validUpTo & 0xff); + } +} + +//# sourceMappingURL=utf8.ts.map diff --git a/ts/fixtures/lang/src/harness.rs b/ts/fixtures/lang/src/harness.rs index a7044df81..10000259d 100644 --- a/ts/fixtures/lang/src/harness.rs +++ b/ts/fixtures/lang/src/harness.rs @@ -3,7 +3,7 @@ use crate::{ chars, chunks, collections, combinators, cows, durations, edits, fmt, generics, ints, iters, keys, nested, refs, - shapes, splits, strs, wrappers, + shapes, splits, strs, utf8, wrappers, }; /// The native side's entry: [`run`], except that a case starting with 0xff prints a @@ -22,7 +22,7 @@ pub fn run(input: &[u8]) -> Vec { return out; }; let (op, kind) = (*op, *kind); - match op % 57 { + match op % 58 { 0 => ints::u8_op(kind, data, &mut out), 1 => ints::u16_op(kind, data, &mut out), 2 => ints::u32_op(kind, data, &mut out), @@ -89,7 +89,8 @@ pub fn run(input: &[u8]) -> Vec { 53 => fmt::rounding(&mut out), 54 => iters::skips(kind, data, &mut out), 55 => wrappers::wrappers(kind, data, &mut out), - _ => edits::edits(kind, data, &mut out), + 56 => edits::edits(kind, data, &mut out), + _ => utf8::decodes(kind, data, &mut out), } out } diff --git a/ts/fixtures/lang/src/lib.rs b/ts/fixtures/lang/src/lib.rs index d39396e18..2c15ee4e8 100644 --- a/ts/fixtures/lang/src/lib.rs +++ b/ts/fixtures/lang/src/lib.rs @@ -1,9 +1,9 @@ //! The language the driver translates, exercised against native Rust: every integer //! width's arithmetic and casts, `char`s and the `str` methods over them, `Cow`, //! `Duration`, control flow, patterns, byte slices split apart, `Copy` aggregates, -//! closures, traits, generic types, formatting, collections, `&mut` to values JS -//! copies, the standard library calls it maps, `Option`'s and `Result`'s combinators -//! from core's MIR, and `Option`s of values that can already be `undefined`. +//! closures, traits, generic types, formatting, collections, UTF-8 decoding, `&mut` to +//! values JS copies, the standard library calls it maps, `Option`'s and `Result`'s +//! combinators from core's MIR, and `Option`s of values that can already be `undefined`. //! //! `harness::run` is the entry point both sides of the differential test run, and //! `fmt::Fault` an error only `caller.ts` prints. @@ -26,6 +26,7 @@ mod refs; mod shapes; mod splits; mod strs; +mod utf8; #[cfg(target_arch = "wasm32")] #[path = "../../wasm.rs"] mod wasm; diff --git a/ts/fixtures/lang/src/utf8.rs b/ts/fixtures/lang/src/utf8.rs new file mode 100644 index 000000000..31283b38c --- /dev/null +++ b/ts/fixtures/lang/src/utf8.rs @@ -0,0 +1,39 @@ +//! `str::from_utf8` from every offset of its input, and the `Utf8Error` it fails with: +//! where it stops, how many bytes are invalid there, and how it prints. Input is either +//! raw, or bytes from an alphabet of UTF-8's boundaries, so random input makes the +//! truncated, overlong, surrogate, and past-the-last-char sequences std refuses as often +//! as the ones it takes. + +use crate::fmt::put; + +/// Lead bytes at each width's edges and past them, then continuation bytes at the edges +/// of the ranges a lead byte allows after it. +const BYTES: [u8; 32] = [ + 0x00, 0x41, 0x7f, 0xc0, 0xc1, 0xc2, 0xdf, 0xe0, 0xe1, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf4, 0xf5, 0x80, 0x8f, 0x90, + 0x9f, 0xa0, 0xbf, 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbf, 0x80, 0xbf, 0x98, 0xbb, +]; + +pub fn decodes(kind: u8, data: &[u8], out: &mut Vec) { + let bytes: Vec = match kind % 2 { + 0 => data.to_vec(), + _ => data.iter().map(|&b| BYTES[(b % 32) as usize]).collect(), + }; + for start in 0..=bytes.len() { + match std::str::from_utf8(&bytes[start..]) { + Ok(s) => { + put(out, &format!("{s:?}")); + out.push(s.len() as u8); + } + Err(e) => { + out.push(e.valid_up_to() as u8); + out.push(e.error_len().map_or(0xff, |n| n as u8)); + put(out, &format!("{e}|{e:>64}|{e:?}|{e:#?}")); + } + } + } + // `str`'s own `from_utf8`, a call of its own. + match ::from_utf8(&bytes) { + Ok(s) => out.push(s.len() as u8), + Err(e) => out.push(e.valid_up_to() as u8), + } +} diff --git a/ts/runtime/rt.ts b/ts/runtime/rt.ts index 8ab3185f0..5b76341ce 100644 --- a/ts/runtime/rt.ts +++ b/ts/runtime/rt.ts @@ -6,7 +6,7 @@ // `writeStr`, a future's `poll`, a waker's, and `[Symbol.dispose]`. import type { Ordering } from "./rt/collections.ts"; -import { type char, type TryFromIntError, type u32, u8Debug, usizeDebug, usizeDisplay } from "./rt/int.ts"; +import { type char, type TryFromIntError, type u32, type usize, u8Debug, usizeDebug, usizeDisplay } from "./rt/int.ts"; import type { Pending } from "./rt/task.ts"; export * from "./rt/collections.ts"; @@ -701,22 +701,27 @@ const encoder = /* @__PURE__ */ new TextEncoder(); // `ignoreBOM` keeps a leading U+FEFF, as Rust does. const decoder = /* @__PURE__ */ new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +/** `core::str::Utf8Error`: where bytes stop being UTF-8. */ +export type Utf8Error = { + /** `Utf8Error::valid_up_to`. */ + readonly validUpTo: usize; + /** `Utf8Error::error_len`: `undefined` if the input ended mid-character. */ + readonly errorLen: usize | undefined; +}; + /** `alloc::string::FromUtf8Error`: the bytes that were not UTF-8, and where they stop being. */ export class FromUtf8Error { readonly bytes: ByteVec; - /** `Utf8Error::valid_up_to`. */ - readonly validUpTo: number; - /** `Utf8Error::error_len`: `undefined` if the input ended mid-character. */ - readonly errorLen: number | undefined; + readonly error: Utf8Error; constructor(bytes: ByteVec) { this.bytes = bytes; - [this.validUpTo, this.errorLen] = utf8Error(byteVecView(bytes)); + this.error = utf8Error(byteVecView(bytes)); } } -/** Where `v` stops being UTF-8 and how many bytes are invalid there, as std reports it. */ -function utf8Error(v: Uint8Array): [number, number | undefined] { +/** Where `v`, which is not UTF-8, stops being so, and how many bytes are invalid there, as std reports it. */ +function utf8Error(v: Uint8Array): Utf8Error { const tail = (b: number | undefined) => b !== undefined && b >= 0x80 && b <= 0xbf; for (let i = 0; i < v.length; ) { const first = v[i]; @@ -725,21 +730,31 @@ function utf8Error(v: Uint8Array): [number, number | undefined] { i++; continue; } - if (width === 0) return [i, 1]; + if (width === 0) return { validUpTo: i, errorLen: 1 }; const second = v[i + 1]; - if (second === undefined) return [i, undefined]; + if (second === undefined) return { validUpTo: i, errorLen: undefined }; const [lo, hi] = first === 0xe0 ? [0xa0, 0xbf] : first === 0xed ? [0x80, 0x9f] : first === 0xf0 ? [0x90, 0xbf] : first === 0xf4 ? [0x80, 0x8f] : [0x80, 0xbf]; - if (second < lo || second > hi) return [i, 1]; + if (second < lo || second > hi) return { validUpTo: i, errorLen: 1 }; for (let k = 2; k < width; k++) { - if (i + k >= v.length) return [i, undefined]; - if (!tail(v[i + k])) return [i, k]; + if (i + k >= v.length) return { validUpTo: i, errorLen: undefined }; + if (!tail(v[i + k])) return { validUpTo: i, errorLen: k }; } i += width; } throw new Error("utf8Error: the bytes are valid UTF-8"); } +/** `str::from_utf8`. */ +export function strFromUtf8(bytes: ByteSlice): Result { + const v = byteSliceView(bytes); + try { + return Ok(decoder.decode(v)); + } catch { + return Err(utf8Error(v)); + } +} + /** `String::from_utf8`. */ export function stringFromUtf8(bytes: ByteVec): Result { try { @@ -1243,28 +1258,36 @@ export function fmtTryFromIntErrorDisplay(e: TryFromIntError, f: Formatter): Fmt return fmtPad(f, `number too ${positive ? "large" : "small"} to fit in target type`); } +/** `Debug` of `Utf8Error`. */ +export function fmtUtf8ErrorDebug(e: Utf8Error, f: Formatter): FmtResult { + return debugStructFinish( + debugStructField( + debugStructField(fmtDebugStruct(f, "Utf8Error"), "valid_up_to", (f) => usizeDebug(e.validUpTo, f)), + "error_len", + (f) => fmtOption(e.errorLen, f, usizeDebug), + ), + ); +} + +/** `Display` of `Utf8Error`, which `write!`s and so ignores the formatter's options. */ +export function fmtUtf8ErrorDisplay(e: Utf8Error, f: Formatter): FmtResult { + if (e.errorLen === undefined) return f.out.writeStr(`incomplete utf-8 byte sequence from index ${e.validUpTo}`); + return f.out.writeStr(`invalid utf-8 sequence of ${e.errorLen} bytes from index ${e.validUpTo}`); +} + /** `Debug` of `FromUtf8Error`. */ export function fmtFromUtf8ErrorDebug(e: FromUtf8Error, f: Formatter): FmtResult { - const error: Fmt = (f) => - debugStructFinish( - debugStructField( - debugStructField(fmtDebugStruct(f, "Utf8Error"), "valid_up_to", (f) => usizeDebug(e.validUpTo, f)), - "error_len", - (f) => fmtOption(e.errorLen, f, u8Debug), - ), - ); return debugStructFinish( debugStructField( debugStructField(fmtDebugStruct(f, "FromUtf8Error"), "bytes", (f) => fmtList(byteVecView(e.bytes), f, u8Debug)), "error", - error, + (f) => fmtUtf8ErrorDebug(e.error, f), ), ); } -/** `Display` of the error, which `write!`s and so ignores the formatter's options. */ +/** `Display` of `FromUtf8Error`: its `Utf8Error`'s. */ export function fmtFromUtf8ErrorDisplay(e: FromUtf8Error, f: Formatter): FmtResult { - if (e.errorLen === undefined) return f.out.writeStr(`incomplete utf-8 byte sequence from index ${e.validUpTo}`); - return f.out.writeStr(`invalid utf-8 sequence of ${e.errorLen} bytes from index ${e.validUpTo}`); + return fmtUtf8ErrorDisplay(e.error, f); } diff --git a/ts/src/lower/fmt.rs b/ts/src/lower/fmt.rs index b90c4c693..444a602a5 100644 --- a/ts/src/lower/fmt.rs +++ b/ts/src/lower/fmt.rs @@ -297,6 +297,8 @@ pub fn std_impl<'tcx>( (Std::TryFromIntError, Trait::Display) => rt("tryFromIntErrorDisplay", vec![value, f]), (Std::FromUtf8Error, Trait::Debug) => rt("fromUtf8ErrorDebug", vec![value, f]), (Std::FromUtf8Error, Trait::Display) => rt("fromUtf8ErrorDisplay", vec![value, f]), + (Std::Utf8Error, Trait::Debug) => rt("utf8ErrorDebug", vec![value, f]), + (Std::Utf8Error, Trait::Display) => rt("utf8ErrorDisplay", vec![value, f]), (Std::Duration, Trait::Debug) => Expr::rt("durationDebug", vec![value, f]), _ => return None, }, diff --git a/ts/src/lower/library.rs b/ts/src/lower/library.rs index c4e1ddd36..7cd61801b 100644 --- a/ts/src/lower/library.rs +++ b/ts/src/lower/library.rs @@ -32,6 +32,7 @@ pub enum Std { /// Its contents. Box, FromUtf8Error, + Utf8Error, TryFromIntError, /// `rt.Formatter`. Formatter, @@ -176,6 +177,7 @@ impl Std { .collect::>() .as_slice() { + ["str", "error", "Utf8Error"] => Some(Self::Utf8Error), ["fmt", "Formatter"] => Some(Self::Formatter), ["fmt", "Arguments"] => Some(Self::Arguments), ["fmt", "Error"] => Some(Self::FmtError), @@ -1273,6 +1275,10 @@ fn mapping<'tcx>( // Strings. ("String" | "str", "as_bytes") => Expr::rt("stringAsBytes", args), ("String", "from_utf8") => Expr::rt("stringFromUtf8", args), + // `str::from_utf8` is `core::str::converts::from_utf8`, or `str`'s own. + ("converts" | "str", "from_utf8") => Expr::rt("strFromUtf8", args), + ("Utf8Error", "valid_up_to") => args.pop()?.member("validUpTo"), + ("Utf8Error", "error_len") => args.pop()?.member("errorLen"), ("String" | "str", "len") => Expr::rt("stringLen", args), ("String", "as_str") => args.pop()?, ("String", "new") => Expr::Str(String::new()), diff --git a/ts/src/lower/mod.rs b/ts/src/lower/mod.rs index cd581f010..f80537383 100644 --- a/ts/src/lower/mod.rs +++ b/ts/src/lower/mod.rs @@ -1194,6 +1194,7 @@ impl<'a, 'tcx> Translator<'a, 'tcx> { library::Std::Box | library::Std::Reverse | library::Std::NonZero => self.ty(args.type_at(0), span), library::Std::Bound => Type::Rt("RangeBound", vec![self.ty(args.type_at(0), span)]), library::Std::FromUtf8Error => Type::Rt("FromUtf8Error", vec![]), + library::Std::Utf8Error => Type::Rt("Utf8Error", vec![]), library::Std::TryFromIntError => Type::Rt("TryFromIntError", vec![]), library::Std::Formatter => Type::Rt("Formatter", vec![]), library::Std::Arguments => Type::Rt("Arguments", vec![]),