feat(ts): translate str::from_utf8 and Utf8Error

quest/m1/README.md

@@ -17,6 +17,5 @@ are folded into the [exporter's](/quest/m1/ffi-export/README.md). The
 ## Required
 
 - [Waiting: publish the UniFFI fork](/quest/m1/uniffi-release.md) - the fork publishes to crates.io under `rust2-*` names, once the maintainer decides it should
-- [Strings from UTF-8 bytes](/quest/m1/ts-utf8.md) - `str::from_utf8` translates, so the translator follows moq past its owned `Path`
 - [Exporter](/quest/m1/ffi-export/README.md) - moq's bindings exported from its crates' `pub` API, replacing the fork
 - [Land main through quest land](/quest/m1/land.md) - every queued PR lands as one tested batch through the shared gate

ts/MIGRATE.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- `rt.FromUtf8Error` holds its `rt.Utf8Error` as std's does: `e.validUpTo` and
+  `e.errorLen` are `e.error.validUpTo` and `e.error.errorLen`.
 - The runtime no longer exports `ownedNth`, `ownedNthBack`, `ownedSkip`, `ownedTake`, or
   `ownedStepBy`: `iterNth`, `iterNthBack`, `iterSkip`, `iterTake`, and `iterStepBy` take
   what drops an item last. `rt.Iter`'s `nthItem` and `nthBackItem` are `nth` and

ts/fixtures/lang/caller.ts

@@ -63,4 +63,22 @@ function wide(n: bigint): Uint8Array {
 	return bytes;
 }
 
-export const cases: Uint8Array[] = [...loops, ...durations];
+/**
+ * `utf8::decodes` of raw bytes: a byte order mark, which std keeps, and the sequences at
+ * each edge of what it takes: surrogates, overlongs, past U+10FFFF, and cut short at the
+ * end or by another byte.
+ */
+const utf8 = [
+	[0xef, 0xbb, 0xbf, 0x61],
+	[0xed, 0x9f, 0xbf, 0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf],
+	[0xc0, 0x80, 0xc1, 0xbf, 0xc2, 0x80],
+	[0xe0, 0x80, 0x80, 0xe0, 0x9f, 0xbf, 0xe0, 0xa0, 0x80],
+	[0xf0, 0x80, 0x80, 0x80, 0xf0, 0x8f, 0xbf, 0xbf, 0xf0, 0x90, 0x80, 0x80],
+	[0xf4, 0x8f, 0xbf, 0xbf, 0xf4, 0x90, 0x80, 0x80, 0xf5, 0x80, 0x80, 0x80],
+	[0xf0, 0x9f, 0x98, 0x80, 0xf0, 0x9f, 0x98, 0x41, 0xe2, 0x82, 0x41, 0xf0, 0x9f, 0x98],
+	[0xe2, 0x82],
+	[0xc2],
+	[0x80, 0xf8, 0xff],
+].map((bytes) => Uint8Array.of(57, 0, ...bytes));
+
+export const cases: Uint8Array[] = [...loops, ...durations, ...utf8];

ts/fixtures/lang/expected/harness.ts

@@ -18,6 +18,7 @@ import { refs } from "./refs.ts";
 import { copies, order, discriminants, loops, patterns, closures, library, flows } from "./shapes.ts";
 import { splits } from "./splits.ts";
 import { strs, slices } from "./strs.ts";
+import { decodes } from "./utf8.ts";
 import { wrappers } from "./wrappers.ts";
 
 /** Run one case. */
@@ -29,7 +30,7 @@ export function run(input: rt.ByteSlice): rt.ByteVec {
 	const _10: [rt.u8, rt.u8] = [input.data[input.start], input.data[input.start + 1]];
 	const op: rt.u8 = _10[0];
 	const kind: rt.u8 = _10[1];
-	const _14: rt.u8 = rt.u8Rem(op, 57);
+	const _14: rt.u8 = rt.u8Rem(op, 58);
 	if (_14 === 0) {
 		u8Op(kind, new rt.ByteSlice(input.data, input.start + 2, input.length - 2), out);
 	} else if (_14 === 1) {
@@ -159,8 +160,10 @@ export function run(input: rt.ByteSlice): rt.ByteVec {
 		skips(kind, new rt.ByteSlice(input.data, input.start + 2, input.length - 2), out);
 	} else if (_14 === 55) {
 		wrappers(kind, new rt.ByteSlice(input.data, input.start + 2, input.length - 2), out);
-	} else {
+	} else if (_14 === 56) {
 		edits(kind, new rt.ByteSlice(input.data, input.start + 2, input.length - 2), out);
+	} else {
+		decodes(kind, new rt.ByteSlice(input.data, input.start + 2, input.length - 2), out);
 	}
 	return out;
 }

ts/fixtures/lang/expected/utf8.ts

@@ -0,0 +1,50 @@
+// Generated by rust2-ts. Do not edit.
+
+import * as rt from "./rt.ts";
+import { mapOr$Option_usize_u8_consumers_closure8 } from "./core/option.ts";
+import { put } from "./fmt.ts";
+
+export function decodes(kind: rt.u8, data: rt.ByteSlice, out: rt.ByteVec): void {
+	let bytes!: rt.ByteVec;
+	if (rt.u8Rem(kind, 2) === 0) {
+		bytes = rt.byteSliceToByteVec(data);
+	} else {
+		bytes = rt.iterToByteVec(rt.iterMap(rt.iterSlice(rt.byteSliceView(data)), (_2: rt.u8): rt.u8 => {
+			const _54: rt.usize = rt.u8Rem(_2, 32) as rt.usize;
+			if (!(_54 < 32)) {
+				rt.panic("index out of bounds: the len is " + 32 + " but the index is " + _54);
+			}
+			return Uint8Array.of(0, 65, 127, 192, 193, 194, 223, 224, 225, 237, 238, 239, 240, 241, 244, 245, 128, 143, 144, 159, 160, 191, 128, 143, 144, 159, 160, 191, 128, 191, 152, 187)[_54];
+		}));
+	}
+	const iter: rt.RangeInclusive<rt.usize> = new rt.RangeInclusive(0, bytes.length, rt.usizeStep);
+	while (true) {
+		const _14: rt.usize | rt.Done = iter.next();
+		if (_14 === rt.DONE) {
+			break;
+		}
+		const _18: rt.Result<string, rt.Utf8Error> = rt.strFromUtf8(rt.byteSliceRange(rt.byteVecSlice(bytes), _14));
+		if (_18.ok) {
+			const s: string = _18.value;
+			const args2: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtStr(s, f)];
+			put(out, rt.fmtFormat([args2[0]]));
+			rt.byteVecPush(out, rt.stringLen(s) & 0xff);
+		} else {
+			const e: rt.Utf8Error = _18.error;
+			rt.byteVecPush(out, e.validUpTo & 0xff);
+			rt.byteVecPush(out, mapOr$Option_usize_u8_consumers_closure8(e.errorLen, 255, (n: rt.usize): rt.u8 => {
+				return n & 0xff;
+			}));
+			const args4: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8ErrorDisplay(e, f), (f: rt.Formatter) => rt.fmtUtf8ErrorDebug(e, f)];
+			put(out, rt.fmtFormat([args4[0], "|", rt.fmtSpec({ align: ">", width: 64 }, args4[0]), "|", args4[1], "|", rt.fmtSpec({ alternate: true }, args4[1])]));
+		}
+	}
+	const _61: rt.Result<string, rt.Utf8Error> = rt.strFromUtf8(rt.byteVecSlice(bytes));
+	if (_61.ok) {
+		rt.byteVecPush(out, rt.stringLen(_61.value) & 0xff);
+	} else {
+		rt.byteVecPush(out, _61.error.validUpTo & 0xff);
+	}
+}
+
+//# sourceMappingURL=utf8.ts.map

ts/fixtures/lang/src/harness.rs

@@ -3,7 +3,7 @@
 
 use crate::{
 	chars, chunks, collections, combinators, cows, durations, edits, fmt, generics, ints, iters, keys, nested, refs,
-	shapes, splits, strs, wrappers,
+	shapes, splits, strs, utf8, wrappers,
 };
 
 /// The native side's entry: [`run`], except that a case starting with 0xff prints a
@@ -22,7 +22,7 @@ pub fn run(input: &[u8]) -> Vec<u8> {
 		return out;
 	};
 	let (op, kind) = (*op, *kind);
-	match op % 57 {
+	match op % 58 {
 		0 => ints::u8_op(kind, data, &mut out),
 		1 => ints::u16_op(kind, data, &mut out),
 		2 => ints::u32_op(kind, data, &mut out),
@@ -89,7 +89,8 @@ pub fn run(input: &[u8]) -> Vec<u8> {
 		53 => fmt::rounding(&mut out),
 		54 => iters::skips(kind, data, &mut out),
 		55 => wrappers::wrappers(kind, data, &mut out),
-		_ => edits::edits(kind, data, &mut out),
+		56 => edits::edits(kind, data, &mut out),
+		_ => utf8::decodes(kind, data, &mut out),
 	}
 	out
 }

ts/fixtures/lang/src/lib.rs

@@ -1,9 +1,9 @@
 //! The language the driver translates, exercised against native Rust: every integer
 //! width's arithmetic and casts, `char`s and the `str` methods over them, `Cow<str>`,
 //! `Duration`, control flow, patterns, byte slices split apart, `Copy` aggregates,
-//! closures, traits, generic types, formatting, collections, `&mut` to values JS
-//! copies, the standard library calls it maps, `Option`'s and `Result`'s combinators
-//! from core's MIR, and `Option`s of values that can already be `undefined`.
+//! closures, traits, generic types, formatting, collections, UTF-8 decoding, `&mut` to
+//! values JS copies, the standard library calls it maps, `Option`'s and `Result`'s
+//! combinators from core's MIR, and `Option`s of values that can already be `undefined`.
 //!
 //! `harness::run` is the entry point both sides of the differential test run, and
 //! `fmt::Fault` an error only `caller.ts` prints.
@@ -26,6 +26,7 @@ mod refs;
 mod shapes;
 mod splits;
 mod strs;
+mod utf8;
 #[cfg(target_arch = "wasm32")]
 #[path = "../../wasm.rs"]
 mod wasm;

ts/fixtures/lang/src/utf8.rs

@@ -0,0 +1,39 @@
+//! `str::from_utf8` from every offset of its input, and the `Utf8Error` it fails with:
+//! where it stops, how many bytes are invalid there, and how it prints. Input is either
+//! raw, or bytes from an alphabet of UTF-8's boundaries, so random input makes the
+//! truncated, overlong, surrogate, and past-the-last-char sequences std refuses as often
+//! as the ones it takes.
+
+use crate::fmt::put;
+
+/// Lead bytes at each width's edges and past them, then continuation bytes at the edges
+/// of the ranges a lead byte allows after it.
+const BYTES: [u8; 32] = [
+	0x00, 0x41, 0x7f, 0xc0, 0xc1, 0xc2, 0xdf, 0xe0, 0xe1, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf4, 0xf5, 0x80, 0x8f, 0x90,
+	0x9f, 0xa0, 0xbf, 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbf, 0x80, 0xbf, 0x98, 0xbb,
+];
+
+pub fn decodes(kind: u8, data: &[u8], out: &mut Vec<u8>) {
+	let bytes: Vec<u8> = match kind % 2 {
+		0 => data.to_vec(),
+		_ => data.iter().map(|&b| BYTES[(b % 32) as usize]).collect(),
+	};
+	for start in 0..=bytes.len() {
+		match std::str::from_utf8(&bytes[start..]) {
+			Ok(s) => {
+				put(out, &format!("{s:?}"));
+				out.push(s.len() as u8);
+			}
+			Err(e) => {
+				out.push(e.valid_up_to() as u8);
+				out.push(e.error_len().map_or(0xff, |n| n as u8));
+				put(out, &format!("{e}|{e:>64}|{e:?}|{e:#?}"));
+			}
+		}
+	}
+	// `str`'s own `from_utf8`, a call of its own.
+	match <str>::from_utf8(&bytes) {
+		Ok(s) => out.push(s.len() as u8),
+		Err(e) => out.push(e.valid_up_to() as u8),
+	}
+}

ts/runtime/rt.ts

@@ -6,7 +6,7 @@
 // `writeStr`, a future's `poll`, a waker's, and `[Symbol.dispose]`.
 
 import type { Ordering } from "./rt/collections.ts";
-import { type char, type TryFromIntError, type u32, u8Debug, usizeDebug, usizeDisplay } from "./rt/int.ts";
+import { type char, type TryFromIntError, type u32, type usize, u8Debug, usizeDebug, usizeDisplay } from "./rt/int.ts";
 import type { Pending } from "./rt/task.ts";
 
 export * from "./rt/collections.ts";
@@ -701,22 +701,27 @@ const encoder = /* @__PURE__ */ new TextEncoder();
 // `ignoreBOM` keeps a leading U+FEFF, as Rust does.
 const decoder = /* @__PURE__ */ new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
 
+/** `core::str::Utf8Error`: where bytes stop being UTF-8. */
+export type Utf8Error = {
+	/** `Utf8Error::valid_up_to`. */
+	readonly validUpTo: usize;
+	/** `Utf8Error::error_len`: `undefined` if the input ended mid-character. */
+	readonly errorLen: usize | undefined;
+};
+
 /** `alloc::string::FromUtf8Error`: the bytes that were not UTF-8, and where they stop being. */
 export class FromUtf8Error {
 	readonly bytes: ByteVec;
-	/** `Utf8Error::valid_up_to`. */
-	readonly validUpTo: number;
-	/** `Utf8Error::error_len`: `undefined` if the input ended mid-character. */
-	readonly errorLen: number | undefined;
+	readonly error: Utf8Error;
 
 	constructor(bytes: ByteVec) {
 		this.bytes = bytes;
-		[this.validUpTo, this.errorLen] = utf8Error(byteVecView(bytes));
+		this.error = utf8Error(byteVecView(bytes));
 	}
 }
 
-/** Where `v` stops being UTF-8 and how many bytes are invalid there, as std reports it. */
-function utf8Error(v: Uint8Array): [number, number | undefined] {
+/** Where `v`, which is not UTF-8, stops being so, and how many bytes are invalid there, as std reports it. */
+function utf8Error(v: Uint8Array): Utf8Error {
 	const tail = (b: number | undefined) => b !== undefined && b >= 0x80 && b <= 0xbf;
 	for (let i = 0; i < v.length; ) {
 		const first = v[i];
@@ -725,21 +730,31 @@ function utf8Error(v: Uint8Array): [number, number | undefined] {
 			i++;
 			continue;
 		}
-		if (width === 0) return [i, 1];
+		if (width === 0) return { validUpTo: i, errorLen: 1 };
 		const second = v[i + 1];
-		if (second === undefined) return [i, undefined];
+		if (second === undefined) return { validUpTo: i, errorLen: undefined };
 		const [lo, hi] =
 			first === 0xe0 ? [0xa0, 0xbf] : first === 0xed ? [0x80, 0x9f] : first === 0xf0 ? [0x90, 0xbf] : first === 0xf4 ? [0x80, 0x8f] : [0x80, 0xbf];
-		if (second < lo || second > hi) return [i, 1];
+		if (second < lo || second > hi) return { validUpTo: i, errorLen: 1 };
 		for (let k = 2; k < width; k++) {
-			if (i + k >= v.length) return [i, undefined];
-			if (!tail(v[i + k])) return [i, k];
+			if (i + k >= v.length) return { validUpTo: i, errorLen: undefined };
+			if (!tail(v[i + k])) return { validUpTo: i, errorLen: k };
 		}
 		i += width;
 	}
 	throw new Error("utf8Error: the bytes are valid UTF-8");
 }
 
+/** `str::from_utf8`. */
+export function strFromUtf8(bytes: ByteSlice): Result<string, Utf8Error> {
+	const v = byteSliceView(bytes);
+	try {
+		return Ok(decoder.decode(v));
+	} catch {
+		return Err(utf8Error(v));
+	}
+}
+
 /** `String::from_utf8`. */
 export function stringFromUtf8(bytes: ByteVec): Result<string, FromUtf8Error> {
 	try {
@@ -1243,28 +1258,36 @@ export function fmtTryFromIntErrorDisplay(e: TryFromIntError, f: Formatter): Fmt
 	return fmtPad(f, `number too ${positive ? "large" : "small"} to fit in target type`);
 }
 
+/** `Debug` of `Utf8Error`. */
+export function fmtUtf8ErrorDebug(e: Utf8Error, f: Formatter): FmtResult {
+	return debugStructFinish(
+		debugStructField(
+			debugStructField(fmtDebugStruct(f, "Utf8Error"), "valid_up_to", (f) => usizeDebug(e.validUpTo, f)),
+			"error_len",
+			(f) => fmtOption(e.errorLen, f, usizeDebug),
+		),
+	);
+}
+
+/** `Display` of `Utf8Error`, which `write!`s and so ignores the formatter's options. */
+export function fmtUtf8ErrorDisplay(e: Utf8Error, f: Formatter): FmtResult {
+	if (e.errorLen === undefined) return f.out.writeStr(`incomplete utf-8 byte sequence from index ${e.validUpTo}`);
+	return f.out.writeStr(`invalid utf-8 sequence of ${e.errorLen} bytes from index ${e.validUpTo}`);
+}
+
 /** `Debug` of `FromUtf8Error`. */
 export function fmtFromUtf8ErrorDebug(e: FromUtf8Error, f: Formatter): FmtResult {
-	const error: Fmt = (f) =>
-		debugStructFinish(
-			debugStructField(
-				debugStructField(fmtDebugStruct(f, "Utf8Error"), "valid_up_to", (f) => usizeDebug(e.validUpTo, f)),
-				"error_len",
-				(f) => fmtOption(e.errorLen, f, u8Debug),
-			),
-		);
 	return debugStructFinish(
 		debugStructField(
 			debugStructField(fmtDebugStruct(f, "FromUtf8Error"), "bytes", (f) => fmtList(byteVecView(e.bytes), f, u8Debug)),
 			"error",
-			error,
+			(f) => fmtUtf8ErrorDebug(e.error, f),
 		),
 	);
 }
 
-/** `Display` of the error, which `write!`s and so ignores the formatter's options. */
+/** `Display` of `FromUtf8Error`: its `Utf8Error`'s. */
 export function fmtFromUtf8ErrorDisplay(e: FromUtf8Error, f: Formatter): FmtResult {
-	if (e.errorLen === undefined) return f.out.writeStr(`incomplete utf-8 byte sequence from index ${e.validUpTo}`);
-	return f.out.writeStr(`invalid utf-8 sequence of ${e.errorLen} bytes from index ${e.validUpTo}`);
+	return fmtUtf8ErrorDisplay(e.error, f);
 }
 

ts/src/lower/fmt.rs

@@ -297,6 +297,8 @@ pub fn std_impl<'tcx>(
 			(Std::TryFromIntError, Trait::Display) => rt("tryFromIntErrorDisplay", vec![value, f]),
 			(Std::FromUtf8Error, Trait::Debug) => rt("fromUtf8ErrorDebug", vec![value, f]),
 			(Std::FromUtf8Error, Trait::Display) => rt("fromUtf8ErrorDisplay", vec![value, f]),
+			(Std::Utf8Error, Trait::Debug) => rt("utf8ErrorDebug", vec![value, f]),
+			(Std::Utf8Error, Trait::Display) => rt("utf8ErrorDisplay", vec![value, f]),
 			(Std::Duration, Trait::Debug) => Expr::rt("durationDebug", vec![value, f]),
 			_ => return None,
 		},

ts/src/lower/library.rs

@@ -32,6 +32,7 @@ pub enum Std {
 	/// Its contents.
 	Box,
 	FromUtf8Error,
+	Utf8Error,
 	TryFromIntError,
 	/// `rt.Formatter`.
 	Formatter,
@@ -176,6 +177,7 @@ impl Std {
 			.collect::<Vec<_>>()
 			.as_slice()
 		{
+			["str", "error", "Utf8Error"] => Some(Self::Utf8Error),
 			["fmt", "Formatter"] => Some(Self::Formatter),
 			["fmt", "Arguments"] => Some(Self::Arguments),
 			["fmt", "Error"] => Some(Self::FmtError),
@@ -1273,6 +1275,10 @@ fn mapping<'tcx>(
 		// Strings.
 		("String" | "str", "as_bytes") => Expr::rt("stringAsBytes", args),
 		("String", "from_utf8") => Expr::rt("stringFromUtf8", args),
+		// `str::from_utf8` is `core::str::converts::from_utf8`, or `str`'s own.
+		("converts" | "str", "from_utf8") => Expr::rt("strFromUtf8", args),
+		("Utf8Error", "valid_up_to") => args.pop()?.member("validUpTo"),
+		("Utf8Error", "error_len") => args.pop()?.member("errorLen"),
 		("String" | "str", "len") => Expr::rt("stringLen", args),
 		("String", "as_str") => args.pop()?,
 		("String", "new") => Expr::Str(String::new()),