From 53a9b81d2b27e1d773ff5be82cdd32b9168e2979 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:56:17 -0700 Subject: [PATCH 1/9] feat: advance CPython Lib/test conformance wave 2 --- Cargo.lock | 1 + crates/weavepy-cli/src/main.rs | 29 + crates/weavepy-cli/src/repl.rs | 5 + crates/weavepy-compiler/src/bytecode.rs | 38 + crates/weavepy-compiler/src/cpython_code.rs | 2 + crates/weavepy-compiler/src/lib.rs | 429 +- crates/weavepy-conformance/src/bin/main.rs | 24 + crates/weavepy-conformance/src/regrtest.rs | 43 +- crates/weavepy-lexer/src/error.rs | 51 +- crates/weavepy-lexer/src/lib.rs | 137 +- crates/weavepy-lexer/src/scanner.rs | 452 +- crates/weavepy-lexer/src/token.rs | 26 +- crates/weavepy-parser/Cargo.toml | 4 + crates/weavepy-parser/src/error.rs | 27 + crates/weavepy-parser/src/lib.rs | 21 +- crates/weavepy-parser/src/parser.rs | 635 +- crates/weavepy-vm/src/builtin_types.rs | 699 +- crates/weavepy-vm/src/builtins.rs | 2233 +++++- crates/weavepy-vm/src/error.rs | 86 + crates/weavepy-vm/src/gc_trace.rs | 12 +- crates/weavepy-vm/src/lib.rs | 4958 +++++++++++-- crates/weavepy-vm/src/object.rs | 668 +- crates/weavepy-vm/src/recursion.rs | 169 + crates/weavepy-vm/src/specialize.rs | 10 +- crates/weavepy-vm/src/stdlib/datetime_mod.rs | 8 +- crates/weavepy-vm/src/stdlib/io.rs | 314 +- crates/weavepy-vm/src/stdlib/math.rs | 231 +- crates/weavepy-vm/src/stdlib/mod.rs | 144 + crates/weavepy-vm/src/stdlib/os.rs | 72 + .../src/stdlib/python/_collections_abc.py | 1195 ++++ .../src/stdlib/python/_collections_user.py | 491 ++ .../weavepy-vm/src/stdlib/python/_colorize.py | 119 + .../weavepy-vm/src/stdlib/python/_py_abc.py | 147 + .../src/stdlib/python/_pydecimal.py | 6351 +++++++++++++++++ .../src/stdlib/python/_weakrefset.py | 205 + crates/weavepy-vm/src/stdlib/python/abc.py | 261 +- .../weavepy-vm/src/stdlib/python/argparse.py | 78 +- .../weavepy-vm/src/stdlib/python/array_mod.py | 6 + .../weavepy-vm/src/stdlib/python/bdb_mod.py | 1012 ++- .../weavepy-vm/src/stdlib/python/calendar.py | 813 +++ crates/weavepy-vm/src/stdlib/python/cmath.py | 174 + .../src/stdlib/python/collections.py | 32 +- .../src/stdlib/python/collections_abc.py | 10 + .../src/stdlib/python/contextlib.py | 813 ++- .../weavepy-vm/src/stdlib/python/copy_mod.py | 235 +- .../weavepy-vm/src/stdlib/python/copyreg.py | 119 +- .../src/stdlib/python/dataclasses.py | 32 +- .../weavepy-vm/src/stdlib/python/decimal.py | 622 +- .../weavepy-vm/src/stdlib/python/doctest.py | 7 +- crates/weavepy-vm/src/stdlib/python/enum.py | 28 + .../weavepy-vm/src/stdlib/python/fractions.py | 1226 +++- .../weavepy-vm/src/stdlib/python/functools.py | 198 +- .../src/stdlib/python/future_module.py | 147 + crates/weavepy-vm/src/stdlib/python/html.py | 171 +- .../src/stdlib/python/html_entities.py | 2513 +++++++ .../weavepy-vm/src/stdlib/python/inspect.py | 51 +- crates/weavepy-vm/src/stdlib/python/locale.py | 186 + .../weavepy-vm/src/stdlib/python/mimetypes.py | 842 ++- .../src/stdlib/python/numbers_mod.py | 12 +- .../src/stdlib/python/operator_mod.py | 419 +- crates/weavepy-vm/src/stdlib/python/pickle.py | 26 +- .../weavepy-vm/src/stdlib/python/reprlib.py | 230 + crates/weavepy-vm/src/stdlib/python/runpy.py | 48 +- crates/weavepy-vm/src/stdlib/python/struct.py | 58 +- .../src/stdlib/python/test_list_tests.py | 577 ++ .../src/stdlib/python/test_pickletester.py | 31 + .../src/stdlib/python/test_seq_tests.py | 445 ++ .../src/stdlib/python/test_string_tests.py | 1598 +++++ .../python/test_support_hashlib_helper.py | 51 + .../stdlib/python/test_support_i18n_helper.py | 22 + .../src/stdlib/python/test_support_init.py | 385 + .../test_tokenizedata_badsyntax_3131.py | 2 + .../stdlib/python/test_tokenizedata_init.py | 6 + crates/weavepy-vm/src/stdlib/python/typing.py | 315 +- .../src/stdlib/python/unittest_mock.py | 17 +- .../weavepy-vm/src/stdlib/python/weakref.py | 38 + crates/weavepy-vm/src/stdlib/random.rs | 136 +- crates/weavepy-vm/src/stdlib/struct_mod.rs | 102 +- crates/weavepy-vm/src/stdlib/sys.rs | 156 +- crates/weavepy-vm/src/stdlib/thread_real.rs | 7 + crates/weavepy-vm/src/stdlib/weakref_real.rs | 32 +- crates/weavepy-vm/src/types.rs | 41 + crates/weavepy-vm/src/weakref_registry.rs | 27 + crates/weavepy/src/lib.rs | 12 +- ...ython-lib-test-conformance-sweep-wave-2.md | 430 ++ tests/regrtest/expectations.toml | 132 +- tests/regrtest/test_control_flow.py | 133 + tests/regrtest/test_numeric_string_format.py | 31 + tests/regrtest/test_pdb_bdb_dropin.py | 34 +- tests/regrtest/test_recursion_guard.py | 104 + tests/regrtest/test_rfc0037_dropin.py | 172 + tests/regrtest/test_strings.py | 52 + 92 files changed, 32361 insertions(+), 3132 deletions(-) create mode 100644 crates/weavepy-vm/src/recursion.rs create mode 100644 crates/weavepy-vm/src/stdlib/python/_collections_abc.py create mode 100644 crates/weavepy-vm/src/stdlib/python/_collections_user.py create mode 100644 crates/weavepy-vm/src/stdlib/python/_colorize.py create mode 100644 crates/weavepy-vm/src/stdlib/python/_py_abc.py create mode 100644 crates/weavepy-vm/src/stdlib/python/_pydecimal.py create mode 100644 crates/weavepy-vm/src/stdlib/python/_weakrefset.py create mode 100644 crates/weavepy-vm/src/stdlib/python/calendar.py create mode 100644 crates/weavepy-vm/src/stdlib/python/cmath.py create mode 100644 crates/weavepy-vm/src/stdlib/python/collections_abc.py create mode 100644 crates/weavepy-vm/src/stdlib/python/future_module.py create mode 100644 crates/weavepy-vm/src/stdlib/python/html_entities.py create mode 100644 crates/weavepy-vm/src/stdlib/python/locale.py create mode 100644 crates/weavepy-vm/src/stdlib/python/reprlib.py create mode 100644 crates/weavepy-vm/src/stdlib/python/test_list_tests.py create mode 100644 crates/weavepy-vm/src/stdlib/python/test_pickletester.py create mode 100644 crates/weavepy-vm/src/stdlib/python/test_seq_tests.py create mode 100644 crates/weavepy-vm/src/stdlib/python/test_string_tests.py create mode 100644 crates/weavepy-vm/src/stdlib/python/test_support_hashlib_helper.py create mode 100644 crates/weavepy-vm/src/stdlib/python/test_support_i18n_helper.py create mode 100644 crates/weavepy-vm/src/stdlib/python/test_tokenizedata_badsyntax_3131.py create mode 100644 crates/weavepy-vm/src/stdlib/python/test_tokenizedata_init.py create mode 100644 docs/rfcs/0037-cpython-lib-test-conformance-sweep-wave-2.md create mode 100644 tests/regrtest/test_recursion_guard.py create mode 100644 tests/regrtest/test_rfc0037_dropin.py diff --git a/Cargo.lock b/Cargo.lock index d14392d..7f2897d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2377,6 +2377,7 @@ version = "0.0.0" dependencies = [ "num-bigint", "thiserror 2.0.18", + "unicode-normalization", "unicode_names2", "weavepy-lexer", ] diff --git a/crates/weavepy-cli/src/main.rs b/crates/weavepy-cli/src/main.rs index 59d1dde..fd3add4 100644 --- a/crates/weavepy-cli/src/main.rs +++ b/crates/weavepy-cli/src/main.rs @@ -255,6 +255,35 @@ The following implementation-specific options are available: "; fn main() -> ExitCode { + run_on_large_stack(main_dispatch) +} + +/// WeavePy evaluates Python by recursive descent, so Python call depth +/// maps onto native (Rust) stack depth (see `crates/weavepy-vm/src/ +/// recursion.rs`). Run the whole interpreter on a thread with a large +/// stack reserve so that `sys.setrecursionlimit` — enforced by the VM's +/// recursion guard (RFC 0037) — is what bounds recursion, rather than +/// the fixed OS main-thread stack (8 MiB on Linux/macOS). This makes the +/// behaviour uniform across platforms *and* build profiles: debug builds +/// have much larger per-activation stack frames than release, so without +/// this a default `setrecursionlimit(1000)` would overflow the native +/// stack in debug before the guard could fire. The reserve is committed +/// lazily by the OS, so it costs address space, not memory. +fn run_on_large_stack(entry: fn() -> ExitCode) -> ExitCode { + const STACK_BYTES: usize = 1024 * 1024 * 1024; // 1 GiB reserve + match std::thread::Builder::new() + .name("weavepy-main".to_owned()) + .stack_size(STACK_BYTES) + .spawn(entry) + { + Ok(handle) => handle.join().unwrap_or(ExitCode::FAILURE), + // Extremely unlikely, but if the OS refuses the thread, fall + // back to running on the current (main) thread. + Err(_) => entry(), + } +} + +fn main_dispatch() -> ExitCode { init_tracing(); let raw: Vec = env::args().collect(); diff --git a/crates/weavepy-cli/src/repl.rs b/crates/weavepy-cli/src/repl.rs index e78d828..0f07968 100644 --- a/crates/weavepy-cli/src/repl.rs +++ b/crates/weavepy-cli/src/repl.rs @@ -277,6 +277,11 @@ fn needs_continuation(source: &str) -> bool { span.end.0 as usize >= source.len().saturating_sub(1) } Err(parser::ParseError::Lex(lexer::LexError::UnterminatedString { .. })) => true, + // An unterminated (possibly triple-quoted) f-string literal is the + // multi-line-continuation case too — the user is still typing it. + // (`FstringExpectingBrace`/`...OrSpec` are real errors, not these.) + Err(parser::ParseError::Lex(lexer::LexError::UnterminatedFstring { .. })) => true, + Err(parser::ParseError::Lex(lexer::LexError::UnterminatedTripleFstring { .. })) => true, Err(parser::ParseError::Lex(lexer::LexError::UnexpectedEof { .. })) => true, Err(_) => false, } diff --git a/crates/weavepy-compiler/src/bytecode.rs b/crates/weavepy-compiler/src/bytecode.rs index 1e19460..27eb53f 100644 --- a/crates/weavepy-compiler/src/bytecode.rs +++ b/crates/weavepy-compiler/src/bytecode.rs @@ -12,6 +12,14 @@ //! - **Experimental** for the binary encoding; we explicitly do not //! promise wire compatibility with CPython's `.pyc` format. +/// Flag bit OR-ed into the [`OpCode::BinaryOp`] argument to mark an +/// *augmented* assignment (`a += b`). The low byte still encodes the +/// [`BinOpKind`]; the VM strips this bit to recover the operator and, +/// when set, first tries the in-place dunder (`__iadd__`, …) before the +/// regular binary fallback. Kept above `0xFF` so `arg as u8` recovers +/// the operator kind unchanged. +pub const BINARY_OP_INPLACE_FLAG: u32 = 0x100; + /// Sub-operation tag for [`OpCode::BinaryOp`]. Mirrors CPython 3.11+'s /// `_NB_*` enumeration. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -94,6 +102,26 @@ impl BinOpKind { } } + /// The in-place dunder name for this operator (`a += b` → `__iadd__`). + /// Used by the VM's augmented-assignment path. + pub fn inplace_dunder(self) -> &'static str { + match self { + Self::Add => "__iadd__", + Self::Sub => "__isub__", + Self::Mult => "__imul__", + Self::Div => "__itruediv__", + Self::FloorDiv => "__ifloordiv__", + Self::Mod => "__imod__", + Self::Pow => "__ipow__", + Self::LShift => "__ilshift__", + Self::RShift => "__irshift__", + Self::BitOr => "__ior__", + Self::BitXor => "__ixor__", + Self::BitAnd => "__iand__", + Self::MatMult => "__imatmul__", + } + } + /// The opcode argument that encodes this binary operator. pub fn as_arg(self) -> u32 { self as u32 @@ -431,6 +459,15 @@ pub enum OpCode { /// `doctest`. In "exec" mode an expression statement uses /// `PopTop` instead. PrintExpr, + + /// Clear the cell at `co_freevars[arg]` (CPython `DELETE_DEREF`). + /// Empties the cell's contents without touching the value stack; + /// raises `NameError` if the cell is already empty. Used for + /// `del NAME` where NAME is a cell or free variable. + /// + /// Appended at the end of the enum so existing `#[repr(u8)]` + /// discriminants stay stable for any cached bytecode. + DeleteDeref, } impl OpCode { @@ -520,6 +557,7 @@ impl OpCode { OpCode::MatchKeys => "MATCH_KEYS", OpCode::GetLen => "GET_LEN", OpCode::PrintExpr => "PRINT_EXPR", + OpCode::DeleteDeref => "DELETE_DEREF", } } } diff --git a/crates/weavepy-compiler/src/cpython_code.rs b/crates/weavepy-compiler/src/cpython_code.rs index 679167d..f1e1f8a 100644 --- a/crates/weavepy-compiler/src/cpython_code.rs +++ b/crates/weavepy-compiler/src/cpython_code.rs @@ -248,6 +248,7 @@ fn map_to_cpython(ins: Instruction, nlocals: u32) -> MappedOp { O::DeleteName => (op::DELETE_NAME, ins.arg), O::LoadDeref => (op::LOAD_DEREF, ins.arg + nlocals), O::StoreDeref => (op::STORE_DEREF, ins.arg + nlocals), + O::DeleteDeref => (op::DELETE_DEREF, ins.arg + nlocals), O::MakeCell => (op::MAKE_CELL, ins.arg + nlocals), // 3.13 has no real LOAD_CLOSURE opcode; cells live in the fast // array and are loaded with LOAD_FAST. @@ -1002,6 +1003,7 @@ fn map_from_cpython(cp_op: u8, arg: u32, nlocals: u32) -> Option<(OpCode, u32)> op::DELETE_NAME => (O::DeleteName, arg), op::LOAD_DEREF => (O::LoadDeref, arg.saturating_sub(nlocals)), op::STORE_DEREF => (O::StoreDeref, arg.saturating_sub(nlocals)), + op::DELETE_DEREF => (O::DeleteDeref, arg.saturating_sub(nlocals)), op::MAKE_CELL => (O::MakeCell, arg.saturating_sub(nlocals)), op::LOAD_ATTR => (O::LoadAttr, arg >> 1), op::STORE_ATTR => (O::StoreAttr, arg), diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index e10d4b9..f0d7742 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -35,7 +35,8 @@ pub mod bytecode; pub mod cpython_code; pub use bytecode::{ - BinOpKind, CacheTable, CompareKind, InlineCache, Instruction, OpCode, UnaryKind, COOLDOWN, + BinOpKind, CacheTable, CompareKind, InlineCache, Instruction, OpCode, UnaryKind, + BINARY_OP_INPLACE_FLAG, COOLDOWN, }; pub use cpython_code::{CpythonCode, Position}; @@ -316,6 +317,22 @@ impl From for Constant { // ---------- public entry point ---------- +/// PEP 563: does this module open with `from __future__ import annotations`? +/// When it does, every annotation in the module (and all nested scopes) is +/// left *unevaluated* — the compiler stores its verbatim source text as a +/// string instead of emitting code to evaluate it at definition time. A +/// `__future__` import is only legal at the top of the module, so a single +/// scan of the module body suffices. +fn has_future_annotations(module: &Module) -> bool { + module.body.iter().any(|stmt| { + matches!( + &stmt.kind, + StmtKind::ImportFrom { module: Some(m), names, .. } + if m == "__future__" && names.iter().any(|a| a.name == "annotations") + ) + }) +} + /// Compile a parsed module into a top-level [`CodeObject`]. pub fn compile_module(module: &Module) -> Result { compile_module_with_filename(module, "") @@ -343,6 +360,8 @@ pub fn compile_module_with_source( filename.to_owned(), CodeKind::Module, Rc::new(line_index), + Rc::from(source), + has_future_annotations(module), ); top.compile_module_body(module)?; Ok(top.finish()) @@ -364,12 +383,37 @@ pub fn compile_interactive_with_source( filename.to_owned(), CodeKind::Module, Rc::new(line_index), + Rc::from(source), + has_future_annotations(module), ); top.interactive = true; top.compile_module_body(module)?; Ok(top.finish()) } +/// Compile in `eval` mode: the single top-level expression *returns* its +/// value (via `OpCode::ReturnValue`) so the resulting code object, +/// evaluated by `eval(...)`, produces the expression result rather than +/// discarding it. Mirrors CPython's `compile(src, fn, "eval")`. +pub fn compile_eval_with_source( + module: &Module, + source: &str, + filename: &str, +) -> Result { + let line_index = LineIndex::new(source); + let mut top = Compiler::new( + "".to_owned(), + filename.to_owned(), + CodeKind::Module, + Rc::new(line_index), + Rc::from(source), + has_future_annotations(module), + ); + top.eval_mode = true; + top.compile_module_body(module)?; + Ok(top.finish()) +} + /// Lookup table that maps a byte offset back to a 1-based line number. /// Filled once per top-level compile and shared by reference into every /// nested `Compiler` for cheap per-instruction line lookups. @@ -477,6 +521,21 @@ struct Compiler { /// fresh `Compiler` instances), matching CPython's /// `c_interactive && nestlevel <= 1` rule. interactive: bool, + /// `True` for the top-level code object compiled in `eval` mode. + /// The (single) top-level expression *returns* its value via + /// `OpCode::ReturnValue` so `eval(compile(src, fn, "eval"))` yields + /// the expression result instead of discarding it. Never set on + /// nested scopes. + eval_mode: bool, + /// The original module source. Used to slice the verbatim text of an + /// annotation under PEP 563 (see [`Self::future_annotations`]). Empty + /// when the caller compiled without source (then PEP 563 is inert). + source: Rc, + /// PEP 563 (`from __future__ import annotations`): when set, parameter + /// and variable annotations are emitted as their unevaluated source + /// strings rather than being evaluated at definition time. Propagated + /// to every nested function/class scope. + future_annotations: bool, } struct LoopFrame { @@ -516,7 +575,14 @@ struct FinallyFrame { } impl Compiler { - fn new(name: String, filename: String, kind: CodeKind, line_index: Rc) -> Self { + fn new( + name: String, + filename: String, + kind: CodeKind, + line_index: Rc, + source: Rc, + future_annotations: bool, + ) -> Self { let mut co = CodeObject::default(); co.name = name; co.filename = filename; @@ -537,22 +603,29 @@ impl Compiler { annotations_initialized: false, code_kind: kind, interactive: false, + eval_mode: false, + source, + future_annotations, } } fn finish(mut self) -> CodeObject { - // Emit an implicit `return None` if the trailing instruction - // isn't already a return — matches CPython's module-level shape. - let needs_return = self - .co - .instructions - .last() - .is_none_or(|ins| ins.op != OpCode::ReturnValue); - if needs_return { - let none_idx = self.co.intern_constant(Constant::None); - self.emit(OpCode::LoadConst, none_idx); - self.emit(OpCode::ReturnValue, 0); - } + // Always terminate the code object with an implicit `return None`, + // matching CPython's "fall off the end of the function" shape. + // + // It is *not* enough to check whether the textually-last instruction + // is a `ReturnValue`: a function whose body ends in an `if/else` + // where the `else` branch returns leaves a `ReturnValue` last, yet + // the `if` branch can still *fall through* to the end-of-code offset + // via a forward jump. If we skip the implicit return in that case the + // jump lands one past the final instruction and the VM trips a + // "pc out of bounds" `InternalError`. Emitting an unconditional + // trailing `return None` keeps the end-of-code offset a valid target; + // when it is genuinely unreachable it is harmless dead code (two + // instructions) exactly as in CPython. + let none_idx = self.co.intern_constant(Constant::None); + self.emit(OpCode::LoadConst, none_idx); + self.emit(OpCode::ReturnValue, 0); // Place freevars (in declaration order) at the end of the // cells/freevars combined index space. self.co.freevars = self.free_order.clone(); @@ -708,13 +781,16 @@ impl Compiler { match &stmt.kind { StmtKind::Expr(e) => { self.compile_expr(e)?; + // `eval` mode: the single top-level expression returns its + // value so `eval(compile(src, fn, "eval"))` yields it. // Interactive ("single") mode: a top-level expression // statement echoes its value via `sys.displayhook` - // instead of being discarded. Only the interactive - // top-level compiler sets this flag; nested scopes get - // fresh `Compiler` instances (always non-interactive), - // so this never fires inside functions/classes. - if self.interactive { + // instead of being discarded. Only the top-level compiler + // sets these flags; nested scopes get fresh `Compiler` + // instances, so this never fires inside functions/classes. + if self.eval_mode { + self.emit(OpCode::ReturnValue, 0); + } else if self.interactive { self.emit(OpCode::PrintExpr, 0); } else { self.emit(OpCode::PopTop, 0); @@ -761,7 +837,10 @@ impl Compiler { StmtKind::AugAssign { target, op, value } => { self.compile_load_target(target)?; self.compile_expr(value)?; - self.emit(OpCode::BinaryOp, bin_op_kind(*op) as u32); + self.emit( + OpCode::BinaryOp, + bin_op_kind(*op) as u32 | crate::bytecode::BINARY_OP_INPLACE_FLAG, + ); self.compile_assign(target)?; } StmtKind::AnnAssign { @@ -1552,6 +1631,8 @@ impl Compiler { self.co.filename.clone(), CodeKind::Function, self.line_index.clone(), + self.source.clone(), + self.future_annotations, ); inner.co.arg_count = arg_count; inner.co.posonly_count = posonly_count; @@ -1667,7 +1748,7 @@ impl Compiler { for (pname, ann) in &annotated_params { let idx = self.co.intern_constant(Constant::Str(pname.clone())); self.emit(OpCode::LoadConst, idx); - self.compile_expr(ann)?; + self.emit_annotation(ann)?; } self.emit(OpCode::BuildMap, annotated_params.len() as u32); flags |= 0x04; @@ -1703,29 +1784,48 @@ impl Compiler { self.compile_expr(d)?; } self.emit(OpCode::LoadBuildClass, 0); - self.build_class_body(name, body)?; - let name_idx = self.co.intern_constant(Constant::Str(name.to_owned())); - self.emit(OpCode::LoadConst, name_idx); - for b in bases { - self.compile_expr(b)?; - } - if keywords.is_empty() { - self.emit(OpCode::Call, (bases.len() + 2) as u32); + + // A `**kwds` in the class header (or a `*bases` splat) can't be + // expressed with the fixed-arity `Call`/`CallKw` shapes, so fall + // back to the same `CallEx` lowering the function-call site uses: + // build a single positional args tuple `(body, name, *bases)` and + // a merged keyword dict, then unpack both into `__build_class__`. + let has_kw_splat = keywords.iter().any(|k| k.arg.is_none()); + let has_starred_base = bases.iter().any(|b| matches!(b.kind, ExprKind::Starred(_))); + + if has_kw_splat || has_starred_base { + self.build_class_body(name, body)?; + let name_idx = self.co.intern_constant(Constant::Str(name.to_owned())); + self.emit(OpCode::LoadConst, name_idx); + self.emit(OpCode::BuildTuple, 2); + self.compile_starred_args_tuple(bases)?; + self.emit(OpCode::BinaryOp, BinOpKind::Add as u32); + if keywords.is_empty() { + self.emit(OpCode::CallEx, 0); + } else { + self.compile_kwargs_dict(keywords)?; + self.emit(OpCode::CallEx, 1); + } } else { - let mut names: Vec = Vec::with_capacity(keywords.len()); - for k in keywords { - let n = k.arg.clone().ok_or_else(|| { - CompileError::NotImplemented( - "**kwargs splat in class header", - "use explicit metaclass=… keyword form", - ) - })?; - names.push(Constant::Str(n)); - self.compile_expr(&k.value)?; + self.build_class_body(name, body)?; + let name_idx = self.co.intern_constant(Constant::Str(name.to_owned())); + self.emit(OpCode::LoadConst, name_idx); + for b in bases { + self.compile_expr(b)?; + } + if keywords.is_empty() { + self.emit(OpCode::Call, (bases.len() + 2) as u32); + } else { + let mut names: Vec = Vec::with_capacity(keywords.len()); + for k in keywords { + let n = k.arg.clone().expect("kw splat handled by CallEx path above"); + names.push(Constant::Str(n)); + self.compile_expr(&k.value)?; + } + let tup_idx = self.co.intern_constant(Constant::Tuple(names)); + self.emit(OpCode::LoadConst, tup_idx); + self.emit(OpCode::CallKw, (bases.len() + 2) as u32); } - let tup_idx = self.co.intern_constant(Constant::Tuple(names)); - self.emit(OpCode::LoadConst, tup_idx); - self.emit(OpCode::CallKw, (bases.len() + 2) as u32); } for _ in decorator_list { self.emit(OpCode::Call, 1); @@ -1744,6 +1844,8 @@ impl Compiler { self.co.filename.clone(), CodeKind::Class, self.line_index.clone(), + self.source.clone(), + self.future_annotations, ); inner.current_line = self.current_line; // Every class body carries a `__class__` cell so methods can @@ -2296,11 +2398,19 @@ impl Compiler { // Exception handler: __exit__(type(exc), exc, None); if truthy, swallow. let handler_start = self.next_offset(); + // RFC 0037 (WS2): the operand-stack depth to restore before + // entering the handler must preserve every enclosing for-loop's + // iterator (each lives on the stack for the loop's duration). + // Hardcoding `0` truncated the stack to empty, so a `with` that + // *suppressed* an exception inside a `for` lost the iterator and + // the next `FOR_ITER` found an empty stack. This matches the + // `body_depth` convention used by `try`/`except` handlers above. + let body_depth = self.loop_stack.iter().filter(|fr| fr.is_for_loop).count() as u32; self.co.exception_table.push(ExcHandler { start: body_start, end: body_end, handler: handler_start, - depth: 0, + depth: body_depth, }); // Stack: [exc] self.emit(OpCode::LoadFast, cm_idx); @@ -2341,6 +2451,45 @@ impl Compiler { // ---------- assignment ---------- + /// Emit the *value* of a single annotation expression onto the stack. + /// + /// Under PEP 563 (`from __future__ import annotations`) annotations are + /// not evaluated: we push the annotation's verbatim source text as a + /// string constant, so `__annotations__` ends up storing e.g. + /// `'list[int]'` instead of the runtime object. This is what lets + /// forward references and not-yet-imported names (e.g. `IO[str]` typed + /// only for the type checker) appear in annotations without raising at + /// definition time. Falls back to evaluating the expression when the + /// future flag is off, or when no source is available to slice. + fn emit_annotation(&mut self, annotation: &Expr) -> Result<(), CompileError> { + if self.future_annotations { + if let Some(text) = self.annotation_source(annotation) { + let idx = self.co.intern_constant(Constant::Str(text)); + self.emit(OpCode::LoadConst, idx); + return Ok(()); + } + } + self.compile_expr(annotation) + } + + /// The verbatim source text covered by `expr`'s span, trimmed of + /// surrounding whitespace. Returns `None` when the compiler holds no + /// source (an AST was compiled directly) or the span is degenerate, so + /// the caller can fall back to eager evaluation. + fn annotation_source(&self, expr: &Expr) -> Option { + let start = expr.span.start.0 as usize; + let end = expr.span.end.0 as usize; + if self.source.is_empty() || end <= start || end > self.source.len() { + return None; + } + let text = self.source.get(start..end)?.trim(); + if text.is_empty() { + None + } else { + Some(text.to_owned()) + } + } + /// Emit code that ensures the current scope's `__annotations__` /// dict exists and records `annotation` against `name`. Used /// for class- and module-body `x: T = ...` statements. @@ -2384,7 +2533,7 @@ impl Compiler { self.annotations_initialized = true; } // __annotations__[name] = annotation - self.compile_expr(annotation)?; + self.emit_annotation(annotation)?; let dict_idx = self.co.intern_name(dict_name); self.emit(OpCode::LoadName, dict_idx); let key_idx = self.co.intern_constant(Constant::Str(name.to_owned())); @@ -2620,12 +2769,13 @@ impl Compiler { self.emit(OpCode::DeleteFast, idx); } Binding::Cell | Binding::Free | Binding::Nonlocal => { - // CPython raises NameError if the cell is empty, but - // simply storing nothing here matches the semantics - // for our current cell representation; emit DeleteDeref - // when we add it. + // `del NAME` clears the cell's contents. This must NOT + // touch the value stack (unlike `StoreDeref`, which pops + // its operand) — emitting `StoreDeref` here underflows + // the stack. `DeleteDeref` empties the cell and raises + // NameError at runtime if it was already empty. let idx = self.cell_or_free_index(name); - self.emit(OpCode::StoreDeref, idx); + self.emit(OpCode::DeleteDeref, idx); } Binding::Global => { let idx = self.co.intern_name(name); @@ -3270,12 +3420,16 @@ impl Compiler { // PEP 530: a comprehension that uses `async for` (or `await` // inside the element / filter) compiles to a coroutine; the // caller awaits the resulting coroutine to get the value. - let is_async_comp = generators.iter().any(|g| g.is_async) - || expr_contains_await(elt) - || value.map(expr_contains_await).unwrap_or(false) - || generators - .iter() - .any(|g| expr_contains_await(&g.iter) || g.ifs.iter().any(expr_contains_await)); + // A comprehension is a coroutine if it has an `async for` + // clause, directly contains an `await`, *or* its element/value + // is itself an async comprehension. The last case is PEP 530's + // implicit propagation: in `[[x async for x in a] for j in b]` + // the inner async comp evaluates to a coroutine, so the outer + // (otherwise synchronous) comprehension must `await` it and is + // therefore async too. `expr_contains_await` deliberately stops + // at nested comprehension scopes, so we detect the nested-async + // case separately with `expr_contains_async_comp`. + let is_async_comp = comp_clause_is_async(generators, elt, value); let name = match kind { CompKind::List => "", CompKind::Set => "", @@ -3287,6 +3441,8 @@ impl Compiler { self.co.filename.clone(), CodeKind::Comprehension, self.line_index.clone(), + self.source.clone(), + self.future_annotations, ); inner.current_line = self.current_line; inner.co.arg_count = 1; @@ -3326,6 +3482,21 @@ impl Compiler { collect_reads_expr(i, &mut reads); } } + // A comprehension's `for` targets are *local to the comprehension* + // and shadow any same-named variable in the enclosing scope. Bind + // them BEFORE free-variable resolution: otherwise a target like `f` + // in `{f for f in xs}` whose name also exists as an enclosing local + // `f` is mistaken for a free reference to that outer `f`. That spuriously + // cell-promotes the enclosing local and shifts every freevar index by + // one — silently aliasing later closure reads. CPython's symtable binds + // comprehension targets first for exactly this reason. + for g in generators { + let mut assigned = HashSet::new(); + collect_target_names(&g.target, &mut assigned); + for n in assigned { + inner.bindings.insert(n, Binding::Local); + } + } for name in reads { if inner.bindings.contains_key(&name) { continue; @@ -3340,12 +3511,40 @@ impl Compiler { } } } - // Collect names assigned by comprehension targets — they're locals. - for g in generators { - let mut assigned = HashSet::new(); - collect_target_names(&g.target, &mut assigned); - for n in assigned { - inner.bindings.insert(n, Binding::Local); + + // RFC 0037 (WS2): a comprehension target (or `.0`) that an inner + // scope — a *nested* comprehension or a lambda inside the + // element / value / filter / inner-iterable — closes over must be + // a **cell**, and that has to be decided *before* the loop body + // is emitted. Otherwise `compile_comp_body` stores the target + // with `STORE_FAST` into a plain local slot while the inner scope + // reads it via `LOAD_DEREF` from an (unwritten) cell — yielding + // `None`, exactly the `[[x for y in ys] for x in xs]` bug. + // Mirrors `analyze_scope_function`'s pre-emission cell promotion. + { + let mut needed_in_inner: HashSet = HashSet::new(); + collect_inner_free_expr(elt, &inner.bindings, &mut needed_in_inner); + if let Some(v) = value { + collect_inner_free_expr(v, &inner.bindings, &mut needed_in_inner); + } + for (gi, g) in generators.iter().enumerate() { + // generators[0].iter is evaluated in the *enclosing* + // scope (passed in as `.0`); every later iter and every + // filter runs inside this comprehension. + if gi > 0 { + collect_inner_free_expr(&g.iter, &inner.bindings, &mut needed_in_inner); + } + for cond in &g.ifs { + collect_inner_free_expr(cond, &inner.bindings, &mut needed_in_inner); + } + } + for name in needed_in_inner { + if matches!(inner.bindings.get(&name), Some(Binding::Local)) { + inner.bindings.insert(name.clone(), Binding::Cell); + if !inner.co.cellvars.contains(&name) { + inner.co.cellvars.push(name); + } + } } } @@ -4038,6 +4237,105 @@ fn expr_contains_await(expr: &Expr) -> bool { } } +/// Does evaluating `expr` produce (and inline-await) the result of a +/// nested *async* list/set/dict comprehension? This drives PEP 530's +/// implicit async propagation: a comprehension whose element contains +/// an async comprehension becomes async itself. We recurse through +/// ordinary sub-expressions but stop at scope boundaries (`lambda`), +/// and we do **not** treat a nested async *generator expression* as +/// propagating — `(x async for x in a)` evaluates to an async-generator +/// object that is not awaited in place. +fn comp_clause_is_async( + generators: &[Comprehension], + elt: &Expr, + value: Option<&Expr>, +) -> bool { + generators.iter().any(|g| g.is_async) + || expr_contains_await(elt) + || value.map(expr_contains_await).unwrap_or(false) + || generators + .iter() + .any(|g| expr_contains_await(&g.iter) || g.ifs.iter().any(expr_contains_await)) + || expr_contains_async_comp(elt) + || value.map(expr_contains_async_comp).unwrap_or(false) + || generators + .iter() + .any(|g| g.ifs.iter().any(expr_contains_async_comp)) +} + +fn expr_contains_async_comp(expr: &Expr) -> bool { + match &expr.kind { + ExprKind::ListComp { elt, generators } | ExprKind::SetComp { elt, generators } => { + comp_clause_is_async(generators, elt, None) + } + ExprKind::DictComp { + key, + value, + generators, + } => comp_clause_is_async(generators, key, Some(value)), + // An async genexpr is an async-generator object, not an + // inline-awaited value, so it does not propagate. + ExprKind::GeneratorExp { .. } => false, + // Scope boundary: an async comprehension inside a lambda body + // belongs to that lambda, not the enclosing comprehension. + ExprKind::Lambda { .. } => false, + ExprKind::Await(_) => false, + ExprKind::Yield(v) => v.as_deref().is_some_and(expr_contains_async_comp), + ExprKind::YieldFrom(v) => expr_contains_async_comp(v), + ExprKind::JoinedStr(parts) => parts.iter().any(expr_contains_async_comp), + ExprKind::FormattedValue { + value, format_spec, .. + } => { + expr_contains_async_comp(value) + || format_spec.as_deref().is_some_and(expr_contains_async_comp) + } + ExprKind::BinOp { left, right, .. } => { + expr_contains_async_comp(left) || expr_contains_async_comp(right) + } + ExprKind::BoolOp { values, .. } => values.iter().any(expr_contains_async_comp), + ExprKind::UnaryOp { operand, .. } => expr_contains_async_comp(operand), + ExprKind::Compare { + left, comparators, .. + } => expr_contains_async_comp(left) || comparators.iter().any(expr_contains_async_comp), + ExprKind::IfExp { test, body, orelse } => { + expr_contains_async_comp(test) + || expr_contains_async_comp(body) + || expr_contains_async_comp(orelse) + } + ExprKind::NamedExpr { target, value } => { + expr_contains_async_comp(target) || expr_contains_async_comp(value) + } + ExprKind::Call { + func, + args, + keywords, + } => { + expr_contains_async_comp(func) + || args.iter().any(expr_contains_async_comp) + || keywords.iter().any(|k| expr_contains_async_comp(&k.value)) + } + ExprKind::Attribute { value, .. } => expr_contains_async_comp(value), + ExprKind::Subscript { value, slice } => { + expr_contains_async_comp(value) || expr_contains_async_comp(slice) + } + ExprKind::Slice { lower, upper, step } => { + lower.as_deref().is_some_and(expr_contains_async_comp) + || upper.as_deref().is_some_and(expr_contains_async_comp) + || step.as_deref().is_some_and(expr_contains_async_comp) + } + ExprKind::Tuple(items) | ExprKind::List(items) | ExprKind::Set(items) => { + items.iter().any(expr_contains_async_comp) + } + ExprKind::Dict { keys, values } => { + keys.iter() + .any(|k| k.as_ref().is_some_and(expr_contains_async_comp)) + || values.iter().any(expr_contains_async_comp) + } + ExprKind::Starred(inner) => expr_contains_async_comp(inner), + ExprKind::Constant(_) | ExprKind::Name(_) => false, + } +} + fn collect_inner_free_expr( expr: &Expr, outer_bindings: &IndexMap, @@ -4334,6 +4632,15 @@ fn collect_decls( StmtKind::AugAssign { target, .. } | StmtKind::AnnAssign { target, .. } => { collect_target_names(target, assigned); } + // `del NAME` is a binding operation in CPython (`DEF_LOCAL`): the + // name is local to this scope, and — crucially — a nested scope + // declaring it `nonlocal` resolves to (and cells) it here. Bare + // names only; `del obj[i]` / `del obj.attr` bind nothing. + StmtKind::Delete(targets) => { + for t in targets { + collect_target_names(t, assigned); + } + } StmtKind::For { target, body, diff --git a/crates/weavepy-conformance/src/bin/main.rs b/crates/weavepy-conformance/src/bin/main.rs index f821452..c26ef8b 100644 --- a/crates/weavepy-conformance/src/bin/main.rs +++ b/crates/weavepy-conformance/src/bin/main.rs @@ -141,6 +141,10 @@ impl From for regrtest::ExecutionMode { } fn main() -> ExitCode { + run_on_large_stack(run_real_main) +} + +fn run_real_main() -> ExitCode { match real_main() { Ok(()) => ExitCode::SUCCESS, Err(err) => { @@ -150,6 +154,26 @@ fn main() -> ExitCode { } } +/// Run the harness on a generously-sized stack, mirroring `weavepy-cli`'s +/// `run_on_large_stack`. `--mode in-process` executes each `test_*.py` +/// inside *this* process, so without a large reserve a deep-but-bounded +/// Python recursion (e.g. a `RecursionError` guard test, or the recursive +/// drop of its traceback chain) overflows the fixed 8 MiB OS main-thread +/// stack before the interpreter's own recursion guard can fire. The 1 GiB +/// reserve is committed lazily by the OS, so it costs address space, not +/// resident memory. +fn run_on_large_stack(entry: fn() -> ExitCode) -> ExitCode { + const STACK_BYTES: usize = 1024 * 1024 * 1024; // 1 GiB reserve + match std::thread::Builder::new() + .name("weavepy-conformance-main".to_owned()) + .stack_size(STACK_BYTES) + .spawn(entry) + { + Ok(handle) => handle.join().unwrap_or(ExitCode::FAILURE), + Err(_) => entry(), + } +} + fn real_main() -> Result<()> { let cli = Cli::parse(); let workspace = resolve_workspace(cli.workspace.as_deref())?; diff --git a/crates/weavepy-conformance/src/regrtest.rs b/crates/weavepy-conformance/src/regrtest.rs index e83efa7..3d6f53a 100644 --- a/crates/weavepy-conformance/src/regrtest.rs +++ b/crates/weavepy-conformance/src/regrtest.rs @@ -43,7 +43,6 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::fmt::Write as _; use std::fs; -use std::io::Read; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -798,25 +797,49 @@ enum ChildOutcome { /// Wait up to `timeout` for `child` to exit. If it doesn't, SIGKILL the /// child and return [`ChildOutcome::TimedOut`]. +/// +/// stdout/stderr are drained on dedicated threads from the moment the +/// child starts. Reading only *after* the child exits (the obvious +/// approach) deadlocks against any child that writes more than one pipe +/// buffer's worth of output (~64 KiB): the child blocks in `write()` +/// waiting for us to read, while we block in `wait()` waiting for it to +/// exit. A `unittest` file with hundreds of failing assertions trips this +/// instantly, so the reader threads are load-bearing for subprocess mode. fn wait_with_timeout(mut child: std::process::Child, timeout: Duration) -> ChildOutcome { + fn drain( + pipe: Option, + ) -> Option>> { + pipe.map(|mut s| { + std::thread::spawn(move || { + let mut buf = Vec::new(); + let _ = s.read_to_end(&mut buf); + buf + }) + }) + } + fn collect(handle: Option>>) -> String { + handle + .and_then(|h| h.join().ok()) + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .unwrap_or_default() + } + + let out_handle = drain(child.stdout.take()); + let err_handle = drain(child.stderr.take()); let start = Instant::now(); loop { match child.try_wait() { Ok(Some(status)) => { - let mut stdout = String::new(); - let mut stderr = String::new(); - if let Some(mut s) = child.stdout.take() { - let _ = s.read_to_string(&mut stdout); - } - if let Some(mut s) = child.stderr.take() { - let _ = s.read_to_string(&mut stderr); - } - return ChildOutcome::Exited(status, stdout, stderr); + return ChildOutcome::Exited(status, collect(out_handle), collect(err_handle)); } Ok(None) => { if start.elapsed() > timeout { let _ = child.kill(); let _ = child.wait(); + // Join the readers so the threads don't outlive us; + // the pipes close on kill, so they return promptly. + let _ = collect(out_handle); + let _ = collect(err_handle); return ChildOutcome::TimedOut; } std::thread::sleep(Duration::from_millis(50)); diff --git a/crates/weavepy-lexer/src/error.rs b/crates/weavepy-lexer/src/error.rs index 88a2fa6..3e3feaa 100644 --- a/crates/weavepy-lexer/src/error.rs +++ b/crates/weavepy-lexer/src/error.rs @@ -7,7 +7,31 @@ use thiserror::Error; pub enum LexError { #[error("unterminated string literal at byte {pos}")] UnterminatedString { pos: u32 }, - #[error("invalid character {ch:?} at byte {pos}")] + // PEP 701 f-string diagnostics. CPython distinguishes an unterminated + // f-string *literal* from an unterminated *replacement field* and uses + // f-string-specific wording, which several `test_fstring` negative + // cases assert on verbatim. + #[error("unterminated f-string literal")] + UnterminatedFstring { pos: u32 }, + #[error("unterminated triple-quoted f-string literal")] + UnterminatedTripleFstring { pos: u32 }, + #[error("f-string: expecting '}}'")] + FstringExpectingBrace { pos: u32 }, + #[error("f-string: expecting '}}', or format specs")] + FstringExpectingBraceOrSpec { pos: u32 }, + #[error("closing parenthesis '{close}' does not match opening parenthesis '{open}'")] + FstringParenMismatch { close: char, open: char, pos: u32 }, + #[error("f-string: unmatched '{close}'")] + FstringUnmatchedParen { close: char, pos: u32 }, + #[error("'{open}' was never closed")] + BracketNeverClosed { open: char, pos: u32 }, + #[error("f-string: newlines are not allowed in format specifiers for single quoted f-strings")] + FstringNewlineInSpec { pos: u32 }, + // CPython renders this as `invalid character '€' (U+20AC)` — the + // glyph in quotes followed by the code point. The byte position is + // carried separately (see [`LexError::byte_offset`]) and surfaces as + // the `SyntaxError`'s `lineno`/`offset`, not in the message text. + #[error("invalid character {ch:?} (U+{codepoint:04X})", codepoint = u32::from(*ch))] InvalidChar { ch: char, pos: u32 }, #[error("inconsistent indentation at byte {pos}")] InconsistentIndent { pos: u32 }, @@ -22,3 +46,28 @@ pub enum LexError { #[error("unexpected EOF at byte {pos}: {message}")] UnexpectedEof { pos: u32, message: String }, } + +impl LexError { + /// Byte offset into the source where the error was detected. Used to + /// compute the `SyntaxError` line/column at the raise site. + pub fn byte_offset(&self) -> u32 { + match self { + LexError::UnterminatedString { pos } + | LexError::UnterminatedFstring { pos } + | LexError::UnterminatedTripleFstring { pos } + | LexError::FstringExpectingBrace { pos } + | LexError::FstringExpectingBraceOrSpec { pos } + | LexError::FstringParenMismatch { pos, .. } + | LexError::FstringUnmatchedParen { pos, .. } + | LexError::BracketNeverClosed { pos, .. } + | LexError::FstringNewlineInSpec { pos } + | LexError::InvalidChar { pos, .. } + | LexError::InconsistentIndent { pos } + | LexError::UnknownDedent { pos } + | LexError::InvalidNumber { pos, .. } + | LexError::InvalidStringPrefix { pos, .. } + | LexError::StrayBackslash { pos } + | LexError::UnexpectedEof { pos, .. } => *pos, + } + } +} diff --git a/crates/weavepy-lexer/src/lib.rs b/crates/weavepy-lexer/src/lib.rs index 0ddaea5..f5b3811 100644 --- a/crates/weavepy-lexer/src/lib.rs +++ b/crates/weavepy-lexer/src/lib.rs @@ -29,8 +29,8 @@ pub mod scanner; pub mod token; pub use error::LexError; -pub use scanner::tokenize; -pub use token::{BytePos, Keyword, Span, StringPrefix, Token, TokenKind}; +pub use scanner::{tokenize, tokenize_with_escapes}; +pub use token::{BytePos, EscapeWarning, Keyword, Span, StringPrefix, Token, TokenKind}; #[cfg(test)] mod tests { @@ -141,4 +141,137 @@ mod tests { let newlines = k.iter().filter(|t| **t == TokenKind::Newline).count(); assert_eq!(newlines, 1); } + + fn lex_err_msg(src: &str) -> String { + tokenize(src) + .expect_err("source should fail to tokenize") + .to_string() + } + + // PEP 701 — the lexer must reproduce CPython's f-string diagnostics + // verbatim; `test_fstring.py` asserts on these exact strings. + #[test] + fn fstring_unterminated_literal_messages() { + // test_not_closing_quotes: bare `f"` / `f'`. + assert_eq!(lex_err_msg("f\""), "unterminated f-string literal"); + assert_eq!(lex_err_msg("f'"), "unterminated f-string literal"); + // A single-line f-string may not span a newline in its literal part. + assert_eq!(lex_err_msg("f'abc\n"), "unterminated f-string literal"); + } + + #[test] + fn fstring_unterminated_triple_messages() { + // test_not_closing_quotes: `f"""` / `f'''`. + assert_eq!( + lex_err_msg("f\"\"\""), + "unterminated triple-quoted f-string literal" + ); + assert_eq!( + lex_err_msg("f'''"), + "unterminated triple-quoted f-string literal" + ); + } + + #[test] + fn fstring_unterminated_field_is_expecting_brace() { + // An open replacement *expression* that runs off the end is + // "f-string: expecting '}'" — including when a same-quote that + // can't find its pair was really the f-string terminator (`f'{3'`). + assert_eq!(lex_err_msg("f'{3'"), "f-string: expecting '}'"); + assert_eq!(lex_err_msg("f'{3!'"), "f-string: expecting '}'"); + assert_eq!(lex_err_msg("f'{3!s'"), "f-string: expecting '}'"); + assert_eq!(lex_err_msg("f'{(3)'"), "f-string: expecting '}'"); + // `{{` is a brace escape; the trailing `{` then opens an (empty) + // field that hits raw EOF. + assert_eq!(lex_err_msg("f'{{{'"), "f-string: expecting '}'"); + } + + #[test] + fn fstring_unterminated_spec_names_format_specs() { + // An open *format spec* gets CPython's spec-specific wording. The + // outer quote inside a single-quoted spec is the terminator (a + // fill-char must use the other quote), so this also triggers it. + assert_eq!( + lex_err_msg("f'{3:'"), + "f-string: expecting '}', or format specs" + ); + assert_eq!( + lex_err_msg("f'{x:>'"), + "f-string: expecting '}', or format specs" + ); + } + + #[test] + fn fstring_same_quote_reuse_is_valid() { + // PEP 701 quote reuse: a same-quote that *does* find its pair is a + // genuine nested string, not the terminator. + assert_eq!(kinds("f'{3 + 'a'}'")[0], TokenKind::String); + assert_eq!(kinds("f'{3''}'")[0], TokenKind::String); // empty nested str + // The other quote is literal inside a format spec. + assert_eq!(kinds("f\"{x:'>10}\"")[0], TokenKind::String); + } + + #[test] + fn fstring_newline_in_single_line_spec() { + // test_newlines_in_format_specifiers: a newline in the format spec + // of a single-line f-string is rejected (CPython's full wording + // ends "...for single quoted f-strings")... + assert_eq!( + lex_err_msg("f'{1:d\n}'"), + "f-string: newlines are not allowed in format specifiers for single quoted f-strings" + ); + // ...but is perfectly legal inside a triple-quoted f-string. + assert_eq!(kinds("f'''{1:d\n}'''")[0], TokenKind::String); + } + + #[test] + fn fstring_bracket_mismatch_messages() { + // A close that doesn't match the innermost opener names both, like + // CPython (test_mismatched_parens). + assert_eq!( + lex_err_msg("f'{((}'"), + "closing parenthesis '}' does not match opening parenthesis '('" + ); + assert_eq!( + lex_err_msg("f'{a[4}'"), + "closing parenthesis '}' does not match opening parenthesis '['" + ); + assert_eq!( + lex_err_msg("f'{a(4}'"), + "closing parenthesis '}' does not match opening parenthesis '('" + ); + } + + #[test] + fn fstring_unmatched_and_never_closed() { + // A `)` with nothing open. + assert_eq!(lex_err_msg("f'{)}'"), "f-string: unmatched ')'"); + assert_eq!(lex_err_msg("f'{)#}'"), "f-string: unmatched ')'"); + // A `#` comment that eats the rest to EOF leaves the innermost + // bracket "never closed" (the field `{`, or a nested opener). + assert_eq!(lex_err_msg("f'{1#}'"), "'{' was never closed"); + assert_eq!(lex_err_msg("f'{#}'"), "'{' was never closed"); + assert_eq!(lex_err_msg("f'{(1#}'"), "'(' was never closed"); + // A comment terminated by a newline is *not* "never closed". + assert_eq!(lex_err_msg("f'{1#}\n'"), "f-string: expecting '}'"); + } + + #[test] + fn fstring_nested_dict_and_calls_still_valid() { + // The stack-based scanner must keep accepting balanced nesting. + assert_eq!(kinds("f'{ {1:2} }'")[0], TokenKind::String); + assert_eq!(kinds("f'{d[\"k\"]}'")[0], TokenKind::String); + assert_eq!(kinds("f'{f(a, b)}'")[0], TokenKind::String); + assert_eq!(kinds("f'{x:{y}}'")[0], TokenKind::String); + } + + #[test] + fn fstring_unterminated_nested_string_stays_string_error() { + // test_unterminated_string: a *different*-quoted nested string is + // what's unterminated, so CPython keeps the generic wording + // ("unterminated string literal", which our Display extends with a + // byte offset — still a regex match for the test). + assert!(lex_err_msg("f'{\"x'").starts_with("unterminated string literal")); + assert!(lex_err_msg("f'{(\"x'").starts_with("unterminated string literal")); + } } diff --git a/crates/weavepy-lexer/src/scanner.rs b/crates/weavepy-lexer/src/scanner.rs index abc6cb5..554e877 100644 --- a/crates/weavepy-lexer/src/scanner.rs +++ b/crates/weavepy-lexer/src/scanner.rs @@ -12,24 +12,39 @@ //! goal here is correctness against CPython 3.13. use crate::error::LexError; -use crate::token::{Keyword, Span, StringPrefix, Token, TokenKind}; +use crate::token::{EscapeWarning, Keyword, Span, StringPrefix, Token, TokenKind}; /// Tokenize a complete Python source buffer. pub fn tokenize(source: &str) -> Result, LexError> { + tokenize_with_escapes(source).0 +} + +/// Tokenize, also returning the invalid-escape [`EscapeWarning`]s found +/// while scanning string/bytes literals. +/// +/// The warnings are returned **even when tokenizing fails**: CPython +/// detects invalid escapes as the tokenizer walks each string, so a +/// `SyntaxWarning` from an earlier literal must still fire before a +/// later hard error (e.g. `eval("'\\e' $")` warns once *and* raises a +/// `SyntaxError` for the stray `$`). Collecting them on the scanner and +/// handing them back regardless of the result preserves that ordering. +pub fn tokenize_with_escapes(source: &str) -> (Result, LexError>, Vec) { let mut scanner = Scanner::new(source); let mut out = Vec::new(); - loop { - match scanner.next_token()? { - Some(tok) => { + let result = loop { + match scanner.next_token() { + Ok(Some(tok)) => { let is_endmarker = matches!(tok.kind, TokenKind::Endmarker); out.push(tok); if is_endmarker { - return Ok(out); + break Ok(out); } } - None => continue, + Ok(None) => continue, + Err(e) => break Err(e), } - } + }; + (result, scanner.escape_warnings) } struct Scanner<'src> { @@ -51,6 +66,10 @@ struct Scanner<'src> { pending_indent: bool, /// True after we emitted ENDMARKER; further calls return None. finished: bool, + /// Invalid-escape `SyntaxWarning`s gathered while scanning string and + /// bytes literals, in source order (the first invalid escape *per + /// literal*, matching CPython's `first_invalid_escape` tracking). + escape_warnings: Vec, } impl<'src> Scanner<'src> { @@ -64,9 +83,84 @@ impl<'src> Scanner<'src> { pending_dedents: 0, pending_indent: false, finished: false, + escape_warnings: Vec::new(), } } + /// Inspect the escape that begins at the backslash at absolute offset + /// `bs` (in a non-raw string/bytes body) and, if it is one CPython + /// would flag, record a [`EscapeWarning`]. Returns `true` when a + /// warning was recorded so the caller can stop after the *first* + /// invalid escape in a literal (CPython warns once per literal). + /// + /// `is_bytes` selects the bytes alphabet, which has no `\N`/`\u`/`\U` + /// named/Unicode escapes — those letters are invalid escapes there. + /// Valid escapes (and the incomplete `\x`/`\u`/`\U`/`\N` forms, which + /// the parser turns into hard `SyntaxError`s at decode time) are left + /// alone here. + fn note_invalid_escape(&mut self, bs: usize, is_bytes: bool) -> bool { + let Some(&esc) = self.src.get(bs + 1) else { + return false; + }; + // Octal escape: warn when the written value exceeds `\377`. + if (b'0'..=b'7').contains(&esc) { + let mut val = (esc - b'0') as u32; + let mut digits = String::new(); + digits.push(esc as char); + let mut k = bs + 2; + for _ in 0..2 { + match self.src.get(k) { + Some(&d) if (b'0'..=b'7').contains(&d) => { + val = val * 8 + (d - b'0') as u32; + digits.push(d as char); + k += 1; + } + _ => break, + } + } + if val > 0o377 { + self.escape_warnings.push(EscapeWarning { + offset: bs as u32, + message: format!("invalid octal escape sequence '\\{digits}'"), + }); + return true; + } + return false; + } + // Recognised single-character / sized escapes. `x`/`u`/`U`/`N` + // are accepted here (a malformed one is a decode-time error, not + // a warning); bytes literals have no `u`/`U`/`N`. + let recognised = matches!( + esc, + b'\n' | b'\r' + | b'\\' + | b'\'' + | b'"' + | b'a' + | b'b' + | b'f' + | b'n' + | b'r' + | b't' + | b'v' + | b'x' + ) || (!is_bytes && matches!(esc, b'u' | b'U' | b'N')); + if recognised { + return false; + } + // Unknown escape — render the *character* (decoding UTF-8 so a + // non-ASCII escape like `\€` shows the glyph, not a stray byte). + let esc_char = std::str::from_utf8(&self.src[bs + 1..]) + .ok() + .and_then(|s| s.chars().next()) + .unwrap_or(esc as char); + self.escape_warnings.push(EscapeWarning { + offset: bs as u32, + message: format!("invalid escape sequence '\\{esc_char}'"), + }); + true + } + /// Produce the next token, or `Ok(None)` if the scanner consumed /// whitespace / a comment with no token to emit at this point. fn next_token(&mut self) -> Result, LexError> { @@ -163,6 +257,21 @@ impl<'src> Scanner<'src> { if is_ident_start(b) { return self.scan_ident_or_prefixed_string().map(Some); } + // PEP 3131: non-ASCII identifier start (e.g. `π`, `名前`, `Δt`). + // The ASCII fast path above is the common case; here we decode a + // single UTF-8 scalar and admit it when it's an `XID_Start` + // character. The continuation loop in + // `scan_ident_or_prefixed_string` already consumes `XID_Continue`, + // so the rest of the identifier falls out uniformly. (NFKC + // normalization of the resulting name is a documented follow-up; + // we currently key identifiers on their source spelling.) + if b >= 0x80 { + if let Some((ch, _)) = decode_utf8(&self.src[self.pos..]) { + if unicode_ident::is_xid_start(ch) { + return self.scan_ident_or_prefixed_string().map(Some); + } + } + } // Numbers. if b.is_ascii_digit() || (b == b'.' && self.peek_at(1).is_some_and(|c| c.is_ascii_digit())) @@ -434,6 +543,21 @@ impl<'src> Scanner<'src> { let quote = self.peek().expect("scan_string at quote"); debug_assert!(quote == b'"' || quote == b'\''); let triple = self.peek_at(1) == Some(quote) && self.peek_at(2) == Some(quote); + // PEP 701 — f-strings need a structure-aware scan so that quotes, + // braces, backslashes, comments, newlines, and even nested + // f-strings *inside* replacement fields don't prematurely + // terminate the literal. We still emit a single `String` token + // (the parser re-scans the interior); this just finds the true + // extent. Non-f strings keep the simple fast paths below. + if prefix.fstring { + if triple { + self.pos += 3; + } else { + self.pos += 1; + } + self.scan_fstring_extent(start, quote, triple, prefix.raw)?; + return Ok(self.token(TokenKind::String, start, self.pos)); + } if triple { self.pos += 3; self.scan_triple_string(start, quote, prefix) @@ -443,6 +567,307 @@ impl<'src> Scanner<'src> { } } + /// PEP 701 — scan the literal part of a (possibly nested) f-string, + /// recursing through `{ ... }` replacement fields. On entry `self.pos` + /// is just past the opening quote(s); on success it ends just past + /// the matching closing quote(s). + fn scan_fstring_extent( + &mut self, + start: usize, + quote: u8, + triple: bool, + _raw: bool, + ) -> Result<(), LexError> { + loop { + let Some(b) = self.peek() else { + // Ran off the end with the literal still open: CPython + // names the quote style ("unterminated f-string literal" + // vs "...triple-quoted f-string literal"). + return Err(if triple { + LexError::UnterminatedTripleFstring { pos: start as u32 } + } else { + LexError::UnterminatedFstring { pos: start as u32 } + }); + }; + if b == quote { + if triple { + if self.peek_at(1) == Some(quote) && self.peek_at(2) == Some(quote) { + self.pos += 3; + return Ok(()); + } + self.pos += 1; + continue; + } + self.pos += 1; + return Ok(()); + } + match b { + // A single-line f-string's *literal* text can't span + // lines; newlines are only legal inside `{ }`. + b'\n' | b'\r' if !triple => { + return Err(LexError::UnterminatedFstring { pos: start as u32 }); + } + // Escape in the literal part — consume the backslash and + // the byte it escapes (full validation happens at decode + // time). This applies in raw f-strings too: the backslash + // stays literal, but per CPython a `\` still does + // not terminate the string (e.g. `fr'\'\"'`), so we must + // consume both bytes here rather than letting the quote + // close the literal early. Exception: `{`/`}` are always + // structural in an f-string (escaped only as `{{`/`}}`, + // never by a backslash), so a backslash never swallows + // them — `fr'\{{'` is a literal backslash followed by the + // brace escape. + b'\\' => { + self.pos += 1; + if matches!(self.peek(), Some(n) if n != b'{' && n != b'}') { + self.pos += 1; + } + } + b'{' => { + if self.peek_at(1) == Some(b'{') { + self.pos += 2; // `{{` literal-brace escape + } else { + self.pos += 1; + self.scan_fstring_field_extent(start, quote, triple)?; + } + } + b'}' => { + // `}}` is a literal-brace escape; a lone `}` is + // invalid, but we defer that diagnostic to the parser, + // which carries span context for a good message. + if self.peek_at(1) == Some(b'}') { + self.pos += 2; + } else { + self.pos += 1; + } + } + _ => self.pos += 1, + } + } + } + + /// Scan a replacement field's *expression* part from just past its + /// opening `{`. Tracks `()[]{}` nesting and skips nested strings + /// (including nested f-strings) and comments so their contents can't + /// close the field early. A top-level `:` hands off to the + /// format-spec scan; a top-level `}` ends the field. + fn scan_fstring_field_extent( + &mut self, + start: usize, + outer_quote: u8, + outer_triple: bool, + ) -> Result<(), LexError> { + // Explicit bracket stack (mirroring the parser) so we reproduce + // CPython's precise PEP 701 diagnostics rather than masking a + // mismatch behind a generic "expecting '}'". `in_comment` records a + // `#` comment that ran to EOF, which CPython reports as the innermost + // open bracket having "never closed" (distinct from a plain + // unterminated field). + let mut stack: Vec = Vec::new(); + let mut in_comment = false; + loop { + let Some(b) = self.peek() else { + if in_comment { + let open = stack.last().copied().unwrap_or(b'{'); + return Err(LexError::BracketNeverClosed { + open: open as char, + pos: start as u32, + }); + } + return Err(LexError::FstringExpectingBrace { pos: start as u32 }); + }; + match b { + b'}' if stack.is_empty() => { + self.pos += 1; + return Ok(()); + } + // Top-level `:` begins the format spec, where `#`, quotes + // and `:` are literal and only `{ }` nest replacement + // fields (e.g. `{x:#06x}`, `{x:.{prec}f}`). + b':' if stack.is_empty() => { + self.pos += 1; + return self.scan_fstring_format_spec_extent(start, outer_quote, outer_triple); + } + b'(' | b'[' | b'{' => { + stack.push(b); + self.pos += 1; + } + b')' | b']' | b'}' => { + let want = match b { + b')' => b'(', + b']' => b'[', + _ => b'{', + }; + match stack.last() { + Some(&open) if open == want => { + stack.pop(); + self.pos += 1; + } + // A close that doesn't match the innermost opener + // ("closing parenthesis 'X' does not match opening + // parenthesis 'Y'"). + Some(&open) => { + return Err(LexError::FstringParenMismatch { + close: b as char, + open: open as char, + pos: self.pos as u32, + }) + } + // A `)`/`]` with nothing open ("f-string: unmatched + // 'X'"). A top-level `}` was the field terminator, + // already handled above. + None => { + return Err(LexError::FstringUnmatchedParen { + close: b as char, + pos: self.pos as u32, + }) + } + } + } + b'"' | b'\'' => self.scan_fstring_nested_string(outer_quote)?, + // In the *expression* part, `#` starts a comment to end + // of line (only meaningful in multiline fields). A comment + // terminated by a newline resumes normal scanning; one that + // reaches EOF leaves the innermost bracket "never closed". + b'#' => { + in_comment = true; + while let Some(c) = self.peek() { + if c == b'\n' { + in_comment = false; + break; + } + self.pos += 1; + } + } + _ => self.pos += 1, + } + } + } + + /// Scan a format spec from just past the field's top-level `:` to the + /// closing `}`. The spec is literal text except that `{` opens a + /// nested replacement field (its own expression) — so `#`, quotes and + /// `:` here are *not* special. + fn scan_fstring_format_spec_extent( + &mut self, + start: usize, + outer_quote: u8, + outer_triple: bool, + ) -> Result<(), LexError> { + loop { + let Some(b) = self.peek() else { + // Spec ran to EOF with the field still open. CPython's spec + // diagnostic names the spec too: "expecting '}', or format + // specs" (vs the plain "expecting '}'" for the expr part). + return Err(LexError::FstringExpectingBraceOrSpec { pos: start as u32 }); + }; + match b { + b'}' => { + self.pos += 1; + return Ok(()); + } + b'{' => { + self.pos += 1; + self.scan_fstring_field_extent(start, outer_quote, outer_triple)?; + } + // The spec is literal text, so the *outer* quote here is the + // f-string's own terminator (a quote-as-fill must use the + // other quote, e.g. `f"{x:'>10}"`). Reaching it means the + // field never closed: "expecting '}', or format specs". + _ if b == outer_quote => { + return Err(LexError::FstringExpectingBraceOrSpec { pos: self.pos as u32 }); + } + // A literal newline in the spec is only legal inside a + // triple-quoted f-string; in a single-line one CPython + // raises the "newlines are not allowed in format + // specifiers..." error. (Newlines reached *inside* a nested + // `{...}` field are consumed by the recursion above.) + b'\n' | b'\r' if !outer_triple => { + return Err(LexError::FstringNewlineInSpec { pos: self.pos as u32 }); + } + _ => self.pos += 1, + } + } + } + + /// Skip a nested string literal that appears inside a replacement + /// field. Detects an immediately-preceding string prefix so a nested + /// f-string recurses (its own fields may reuse the outer quote). + fn scan_fstring_nested_string(&mut self, outer_quote: u8) -> Result<(), LexError> { + let quote = self.peek().expect("nested string at quote"); + let triple = self.peek_at(1) == Some(quote) && self.peek_at(2) == Some(quote); + // When a lone quote *matching the enclosing f-string's* quote can't + // form a complete string (runs to EOF unpaired), it was never a + // nested string — it's the f-string's own terminator, and the field + // is what's unterminated. CPython surfaces "f-string: expecting '}'", + // not "unterminated string literal". (`f'{3'` vs the valid `f'{3''}'` + // empty string, or `f'{3 + 'a'}'` which finds its pair.) + let unterminated = |pos: u32| { + if quote == outer_quote { + LexError::FstringExpectingBrace { pos } + } else { + LexError::UnterminatedString { pos } + } + }; + // Walk back over the immediately-preceding ASCII-letter run to + // recover any prefix (`f`, `r`, `rb`, ...). It's a real prefix + // only when not glued to a longer identifier. + let mut s = self.pos; + while s > 0 && self.src[s - 1].is_ascii_alphabetic() { + s -= 1; + } + let glued_to_ident = + s > 0 && (self.src[s - 1] == b'_' || self.src[s - 1].is_ascii_digit()); + let prefix = if !glued_to_ident && s < self.pos { + std::str::from_utf8(&self.src[s..self.pos]) + .ok() + .and_then(StringPrefix::parse) + .unwrap_or_default() + } else { + StringPrefix::default() + }; + if triple { + self.pos += 3; + } else { + self.pos += 1; + } + if prefix.fstring { + return self.scan_fstring_extent(self.pos, quote, triple, prefix.raw); + } + let _ = prefix.raw; + loop { + let Some(b) = self.peek() else { + return Err(unterminated(self.pos as u32)); + }; + if b == b'\\' { + // A backslash escapes the next byte for tokenizing in raw + // and non-raw strings alike (raw-ness only changes decode). + self.pos += 1; + if self.peek().is_some() { + self.pos += 1; + } + continue; + } + if b == quote { + if triple { + if self.peek_at(1) == Some(quote) && self.peek_at(2) == Some(quote) { + self.pos += 3; + return Ok(()); + } + self.pos += 1; + continue; + } + self.pos += 1; + return Ok(()); + } + if (b == b'\n' || b == b'\r') && !triple { + return Err(unterminated(self.pos as u32)); + } + self.pos += 1; + } + } + fn scan_single_line_string( &mut self, start: usize, @@ -450,11 +875,15 @@ impl<'src> Scanner<'src> { prefix: StringPrefix, ) -> Result { let raw = prefix.raw; + let mut warned = false; while let Some(b) = self.peek() { if b == b'\n' || b == b'\r' { return Err(LexError::UnterminatedString { pos: start as u32 }); } if b == b'\\' && !raw { + if !warned { + warned = self.note_invalid_escape(self.pos, prefix.bytes); + } // Skip the backslash and one following byte (the escape). self.pos += 1; if let Some(next) = self.peek() { @@ -501,11 +930,18 @@ impl<'src> Scanner<'src> { prefix: StringPrefix, ) -> Result { let raw = prefix.raw; + let mut warned = false; loop { let Some(b) = self.peek() else { return Err(LexError::UnterminatedString { pos: start as u32 }); }; - if b == b'\\' && !raw { + if b == b'\\' { + // Backslash escapes the next byte for tokenizing in raw + // and non-raw triple strings alike (a raw `\"""` therefore + // does not close the literal); decode handles raw-ness. + if !raw && !warned { + warned = self.note_invalid_escape(self.pos, prefix.bytes); + } self.pos += 1; if self.peek().is_some() { self.pos += 1; diff --git a/crates/weavepy-lexer/src/token.rs b/crates/weavepy-lexer/src/token.rs index 95c2e0f..38d68ed 100644 --- a/crates/weavepy-lexer/src/token.rs +++ b/crates/weavepy-lexer/src/token.rs @@ -41,6 +41,22 @@ impl Span { } } +/// A deferred compile-time diagnostic discovered while scanning a string +/// or bytes literal: CPython's invalid-escape and oversized-octal-escape +/// `SyntaxWarning`s (e.g. `invalid escape sequence '\z'`). +/// +/// The tokenizer detects these (matching CPython, which warns from the +/// tokenizer/parser) but cannot emit them — that needs the runtime +/// `warnings` machinery. They are surfaced to the compile path, which +/// replays them through `warnings.warn_explicit`; an active `error` +/// filter then turns them into `SyntaxError`s. `offset` is the absolute +/// byte offset of the offending backslash within the source buffer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EscapeWarning { + pub offset: u32, + pub message: String, +} + /// The lexical category of a token. /// /// Operator and punctuation variants are distinct so parser dispatch @@ -318,8 +334,14 @@ impl StringPrefix { _ => return None, } } - // CPython rejects bytes + unicode, bytes + f, f + u combinations. - if (p.bytes && p.unicode) || (p.bytes && p.fstring) || (p.fstring && p.unicode) { + // CPython rejects every combination of the `u` prefix with another + // marker (`ur`, `ru`, `bu`, `fu`) and of bytes with `f`. The `u` + // prefix is only valid standing alone (kept for Py2 source compat). + if (p.bytes && p.unicode) + || (p.bytes && p.fstring) + || (p.fstring && p.unicode) + || (p.raw && p.unicode) + { return None; } Some(p) diff --git a/crates/weavepy-parser/Cargo.toml b/crates/weavepy-parser/Cargo.toml index d1820d6..be0f135 100644 --- a/crates/weavepy-parser/Cargo.toml +++ b/crates/weavepy-parser/Cargo.toml @@ -22,5 +22,9 @@ num-bigint = { workspace = true } # full UCD name table shipped by this crate (CPython parity). unicode_names2 = { workspace = true } +# PEP 3131: identifiers are NFKC-normalized at parse time, so `µ` (U+00B5) +# and `μ` (U+03BC) name the same binding and `𝔘𝔫𝔦𝔠𝔬𝔡𝔢` folds to `Unicode`. +unicode-normalization = { workspace = true } + [lints] workspace = true diff --git a/crates/weavepy-parser/src/error.rs b/crates/weavepy-parser/src/error.rs index 752379b..6f7353a 100644 --- a/crates/weavepy-parser/src/error.rs +++ b/crates/weavepy-parser/src/error.rs @@ -17,3 +17,30 @@ pub enum ParseError { rfc: &'static str, }, } + +impl ParseError { + /// Byte offset into the source where the error was detected. Drives + /// the `SyntaxError` `lineno`/`offset` computed at the raise site. + pub fn byte_offset(&self) -> u32 { + match self { + ParseError::Lex(e) => e.byte_offset(), + ParseError::Unexpected { span, .. } | ParseError::NotImplemented { span, .. } => { + span.start.0 + } + } + } + + /// The bare message for a CPython-shaped `SyntaxError.msg`, without + /// the `"lexical error: "` wrapper that the [`Display`] impl adds for + /// diagnostics. For a lexer error this is exactly CPython's text + /// (e.g. `invalid character '€' (U+20AC)`). + /// + /// [`Display`]: std::fmt::Display + pub fn syntax_message(&self) -> String { + match self { + ParseError::Lex(e) => e.to_string(), + ParseError::Unexpected { message, .. } => message.clone(), + ParseError::NotImplemented { .. } => self.to_string(), + } + } +} diff --git a/crates/weavepy-parser/src/lib.rs b/crates/weavepy-parser/src/lib.rs index fda9d0b..74a40d4 100644 --- a/crates/weavepy-parser/src/lib.rs +++ b/crates/weavepy-parser/src/lib.rs @@ -19,11 +19,28 @@ mod parser; pub use ast::{dump_module, Module}; pub use error::ParseError; +pub use weavepy_lexer::EscapeWarning; /// Parse a Python source buffer into a [`Module`]. pub fn parse_module(source: &str) -> Result { - let tokens = weavepy_lexer::tokenize(source)?; - parser::parse(source, tokens) + parse_module_with_warnings(source).0 +} + +/// Like [`parse_module`], but also returns the deferred [`EscapeWarning`]s +/// the tokenizer found in string/bytes literals. +/// +/// Warnings are returned on **both** the success and error paths: an +/// invalid escape in an earlier literal must still surface (as a +/// `SyntaxWarning`, or a `SyntaxError` under an `error` filter) even when +/// a later token fails to lex/parse — e.g. `eval("'\\e' $")`. The VM +/// replays the warnings before propagating any parse error. +pub fn parse_module_with_warnings(source: &str) -> (Result, Vec) { + let (tok_result, warnings) = weavepy_lexer::tokenize_with_escapes(source); + let module = match tok_result { + Ok(tokens) => parser::parse(source, tokens), + Err(e) => Err(ParseError::from(e)), + }; + (module, warnings) } #[cfg(test)] diff --git a/crates/weavepy-parser/src/parser.rs b/crates/weavepy-parser/src/parser.rs index e1243b7..0103d78 100644 --- a/crates/weavepy-parser/src/parser.rs +++ b/crates/weavepy-parser/src/parser.rs @@ -55,6 +55,26 @@ impl<'src> Parser<'src> { &self.source[span.start.0 as usize..span.end.0 as usize] } + /// Identifier text for a NAME token, NFKC-normalized per PEP 3131. + /// + /// CPython normalizes every identifier to Normalization Form KC at + /// parse time (`unicodeobject.c: _PyUnicode_TransformDecimalAndSpaceToASCII` + /// → `compile.c`/`tokenizer` actually use `PyUnicode_FromString` + + /// `unicodedata.normalize('NFKC', …)`), so `µ` (U+00B5) and `μ` + /// (U+03BC) bind the same name, and the mathematical-alphabet + /// `𝔘𝔫𝔦𝔠𝔬𝔡𝔢` folds to plain `Unicode`. ASCII identifiers — the + /// overwhelmingly common case — are already in NFKC, so we return the + /// borrowed slice without touching the normalizer. + fn ident(&self, span: Span) -> String { + let raw = self.lexeme(span); + if raw.is_ascii() { + raw.to_owned() + } else { + use unicode_normalization::UnicodeNormalization; + raw.nfkc().collect() + } + } + fn peek(&self) -> &TokenKind { &self.tokens[self.pos].kind } @@ -251,7 +271,7 @@ impl<'src> Parser<'src> { fn parse_type_alias_stmt(&mut self) -> Result { let type_tok = self.bump(); // `type` let name_tok = self.expect(&TokenKind::Name, "type alias name")?; - let name = self.lexeme(name_tok.span).to_owned(); + let name = self.ident(name_tok.span); let type_params = self.collect_pep695_type_params()?; self.expect(&TokenKind::Equal, "`=`")?; let value = self.parse_expression_list(true)?; @@ -293,7 +313,7 @@ impl<'src> Parser<'src> { break; } let name_tok = self.expect(&TokenKind::Name, "type parameter name")?; - names.push(self.lexeme(name_tok.span).to_owned()); + names.push(self.ident(name_tok.span)); // Skip optional `: bound` and `= default`. if matches!(self.peek(), TokenKind::Colon) { self.bump(); @@ -453,9 +473,12 @@ impl<'src> Parser<'src> { } Ok(()) } - other => Err(ParseError::Unexpected { + // Leftover tokens after an otherwise-complete statement are + // CPython's catch-all "invalid syntax" (e.g. `1 2`, or a bad + // string prefix like `fu''` which tokenises as NAME + STRING). + _ => Err(ParseError::Unexpected { span: self.peek_token().span, - message: format!("expected end of statement, got {other:?}"), + message: "invalid syntax".to_owned(), }), } } @@ -562,7 +585,7 @@ impl<'src> Parser<'src> { fn parse_function_def(&mut self, decorator_list: Vec) -> Result { let def_tok = self.bump(); // `def` let name_tok = self.expect(&TokenKind::Name, "function name")?; - let name = self.lexeme(name_tok.span).to_owned(); + let name = self.ident(name_tok.span); // PEP 695: optional `[T, *Ts, **P]` type-parameter list. // Consumed-and-discarded for now — the names are real `TypeVar`-shaped // objects in CPython, but the parser tolerates the syntax so generic @@ -669,7 +692,7 @@ impl<'src> Parser<'src> { fn parse_class_def(&mut self, decorator_list: Vec) -> Result { let class_tok = self.bump(); // `class` let name_tok = self.expect(&TokenKind::Name, "class name")?; - let name = self.lexeme(name_tok.span).to_owned(); + let name = self.ident(name_tok.span); // PEP 695: optional `[T, *Ts, **P]` type-parameter list (same as // function form). self.skip_pep695_type_params()?; @@ -733,7 +756,7 @@ impl<'src> Parser<'src> { let n = if self.at_keyword(Keyword::As) { self.bump(); let nt = self.expect(&TokenKind::Name, "name after `as`")?; - Some(self.lexeme(nt.span).to_owned()) + Some(self.ident(nt.span)) } else { None }; @@ -904,6 +927,10 @@ impl<'src> Parser<'src> { // 2 = keyword-only (after `*`) let mut phase = 0u8; let mut had_default = false; + // A bare `*` separator (no `*args` name) requires at least one + // keyword-only argument to follow it; CPython rejects `def f(p, *)` + // and `def f(p, *, **kw)` with "named arguments must follow bare *". + let mut bare_star_span: Option = None; loop { if self.check(&TokenKind::RPar) || self.check(&TokenKind::Colon) { break; @@ -913,10 +940,12 @@ impl<'src> Parser<'src> { if matches!(self.peek(), TokenKind::Name) { let n = self.bump(); args.vararg = Some(Arg { - name: self.lexeme(n.span).to_owned(), + name: self.ident(n.span), annotation: self.try_arg_annotation(allow_annotation)?, span: n.span, }); + } else { + bare_star_span = Some(self.peek_token().span); } phase = 2; if !self.eat(&TokenKind::Comma) { @@ -928,7 +957,7 @@ impl<'src> Parser<'src> { if self.eat(&TokenKind::DoubleStar) { let n = self.expect(&TokenKind::Name, "kwarg name")?; args.kwarg = Some(Arg { - name: self.lexeme(n.span).to_owned(), + name: self.ident(n.span), annotation: self.try_arg_annotation(allow_annotation)?, span: n.span, }); @@ -948,7 +977,7 @@ impl<'src> Parser<'src> { } let n = self.expect(&TokenKind::Name, "parameter name")?; - let name = self.lexeme(n.span).to_owned(); + let name = self.ident(n.span); let annotation = self.try_arg_annotation(allow_annotation)?; let default = if self.eat(&TokenKind::Equal) { Some(self.parse_expression(false)?) @@ -980,6 +1009,38 @@ impl<'src> Parser<'src> { break; } } + // A bare `*` must be followed by at least one keyword-only + // argument (CPython: "named arguments must follow bare *"). + if let Some(span) = bare_star_span { + if args.kwonlyargs.is_empty() { + return Err(ParseError::Unexpected { + span, + message: "named arguments must follow bare *".to_owned(), + }); + } + } + // No parameter name may repeat across any section + // (positional-only, positional-or-keyword, `*args`, keyword-only, + // `**kwargs`) — CPython raises "duplicate argument '' in + // function definition". + let mut seen: Vec<&str> = Vec::new(); + let dup_span = |span: Span, name: &str| ParseError::Unexpected { + span, + message: format!("duplicate argument '{name}' in function definition"), + }; + for a in args + .posonlyargs + .iter() + .chain(args.args.iter()) + .chain(args.vararg.iter()) + .chain(args.kwonlyargs.iter()) + .chain(args.kwarg.iter()) + { + if seen.contains(&a.name.as_str()) { + return Err(dup_span(a.span, &a.name)); + } + seen.push(a.name.as_str()); + } Ok(args) } @@ -1104,7 +1165,7 @@ impl<'src> Parser<'src> { let asname = if self.at_keyword(Keyword::As) { self.bump(); let n = self.expect(&TokenKind::Name, "name after `as`")?; - Some(self.lexeme(n.span).to_owned()) + Some(self.ident(n.span)) } else { None }; @@ -1125,11 +1186,11 @@ impl<'src> Parser<'src> { fn parse_dotted_name(&mut self) -> Result { let first = self.expect(&TokenKind::Name, "module name")?; - let mut out = self.lexeme(first.span).to_owned(); + let mut out = self.ident(first.span); while self.eat(&TokenKind::Dot) { let n = self.expect(&TokenKind::Name, "name after `.`")?; out.push('.'); - out.push_str(self.lexeme(n.span)); + out.push_str(&self.ident(n.span)); } Ok(out) } @@ -1167,11 +1228,11 @@ impl<'src> Parser<'src> { break; } let n = self.expect(&TokenKind::Name, "imported name")?; - let name = self.lexeme(n.span).to_owned(); + let name = self.ident(n.span); let asname = if self.at_keyword(Keyword::As) { self.bump(); let n2 = self.expect(&TokenKind::Name, "name after `as`")?; - Some(self.lexeme(n2.span).to_owned()) + Some(self.ident(n2.span)) } else { None }; @@ -1263,7 +1324,7 @@ impl<'src> Parser<'src> { let mut names = Vec::new(); loop { let n = self.expect(&TokenKind::Name, "name")?; - names.push(self.lexeme(n.span).to_owned()); + names.push(self.ident(n.span)); if !self.eat(&TokenKind::Comma) { break; } @@ -1362,7 +1423,7 @@ impl<'src> Parser<'src> { if self.at_keyword(Keyword::As) { self.bump(); let n = self.expect(&TokenKind::Name, "name after `as`")?; - let name = self.lexeme(n.span).to_owned(); + let name = self.ident(n.span); if name == "_" { return Err(ParseError::Unexpected { span: n.span, @@ -1399,7 +1460,7 @@ impl<'src> Parser<'src> { let name = match self.peek() { TokenKind::Name => { let tok = self.bump(); - let s = self.lexeme(tok.span).to_owned(); + let s = self.ident(tok.span); if s == "_" { None } else { @@ -1500,7 +1561,7 @@ impl<'src> Parser<'src> { /// value pattern; `(` makes it a class pattern; otherwise capture. fn parse_name_pattern(&mut self) -> Result { let first = self.bump(); - let first_name = self.lexeme(first.span).to_owned(); + let first_name = self.ident(first.span); // Dotted: value pattern. if self.check(&TokenKind::Dot) { let mut expr = Expr { @@ -1509,7 +1570,7 @@ impl<'src> Parser<'src> { }; while self.eat(&TokenKind::Dot) { let n = self.expect(&TokenKind::Name, "attribute name in pattern")?; - let attr = self.lexeme(n.span).to_owned(); + let attr = self.ident(n.span); let span = expr.span.merge(n.span); expr = Expr { kind: ExprKind::Attribute { @@ -1550,7 +1611,7 @@ impl<'src> Parser<'src> { && matches!(self.peek_at(1), Some(TokenKind::Equal)) { let n = self.bump(); - let name = self.lexeme(n.span).to_owned(); + let name = self.ident(n.span); self.bump(); // `=` let p = self.parse_pattern()?; keywords.push((name, p)); @@ -1625,7 +1686,7 @@ impl<'src> Parser<'src> { while !self.check(&TokenKind::RBrace) { if self.eat(&TokenKind::DoubleStar) { let n = self.expect(&TokenKind::Name, "name after `**` in mapping pattern")?; - let name = self.lexeme(n.span).to_owned(); + let name = self.ident(n.span); rest = Some(if name == "_" { None } else { Some(name) }); if !self.eat(&TokenKind::Comma) { break; @@ -1682,12 +1743,12 @@ impl<'src> Parser<'src> { // Dotted name as a value key. let n = self.expect(&TokenKind::Name, "key")?; let mut expr = Expr { - kind: ExprKind::Name(self.lexeme(n.span).to_owned()), + kind: ExprKind::Name(self.ident(n.span)), span: n.span, }; while self.eat(&TokenKind::Dot) { let attr_tok = self.expect(&TokenKind::Name, "attribute name in key")?; - let attr = self.lexeme(attr_tok.span).to_owned(); + let attr = self.ident(attr_tok.span); let span = expr.span.merge(attr_tok.span); expr = Expr { kind: ExprKind::Attribute { @@ -1725,9 +1786,39 @@ impl<'src> Parser<'src> { } Ok(body) } else { - // Inline single-statement block: `if x: y = 1`, `class A: pass`. - let s = self.parse_statement()?; - Ok(vec![s]) + // Inline suite after `:` — Python's `simple_stmt`: + // small_stmt (';' small_stmt)* [';'] NEWLINE + // e.g. `if x: y = 1`, `class A: pass`, and crucially the + // multi-statement form `def f(): a = 1; return a`. We used to + // parse only the first statement, leaving `; return a` to be + // re-parsed by the *enclosing* scope — which then rejected the + // `return` as "outside function". Each `parse_statement` + // consumes its own terminator (`;` or NEWLINE via + // `consume_stmt_end`), so we keep going while that terminator + // was a `;` and another small statement follows on the line. + let mut body = Vec::new(); + loop { + body.push(self.parse_statement()?); + let ended_with_semi = matches!( + self.tokens.get(self.pos.wrapping_sub(1)).map(|t| &t.kind), + Some(TokenKind::Semi) + ); + if !ended_with_semi { + break; + } + // A trailing `;` right before the line break (`a = 1;`) + // ends the suite; consume the closing NEWLINE so the + // caller resumes from a clean statement boundary. + match self.peek() { + TokenKind::Newline => { + self.bump(); + break; + } + TokenKind::Endmarker | TokenKind::Dedent => break, + _ => {} + } + } + Ok(body) } } @@ -1818,7 +1909,7 @@ impl<'src> Parser<'src> { if let Some(next) = self.tokens.get(self.pos + 1) { if matches!(next.kind, TokenKind::ColonEqual) { let name_tok = self.peek_token().clone(); - let name = self.lexeme(name_tok.span).to_owned(); + let name = self.ident(name_tok.span); self.bump(); // name self.bump(); // := let value = self.parse_ternary()?; @@ -2250,7 +2341,7 @@ impl<'src> Parser<'src> { TokenKind::Dot => { self.bump(); let n = self.expect(&TokenKind::Name, "attribute name")?; - let attr = self.lexeme(n.span).to_owned(); + let attr = self.ident(n.span); let span = base.span.merge(n.span); base = Expr { kind: ExprKind::Attribute { @@ -2272,9 +2363,16 @@ impl<'src> Parser<'src> { if self.check(&TokenKind::RPar) { return Ok((args, keywords)); } + // Track keyword state so we can reject a plain positional argument + // that follows a keyword (CPython: "positional argument follows + // keyword argument") and a repeated keyword name (CPython: + // "keyword argument repeated: "). + let mut seen_keyword = false; + let mut kw_names: Vec = Vec::new(); loop { if self.eat(&TokenKind::DoubleStar) { let val = self.parse_ternary()?; + seen_keyword = true; keywords.push(KwArg { arg: None, value: val, @@ -2292,14 +2390,28 @@ impl<'src> Parser<'src> { && matches!(self.peek_at(1), Some(TokenKind::Equal)) { let nt = self.bump(); - let name = self.lexeme(nt.span).to_owned(); + let name = self.ident(nt.span); + if kw_names.contains(&name) { + return Err(ParseError::Unexpected { + span: nt.span, + message: format!("keyword argument repeated: {name}"), + }); + } self.bump(); // `=` let val = self.parse_ternary()?; + seen_keyword = true; + kw_names.push(name.clone()); keywords.push(KwArg { arg: Some(name), value: val, }); } else { + if seen_keyword { + return Err(ParseError::Unexpected { + span: self.peek_token().span, + message: "positional argument follows keyword argument".to_owned(), + }); + } let e = self.parse_ternary()?; // Generator expression as single argument: `f(x for x in xs)`. if (self.at_keyword(Keyword::For) || self.at_keyword(Keyword::Async)) @@ -2442,7 +2554,7 @@ impl<'src> Parser<'src> { TokenKind::Name => { self.bump(); Ok(Expr { - kind: ExprKind::Name(self.lexeme(tok.span).to_owned()), + kind: ExprKind::Name(self.ident(tok.span)), span: tok.span, }) } @@ -2982,6 +3094,42 @@ impl<'src> Parser<'src> { let mut i = 0usize; while i < bytes.len() { let b = bytes[i]; + // Non-raw backslash escapes are copied into the literal as a + // unit so the decoder interprets them — and, crucially, so an + // escaped backslash (`\\`) can't have its second byte misread + // as the start of a new escape (e.g. `f'\\N{AMPERSAND}'` is a + // literal `\` then the field `{AMPERSAND}`, not `\N{...}`). + if b == b'\\' && !raw { + // `\N{NAME}` named-character escape: the brace group is + // the Unicode character name, not a replacement field. + if bytes.get(i + 1) == Some(&b'N') && bytes.get(i + 2) == Some(&b'{') { + let mut j = i + 3; + while j < bytes.len() && bytes[j] != b'}' { + j += 1; + } + if j < bytes.len() { + j += 1; // include the closing `}` + } + literal.push_str(&body[i..j]); + i = j; + continue; + } + // Any other escape: copy the backslash, then its escaped + // character — except `{`/`}`, which stay structural (a + // lone `\` before a brace is a literal backslash followed + // by a replacement field / brace escape, e.g. `\{6*7}`). + literal.push('\\'); + i += 1; + if let Some(&n) = bytes.get(i) { + if n != b'{' && n != b'}' { + let ch_len = utf8_char_len(n); + let end = (i + ch_len).min(bytes.len()); + literal.push_str(&body[i..end]); + i = end; + } + } + continue; + } if b == b'{' { if i + 1 < bytes.len() && bytes[i + 1] == b'{' { literal.push('{'); @@ -3014,7 +3162,7 @@ impl<'src> Parser<'src> { } return Err(ParseError::Unexpected { span: anchor, - message: "single '}' is not allowed in f-string".to_owned(), + message: "f-string: single '}' is not allowed".to_owned(), }); } // Append the next UTF-8 character (one or more bytes). @@ -3036,8 +3184,16 @@ impl<'src> Parser<'src> { Ok(parts) } - /// Scan from just past the opening `{` to the matching `}` at depth 0. - /// Returns the field text and the index of the closing `}`. + /// Scan from just past the opening `{` to the matching `}` at the + /// field's top level. Returns the field text and the index of that + /// closing `}`. + /// + /// Bracket nesting is tracked with an explicit stack so we can report + /// CPython's PEP 701 diagnostics: a `)`/`]`/`}` that doesn't match the + /// innermost opener yields "closing parenthesis 'X' does not match + /// opening parenthesis 'Y'", a `)`/`]` with nothing open yields + /// "f-string: unmatched ')'", and running off the end yields + /// "f-string: expecting '}'". fn scan_fstring_field( &self, body: &str, @@ -3045,11 +3201,18 @@ impl<'src> Parser<'src> { anchor: Span, ) -> Result<(String, usize), ParseError> { let bytes = body.as_bytes(); - let mut depth = 1i32; + // Openers seen *inside* the field (the field's own `{` is implicit + // and not pushed); a top-level `}` closes the field. + let mut stack: Vec = Vec::new(); let mut i = start; - // String state machine for backtick-free quotes inside the field. + // String state machine for quotes inside the field. let mut in_str: Option = None; let mut triple = false; + // Once the top-level `:` is seen we're in the format spec, where + // `#` is literal (e.g. `{x:#06x}`); before it, in the expression + // part, `#` starts a comment to end of line (legal in multi-line + // f-strings, PEP 701). + let mut in_spec = false; while i < bytes.len() { let b = bytes[i]; if let Some(q) = in_str { @@ -3088,26 +3251,60 @@ impl<'src> Parser<'src> { } } b'(' | b'[' | b'{' => { - depth += 1; + stack.push(b); i += 1; } - b')' | b']' => { - depth -= 1; + b')' => match stack.last() { + Some(b'(') => { + stack.pop(); + i += 1; + } + Some(&open) => return Err(fstring_paren_mismatch(')', open, anchor)), + None => { + return Err(ParseError::Unexpected { + span: anchor, + message: "f-string: unmatched ')'".to_owned(), + }) + } + }, + b']' => match stack.last() { + Some(b'[') => { + stack.pop(); + i += 1; + } + Some(&open) => return Err(fstring_paren_mismatch(']', open, anchor)), + None => { + return Err(ParseError::Unexpected { + span: anchor, + message: "f-string: unmatched ']'".to_owned(), + }) + } + }, + b'}' => match stack.last() { + None => return Ok((body[start..i].to_owned(), i)), + Some(b'{') => { + stack.pop(); + i += 1; + } + Some(&open) => return Err(fstring_paren_mismatch('}', open, anchor)), + }, + b':' if stack.is_empty() && !in_spec => { + in_spec = true; i += 1; } - b'}' => { - if depth == 1 { - return Ok((body[start..i].to_owned(), i)); + b'#' if !in_spec => { + // Comment to end of line; the brackets/quotes it may + // contain must not perturb depth or string tracking. + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; } - depth -= 1; - i += 1; } _ => i += 1, } } Err(ParseError::Unexpected { span: anchor, - message: "expected '}' to close f-string replacement field".to_owned(), + message: "f-string: expecting '}'".to_owned(), }) } @@ -3115,16 +3312,11 @@ impl<'src> Parser<'src> { /// `FormattedValue` (possibly preceded by a synthetic literal /// for `{x = }` debug form). fn parse_fstring_field(&self, field: &str, anchor: Span) -> Result { - // Split into expr, conversion, format_spec. Backslashes inside - // an f-string field aren't allowed in CPython <3.12 — we - // surface that as a parse error for clarity. - if field.contains('\\') { - return Err(ParseError::NotImplemented { - span: anchor, - feature: "backslashes inside f-string replacement fields", - rfc: "RFC 0005-B", - }); - } + // PEP 701 (3.12+): backslashes *are* allowed inside replacement + // fields (e.g. `f"{d["a\tb"]}"`). The expression is re-tokenized + // below, so escapes inside nested string literals are handled by + // the sub-lexer; a stray backslash in the expression itself just + // surfaces as a normal sub-parse error. let bytes = field.as_bytes(); // Find the `!conv` and `:spec` boundaries at top level (not // inside nested parens/brackets/braces or string literals). @@ -3137,6 +3329,20 @@ impl<'src> Parser<'src> { let mut i = 0; while i < bytes.len() { let b = bytes[i]; + // A `#` in the expression part (before any `!conv`/`:spec`, + // and not inside a string) is a comment to end of line. Skip + // it so quotes/`!`/`:` it contains can't be mistaken for + // string delimiters or conv/spec boundaries. + if in_str.is_none() + && b == b'#' + && conv_start.is_none() + && spec_start.is_none() + { + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; + } + continue; + } if let Some(q) = in_str { if b == q { if triple { @@ -3175,12 +3381,16 @@ impl<'src> Parser<'src> { } if depth == 0 { if b == b'!' && conv_start.is_none() && spec_start.is_none() { - // `!=` and `!<` etc. are comparison; `!` followed - // by `s` / `r` / `a` is conversion. - if i + 1 < bytes.len() && matches!(bytes[i + 1], b's' | b'r' | b'a') { + // `!=` is the only `!` that stays part of the + // expression (comparison); any other `!` ends the + // expression and opens the conversion clause. Catching + // it here (rather than only before `s`/`r`/`a`) lets an + // empty expression before `!` surface CPython's + // "valid expression required before '!'". + if bytes.get(i + 1) != Some(&b'=') { expr_end = i; conv_start = Some(i + 1); - i += 2; + i += 1; continue; } } else if b == b':' && spec_start.is_none() { @@ -3192,39 +3402,122 @@ impl<'src> Parser<'src> { } i += 1; } - let expr_text = &field[..expr_end]; - // Debug form `{x = }`: literal "x = " prepended, conversion - // forced to `r` if no explicit conversion / spec. - let (expr_text, debug_lit) = if expr_text.trim_end().ends_with('=') { - let trimmed = expr_text.trim_end(); - let without_eq = trimmed.trim_end_matches('='); - let literal = format!("{without_eq}="); - (without_eq.trim(), Some(literal)) + let expr_slice = &field[..expr_end]; + // Debug form `{expr=}`: CPython echoes the *verbatim* source of + // the expression part (preserving the author's whitespace, e.g. + // `{val = }` -> "val = 7") and then formats the value. A trailing + // single `=` triggers it, but `==`/`!=`/`<=`/`>=` must not. + // + // PEP 701 allows `#` comments inside (multi-line) replacement + // fields, e.g. + // f"{1+2 = # my comment + // }" == '1+2 = \n 3' + // The comment is removed but the surrounding whitespace stays, and + // it must not hide the debug `=`. Strip comments first, then both + // the detection and the echoed literal work on the cleaned text. + let clean = strip_fstring_field_comments(expr_slice); + // Only ASCII whitespace is insignificant around the expression + // (space, tab, formfeed, CR/LF, VT). Notably *not* U+00A0 etc. — + // CPython rejects those as "invalid non-printable character". + let ws = |c: char| matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c'); + let trimmed_end = clean.trim_end_matches(ws); + let is_debug = trimmed_end.ends_with('=') + && !trimmed_end.ends_with("==") + && !trimmed_end.ends_with("!=") + && !trimmed_end.ends_with("<=") + && !trimmed_end.ends_with(">="); + let (expr_text, debug_lit) = if is_debug { + let value_src = trimmed_end[..trimmed_end.len() - 1].trim_matches(ws); + // Verbatim expression-part slice (through the `=`, including + // any surrounding spaces, comments removed) is echoed. + (value_src, Some(clean.clone())) } else { - (expr_text.trim(), None) + (clean.trim_matches(ws), None) }; if expr_text.is_empty() { + // Name the terminator that followed the (empty) expression, + // mirroring CPython: "f-string: valid expression required + // before '}'/'!'/':'/'='". + let before = if is_debug { + '=' + } else if conv_start.is_some() { + '!' + } else if spec_start.is_some() { + ':' + } else { + '}' + }; return Err(ParseError::Unexpected { span: anchor, - message: "empty expression in f-string replacement field".to_owned(), + message: format!("f-string: valid expression required before '{before}'"), }); } - // Recursively tokenize+parse the expression. - let tokens = weavepy_lexer::tokenize(expr_text)?; - let mut sub = Parser::new(expr_text, tokens); - sub.skip_trivia_and_newlines(); - let value = sub.parse_expression_list(false)?; - sub.skip_trivia_and_newlines(); - if !matches!(sub.peek(), TokenKind::Endmarker) { + // A field whose expression can't even *begin* (a leading `,`, or a + // `.` not starting a float) is CPython's "expecting a valid + // expression after '{'", distinct from a malformed-but-started + // expression (which is just "invalid syntax"). + if fstring_expr_cannot_start(expr_text) { return Err(ParseError::Unexpected { span: anchor, - message: "trailing tokens in f-string expression".to_owned(), + message: "f-string: expecting a valid expression after '{'".to_owned(), }); } + // Recursively tokenize+parse the expression. Inside an f-string + // replacement field, newlines, comments and indentation are + // insignificant (PEP 701: the field is parsed in the same + // implicit line-continuation context as the surrounding `{...}`), + // so a multi-line field like + // f'''{ + // 40 # forty + // + 2 # two + // }''' + // must read as `40 + 2`. Wrapping the expression in parentheses + // reproduces that joining exactly; the parens are transparent for + // a plain expression (and for a top-level comma the result is the + // same tuple `parse_expression_list` would have built). The + // closing paren goes on its own line so a trailing `# comment` in + // the field can't swallow it. + // Any failure parsing the embedded expression collapses to + // CPython's generic "invalid syntax" (the specific shapes it does + // name — empty expression, bad leading token, bracket mismatch — + // were already handled above / during the field scan). + let value = (|| -> Result { + let wrapped = format!("({expr_text}\n)"); + let tokens = weavepy_lexer::tokenize(&wrapped)?; + let mut sub = Parser::new(&wrapped, tokens); + sub.skip_trivia_and_newlines(); + let value = sub.parse_expression_list(false)?; + sub.skip_trivia_and_newlines(); + if !matches!(sub.peek(), TokenKind::Endmarker) { + return Err(ParseError::Unexpected { + span: anchor, + message: "trailing".to_owned(), + }); + } + Ok(value) + })() + .map_err(|_| ParseError::Unexpected { + span: anchor, + message: "invalid syntax".to_owned(), + })?; let conversion = match conv_start { - Some(idx) => i32::from(field.as_bytes()[idx]), - None if debug_lit.is_some() => i32::from(b'r'), + // A `!` with no following conversion char (e.g. `f'{a!}'`) is + // malformed; fall through to a generic error rather than + // indexing past the field. + Some(idx) => match field.as_bytes().get(idx) { + Some(&c) => i32::from(c), + None => { + return Err(ParseError::Unexpected { + span: anchor, + message: "f-string: expecting '}'".to_owned(), + }) + } + }, + // Debug form defaults to `!r`, but only when no explicit + // conversion *and* no format spec is given (`{x=:.2f}` uses + // the spec, not repr). + None if debug_lit.is_some() && spec_start.is_none() => i32::from(b'r'), None => -1, }; let format_spec = match spec_start { @@ -3281,6 +3574,117 @@ fn utf8_char_len(b: u8) -> usize { } } +/// Build CPython's "closing parenthesis 'X' does not match opening +/// parenthesis 'Y'" diagnostic for a mismatched bracket inside an f-string +/// replacement field. +fn fstring_paren_mismatch(close: char, open: u8, anchor: Span) -> ParseError { + ParseError::Unexpected { + span: anchor, + message: format!( + "closing parenthesis '{close}' does not match opening parenthesis '{}'", + open as char + ), + } +} + +/// True when an f-string replacement-field expression can't even begin — +/// i.e. it leads with a token that is never a valid expression start. We +/// only flag the cases CPython names distinctly with "expecting a valid +/// expression after '{'": a leading `,`, or a `.` that isn't the start of +/// a float literal (`.5`). Anything else that fails to parse is reported as +/// the generic "invalid syntax". +fn fstring_expr_cannot_start(expr: &str) -> bool { + let mut chars = expr.chars(); + match chars.next() { + Some(',') => true, + Some('.') => !matches!(chars.next(), Some(c) if c.is_ascii_digit()), + _ => false, + } +} + +/// Remove `#` comments from the expression part of an f-string replacement +/// field while leaving everything else (including whitespace and newlines) +/// byte-for-byte intact. PEP 701 permits comments inside multi-line fields; +/// a `#` only starts a comment outside of string literals, so this tracks +/// single/triple-quoted strings (and their backslash escapes) to avoid +/// mangling a `#` that lives inside a string. A comment runs to the next +/// newline (the newline itself is preserved). +fn strip_fstring_field_comments(s: &str) -> String { + let bytes = s.as_bytes(); + let mut out = String::with_capacity(s.len()); + let mut i = 0usize; + // `Some(quote)` while inside a string literal; `triple` tracks `"""`/`'''`. + let mut in_str: Option = None; + let mut triple = false; + while i < bytes.len() { + let b = bytes[i]; + if let Some(q) = in_str { + if b == b'\\' { + // Copy the backslash and its escaped char as a unit so an + // escaped quote can't be read as closing the string. + out.push('\\'); + i += 1; + if i < bytes.len() { + let cl = utf8_char_len(bytes[i]); + let e = (i + cl).min(bytes.len()); + out.push_str(&s[i..e]); + i = e; + } + continue; + } + if b == q { + if triple { + if i + 2 < bytes.len() && bytes[i + 1] == q && bytes[i + 2] == q { + out.push_str(&s[i..i + 3]); + i += 3; + in_str = None; + triple = false; + continue; + } + } else { + out.push(q as char); + i += 1; + in_str = None; + continue; + } + } + let cl = utf8_char_len(b); + let e = (i + cl).min(bytes.len()); + out.push_str(&s[i..e]); + i = e; + continue; + } + match b { + b'#' => { + // Drop the comment up to (but not including) the newline. + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; + } + } + b'"' | b'\'' => { + if i + 2 < bytes.len() && bytes[i + 1] == b && bytes[i + 2] == b { + in_str = Some(b); + triple = true; + out.push_str(&s[i..i + 3]); + i += 3; + } else { + in_str = Some(b); + triple = false; + out.push(b as char); + i += 1; + } + } + _ => { + let cl = utf8_char_len(b); + let e = (i + cl).min(bytes.len()); + out.push_str(&s[i..e]); + i = e; + } + } + } + out +} + /// Working state while concatenating adjacent string tokens. enum AccumString { Plain(String), @@ -3328,12 +3732,17 @@ fn strip_quotes(s: &str) -> &str { } } +/// Decode a (non-f) string-literal body. Returns the decoded text plus +/// any invalid-escape diagnostics CPython would surface as a +/// `SyntaxWarning` (unrecognised escapes and octal escapes `> \377`). +/// Each diagnostic carries the byte offset of its backslash *within the +/// body* so the caller can map it back to an absolute source position. fn decode_str_body(s: &str, raw: bool) -> Result { if raw { return Ok(s.to_owned()); } let mut out = String::with_capacity(s.len()); - let mut chars = s.chars(); + let mut chars = s.chars().peekable(); while let Some(c) = chars.next() { if c != '\\' { out.push(c); @@ -3350,7 +3759,23 @@ fn decode_str_body(s: &str, raw: bool) -> Result { '\\' => out.push('\\'), '\'' => out.push('\''), '"' => out.push('"'), - '0' => out.push('\0'), + // Octal escape `\ooo`: 1–3 octal digits (CPython accepts up + // to `\777` = 511 in a str literal). `\0` is just the + // zero-length-tail case of this rule. Values above `\377` + // draw a `SyntaxWarning`, detected by the lexer. + '0'..='7' => { + let mut val = esc as u32 - '0' as u32; + for _ in 0..2 { + match chars.peek().copied() { + Some(d @ '0'..='7') => { + val = val * 8 + (d as u32 - '0' as u32); + chars.next(); + } + _ => break, + } + } + out.push(char::from_u32(val).unwrap_or('\u{FFFD}')); + } 'a' => out.push('\x07'), 'b' => out.push('\x08'), 'f' => out.push('\x0c'), @@ -3366,7 +3791,8 @@ fn decode_str_body(s: &str, raw: bool) -> Result { 'u' => { let mut hex = String::new(); for _ in 0..4 { - hex.push(chars.next().ok_or("incomplete \\u escape")?); + let h = chars.next().ok_or("incomplete \\u escape")?; + hex.push(h); } let n = u32::from_str_radix(&hex, 16).map_err(|e| e.to_string())?; out.push(char::from_u32(n).unwrap_or('\u{FFFD}')); @@ -3377,7 +3803,8 @@ fn decode_str_body(s: &str, raw: bool) -> Result { // surrogate values, so we surface a clear error. let mut hex = String::new(); for _ in 0..8 { - hex.push(chars.next().ok_or("incomplete \\U escape")?); + let h = chars.next().ok_or("incomplete \\U escape")?; + hex.push(h); } let n = u32::from_str_radix(&hex, 16).map_err(|e| e.to_string())?; let ch = char::from_u32(n).ok_or_else(|| { @@ -3390,7 +3817,7 @@ fn decode_str_body(s: &str, raw: bool) -> Result { // the full UCD name table. CPython requires the brace form // and raises a SyntaxError ("malformed \N character escape" // / "unknown Unicode character name") otherwise. - if chars.next() != Some('{') { + if !matches!(chars.next(), Some('{')) { return Err("malformed \\N character escape".to_owned()); } let mut name = String::new(); @@ -3407,8 +3834,8 @@ fn decode_str_body(s: &str, raw: bool) -> Result { out.push(ch); } other => { - // CPython issues a DeprecationWarning for unknown - // escapes but emits both characters literally. + // CPython issues a `SyntaxWarning` for unknown escapes (the + // lexer records it) but emits both characters literally. out.push('\\'); out.push(other); } @@ -3417,12 +3844,25 @@ fn decode_str_body(s: &str, raw: bool) -> Result { Ok(out) } +/// Decode a bytes-literal body. Like [`decode_str_body`] but bytes-valued +/// and ASCII-only: a non-ASCII source character is a `SyntaxError` ("bytes +/// can only contain ASCII literal characters") in both raw and cooked +/// forms, and octal escapes wrap mod 256. Invalid-escape `SyntaxWarning`s +/// are detected separately by the lexer (see +/// [`weavepy_lexer::tokenize_with_escapes`]). fn decode_bytes_body(s: &str, raw: bool) -> Result, String> { if raw { - return Ok(s.as_bytes().to_vec()); + let mut out = Vec::with_capacity(s.len()); + for c in s.chars() { + if !c.is_ascii() { + return Err("bytes can only contain ASCII literal characters".to_owned()); + } + out.push(c as u8); + } + return Ok(out); } let mut out = Vec::with_capacity(s.len()); - let mut chars = s.chars(); + let mut chars = s.chars().peekable(); while let Some(c) = chars.next() { if c.is_ascii() { if c != '\\' { @@ -3430,7 +3870,7 @@ fn decode_bytes_body(s: &str, raw: bool) -> Result, String> { continue; } } else { - return Err("non-ascii character in bytes literal".to_owned()); + return Err("bytes can only contain ASCII literal characters".to_owned()); } let Some(esc) = chars.next() else { out.push(b'\\'); @@ -3443,7 +3883,22 @@ fn decode_bytes_body(s: &str, raw: bool) -> Result, String> { '\\' => out.push(b'\\'), '\'' => out.push(b'\''), '"' => out.push(b'"'), - '0' => out.push(0), + // Octal escape `\ooo` (1–3 digits). In a bytes literal the + // value is stored as a single byte, so CPython wraps it mod + // 256 (`b'\400'` -> 0x00, `b'\777'` -> 0xFF). + '0'..='7' => { + let mut val: u32 = esc as u32 - '0' as u32; + for _ in 0..2 { + match chars.peek().copied() { + Some(d @ '0'..='7') => { + val = val * 8 + (d as u32 - '0' as u32); + chars.next(); + } + _ => break, + } + } + out.push((val & 0xFF) as u8); + } 'a' => out.push(0x07), 'b' => out.push(0x08), 'f' => out.push(0x0c), diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index b12dc5f..f066420 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -52,6 +52,7 @@ pub struct BuiltinTypes { pub not_implemented_type_: Rc, pub simple_namespace_: Rc, pub function_: Rc, + pub method_: Rc, pub generator_: Rc, pub coroutine_: Rc, pub async_generator_: Rc, @@ -80,6 +81,10 @@ pub struct BuiltinTypes { pub type_error: Rc, pub unbound_local_error: Rc, pub value_error: Rc, + pub unicode_error: Rc, + pub unicode_encode_error: Rc, + pub unicode_decode_error: Rc, + pub unicode_translate_error: Rc, pub zero_division_error: Rc, pub generator_exit: Rc, pub keyboard_interrupt: Rc, @@ -176,6 +181,10 @@ impl BuiltinTypes { let not_implemented_type_ = mk("NotImplementedType", vec![object_.clone()]); let simple_namespace_ = mk("SimpleNamespace", vec![object_.clone()]); let function_ = mk("function", vec![object_.clone()]); + // `types.MethodType` — the bound-method type. Distinct from + // `function` so `type(obj.meth)` is `method` (as in CPython) and + // `types.MethodType(func, obj)` can construct a bound method. + let method_ = mk("method", vec![object_.clone()]); let generator_ = mk("generator", vec![object_.clone()]); let coroutine_ = mk("coroutine", vec![object_.clone()]); let async_generator_ = mk("async_generator", vec![object_.clone()]); @@ -212,11 +221,30 @@ impl BuiltinTypes { // in CPython's hierarchy, not a subclass. let stop_async_iteration = exc("StopAsyncIteration", exception.clone()); let syntax_error = exc("SyntaxError", exception.clone()); + // CPython's `SyntaxError.__init__` unpacks the + // `(filename, lineno, offset, text[, end_lineno, end_offset])` + // detail tuple into attributes, and its `__str__` appends + // `" (, line N)"`. Install both so the type behaves as a + // drop-in whether constructed from Python or raised from Rust. + install_syntax_error_dunders(&syntax_error); // `TimeoutError` lands here so `asyncio.wait_for` raises a // public, importable type rather than a synthetic shim. let timeout_error = exc("TimeoutError", os_error.clone()); let type_error = exc("TypeError", exception.clone()); let value_error = exc("ValueError", exception.clone()); + // Unicode error hierarchy: `UnicodeError` derives from + // `ValueError`, and the three concrete codecs errors derive from + // it. CPython gives the concrete three extra attributes + // (`encoding`/`object`/`start`/`end`/`reason`) populated by their + // `__init__`; install those so `str(UnicodeDecodeError(...))` and + // attribute access match. + let unicode_error = exc("UnicodeError", value_error.clone()); + let unicode_encode_error = exc("UnicodeEncodeError", unicode_error.clone()); + let unicode_decode_error = exc("UnicodeDecodeError", unicode_error.clone()); + let unicode_translate_error = exc("UnicodeTranslateError", unicode_error.clone()); + install_unicode_error_dunders(&unicode_encode_error, UnicodeErrorKind::Encode); + install_unicode_error_dunders(&unicode_decode_error, UnicodeErrorKind::Decode); + install_unicode_error_dunders(&unicode_translate_error, UnicodeErrorKind::Translate); let generator_exit = exc("GeneratorExit", base_exception.clone()); let keyboard_interrupt = exc("KeyboardInterrupt", base_exception.clone()); let system_exit = exc("SystemExit", base_exception.clone()); @@ -313,6 +341,7 @@ impl BuiltinTypes { not_implemented_type_, simple_namespace_, function_, + method_, generator_, coroutine_, async_generator_, @@ -339,6 +368,10 @@ impl BuiltinTypes { type_error, unbound_local_error, value_error, + unicode_error, + unicode_encode_error, + unicode_decode_error, + unicode_translate_error, zero_division_error, generator_exit, keyboard_interrupt, @@ -440,6 +473,10 @@ impl BuiltinTypes { pair!(type_error, "TypeError"), pair!(unbound_local_error, "UnboundLocalError"), pair!(value_error, "ValueError"), + pair!(unicode_error, "UnicodeError"), + pair!(unicode_encode_error, "UnicodeEncodeError"), + pair!(unicode_decode_error, "UnicodeDecodeError"), + pair!(unicode_translate_error, "UnicodeTranslateError"), pair!(zero_division_error, "ZeroDivisionError"), pair!(generator_exit, "GeneratorExit"), pair!(keyboard_interrupt, "KeyboardInterrupt"), @@ -527,6 +564,10 @@ impl BuiltinTypes { "TypeError" => Some(self.type_error.clone()), "UnboundLocalError" => Some(self.unbound_local_error.clone()), "ValueError" => Some(self.value_error.clone()), + "UnicodeError" => Some(self.unicode_error.clone()), + "UnicodeEncodeError" => Some(self.unicode_encode_error.clone()), + "UnicodeDecodeError" => Some(self.unicode_decode_error.clone()), + "UnicodeTranslateError" => Some(self.unicode_translate_error.clone()), "ZeroDivisionError" => Some(self.zero_division_error.clone()), "GeneratorExit" => Some(self.generator_exit.clone()), "KeyboardInterrupt" => Some(self.keyboard_interrupt.clone()), @@ -610,6 +651,104 @@ pub fn make_exception(class_name: &str, message: impl Into) -> Object { make_exception_with_class(class, message) } +/// Extract the elements of a *concrete* iterable (one that doesn't need +/// the interpreter to drive). Used by `object.__new__` to seed the +/// native payload of an immutable-container subclass from a +/// `__getnewargs__`-supplied value. Returns `None` for anything that +/// would require VM iteration (generators, user iterators), which +/// `object.__new__` can't run. +fn concrete_elements(obj: &Object) -> Option> { + match obj { + Object::List(items) => Some(items.borrow().clone()), + Object::Tuple(items) => Some(items.to_vec()), + Object::Set(s) => Some(s.borrow().iter().map(|k| k.0.clone()).collect()), + Object::FrozenSet(s) => Some(s.iter().map(|k| k.0.clone()).collect()), + Object::Str(s) => Some(s.chars().map(|c| Object::from_str(c.to_string())).collect()), + Object::Bytes(b) => Some(b.iter().map(|&x| Object::Int(i64::from(x))).collect()), + Object::ByteArray(b) => Some( + b.borrow() + .iter() + .map(|&x| Object::Int(i64::from(x))) + .collect(), + ), + // A subclass instance wrapping a concrete native container. + Object::Instance(inst) => inst.native.as_ref().and_then(concrete_elements), + _ => None, + } +} + +/// Build the native payload `object.__new__(cls, value?)` should stash +/// on an instance of a value/container built-in subclass, or `None` for +/// an ordinary `object` subclass. Mutable containers (`list`/`dict`/ +/// `set`/`bytearray`) start empty regardless of `value` — they're filled +/// afterwards by `__init__`/`__setstate__`/the copy reconstruction loop; +/// immutable ones (`int`/`float`/`complex`/`str`/`bytes`/`tuple`/ +/// `frozenset`) capture `value` here because they can't be mutated later. +fn native_seed_for_new(cls: &Rc, value: Option<&Object>) -> Option { + if cls.flags.is_builtin { + return None; + } + let bt = builtin_types(); + let is_strict = |base: &Rc| cls.is_subclass_of(base) && !Rc::ptr_eq(cls, base); + if is_strict(&bt.int_) { + return Some(match value { + None => Object::Int(0), + Some(o @ (Object::Int(_) | Object::Long(_))) => o.clone(), + Some(Object::Bool(b)) => Object::Int(i64::from(*b)), + Some(o) => o.native_value().unwrap_or_else(|| Object::Int(o.as_i64().unwrap_or(0))), + }); + } + if is_strict(&bt.float_) { + let f = value.and_then(Object::as_f64).unwrap_or(0.0); + return Some(Object::Float(f)); + } + if is_strict(&bt.complex_) { + return Some(match value { + Some(c @ Object::Complex(_)) => c.clone(), + Some(o) => o.native_value().filter(|n| matches!(n, Object::Complex(_))).unwrap_or(o.clone()), + None => Object::new_complex(0.0, 0.0), + }); + } + if is_strict(&bt.str_) { + return Some(match value { + Some(s @ Object::Str(_)) => s.clone(), + _ => Object::from_static(""), + }); + } + if is_strict(&bt.bytearray_) { + let bytes = value + .and_then(concrete_elements) + .map(|els| els.iter().filter_map(|o| o.as_i64()).map(|i| i as u8).collect()) + .unwrap_or_default(); + return Some(Object::ByteArray(Rc::new(RefCell::new(bytes)))); + } + if is_strict(&bt.bytes_) { + let bytes: Vec = value + .and_then(concrete_elements) + .map(|els| els.iter().filter_map(|o| o.as_i64()).map(|i| i as u8).collect()) + .unwrap_or_default(); + return Some(Object::Bytes(Rc::from(bytes.as_slice()))); + } + if is_strict(&bt.tuple_) { + let els = value.and_then(concrete_elements).unwrap_or_default(); + return Some(Object::new_tuple(els)); + } + if is_strict(&bt.frozenset_) { + let els = value.and_then(concrete_elements).unwrap_or_default(); + return Some(Object::new_frozenset_from(els)); + } + if is_strict(&bt.list_) { + return Some(Object::new_list(Vec::new())); + } + if is_strict(&bt.set_) { + return Some(Object::new_set_from(Vec::::new())); + } + if is_strict(&bt.dict_) { + return Some(Object::Dict(Rc::new(RefCell::new(DictData::new())))); + } + None +} + /// Install `object.__new__`, `object.__init__`, `object.__setattr__` /// and `object.__delattr__` on the root class. These are the implicit /// base methods every user class inherits. @@ -626,18 +765,15 @@ fn install_object_dunders(object_: &Rc) { )) } }; - // When `cls` derives from a primitive immutable built-in (so far - // `int` — covering `_NamedIntConstant`, `enum.IntEnum`/`IntFlag` - // and hand-written `class C(int)`), capture the value the - // instance wraps. `super().__new__(cls, value)` passes it as the - // second positional argument; absent that it defaults to 0. - if cls.is_subclass_of(&builtin_types().int_) { - let native = match args.get(1) { - None => Object::Int(0), - Some(o @ (Object::Int(_) | Object::Long(_))) => o.clone(), - Some(Object::Bool(b)) => Object::Int(i64::from(*b)), - Some(o) => Object::Int(o.as_i64().unwrap_or(0)), - }; + // When `cls` derives from a value/container built-in (`int`, + // `float`, `str`, `tuple`, `list`, `dict`, …) capture the native + // payload the instance wraps so the inherited protocols keep + // firing through the subclass. `super().__new__(cls, value)` + // passes the seed value as the second positional argument (how + // `copyreg.__newobj__` reconstructs immutable subclasses); mutable + // containers start empty and are filled by `__init__` / + // `__setstate__` / the `_reconstruct` append-and-update loop. + if let Some(native) = native_seed_for_new(&cls, args.get(1)) { return Ok(Object::Instance(Rc::new(PyInstance::with_native( cls, native, )))); @@ -708,7 +844,24 @@ fn install_object_dunders(object_: &Rc) { } Ok(Object::None) } + fn object_hash(args: &[Object]) -> Result { + // Default `object.__hash__`: the same canonical hash the `hash()` + // builtin falls back to when no custom `__hash__` is defined, so + // `object.__hash__(x) == hash(x)` for any object using the default. + let obj = args + .first() + .ok_or_else(|| crate::error::type_error("object.__hash__() takes exactly 1 argument"))?; + crate::builtins::hash_object(obj) + } let mut dict = object_.dict.borrow_mut(); + dict.insert( + DictKey(Object::from_static("__hash__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__hash__", + call: Box::new(object_hash), + call_kw: None, + })), + ); dict.insert( DictKey(Object::from_static("__new__")), Object::StaticMethod(Rc::new(Object::Builtin(Rc::new(BuiltinFn { @@ -755,6 +908,75 @@ fn install_object_dunders(object_: &Rc) { call_kw: None, })))), ); + // `object.__subclasshook__(cls, subclass)` returns `NotImplemented` + // by default (CPython), telling `issubclass`/ABCMeta to fall back to + // the normal MRO/registry check. ABCs override it to implement + // structural ("duck typing") subclass tests. + fn object_subclasshook(_args: &[Object]) -> Result { + Ok(crate::vm_singletons::not_implemented()) + } + dict.insert( + DictKey(Object::from_static("__subclasshook__")), + Object::ClassMethod(Rc::new(Object::Builtin(Rc::new(BuiltinFn { + name: "__subclasshook__", + call: Box::new(object_subclasshook), + call_kw: None, + })))), + ); + // `object.__reduce_ex__(self, protocol)` / `object.__reduce__(self)` + // need interpreter access (to import `copyreg` and call the receiver's + // `__getstate__`/`__getnewargs__` hooks), so they are registered under + // sentinel names that `Interpreter::call` intercepts (see the + // `.object_reduce_ex` / `.object_reduce` arms there). Plain + // `BuiltinFn::call` is a `fn(&[Object])` and can't reach the VM. + fn object_reduce_ex_sentinel(_args: &[Object]) -> Result { + Err(crate::error::runtime_error( + "object.__reduce_ex__ must be dispatched via Interpreter::call", + )) + } + fn object_reduce_sentinel(_args: &[Object]) -> Result { + Err(crate::error::runtime_error( + "object.__reduce__ must be dispatched via Interpreter::call", + )) + } + dict.insert( + DictKey(Object::from_static("__reduce_ex__")), + Object::Builtin(Rc::new(BuiltinFn { + name: ".object_reduce_ex", + call: Box::new(object_reduce_ex_sentinel), + call_kw: None, + })), + ); + dict.insert( + DictKey(Object::from_static("__reduce__")), + Object::Builtin(Rc::new(BuiltinFn { + name: ".object_reduce", + call: Box::new(object_reduce_sentinel), + call_kw: None, + })), + ); + // `object.__getattribute__(self, name)` — the default attribute + // lookup (data descriptor → instance dict → class attr → AttributeError). + // Needs VM access to run the descriptor protocol and walk the MRO, so it + // is wired through a sentinel name that `Interpreter::call` intercepts + // (both bound `x.__getattribute__(name)` and unbound + // `object.__getattribute__(x, name)` forms). Exposing it here lets a + // user-defined `__getattribute__` delegate to `object.__getattribute__` + // (the canonical CPython idiom), and lets `load_attr` distinguish a real + // override from this default without a special is-defined-on-object flag. + fn object_getattribute_sentinel(_args: &[Object]) -> Result { + Err(crate::error::runtime_error( + "object.__getattribute__ must be dispatched via Interpreter::call", + )) + } + dict.insert( + DictKey(Object::from_static("__getattribute__")), + Object::Builtin(Rc::new(BuiltinFn { + name: ".object_getattribute", + call: Box::new(object_getattribute_sentinel), + call_kw: None, + })), + ); } /// Install `type.__new__` and `type.__init__` so user metaclasses @@ -830,6 +1052,329 @@ fn install_os_error_init(os_error: &Rc) { ); } +/// Which of the three concrete unicode errors we're installing dunders +/// for. They share storage (`object`/`start`/`end`/`reason`, plus +/// `encoding` for the codec variants) but differ in constructor arity +/// and the `__str__` message shape. +#[derive(Clone, Copy, PartialEq, Eq)] +enum UnicodeErrorKind { + Encode, + Decode, + Translate, +} + +/// Install `__init__` / `__str__` for `UnicodeEncodeError`, +/// `UnicodeDecodeError`, and `UnicodeTranslateError`, mirroring CPython's +/// `Objects/exceptions.c` (`UnicodeEncodeError_init`, `…_str`, etc.). +/// +/// Constructors: +/// * encode/decode: `(encoding, object, start, end, reason)` +/// * translate: `(object, start, end, reason)` +/// +/// `__str__` reproduces the exact CPython wording, including the +/// single-element `'\\xXX'` / `'\\uXXXX'` / `'\\UXXXXXXXX'` escape for a +/// one-position slice and the `position M-N` form for a range. +fn install_unicode_error_dunders(ty: &Rc, kind: UnicodeErrorKind) { + use crate::object::BuiltinFn; + + fn set(dict: &mut crate::object::DictData, name: &'static str, value: Object) { + dict.insert(DictKey(Object::from_static(name)), value); + } + + let init = move |args: &[Object]| -> Result { + let Some(Object::Instance(inst_rc)) = args.first() else { + return Ok(Object::None); + }; + let rest = if args.len() > 1 { &args[1..] } else { &[][..] }; + let want = if kind == UnicodeErrorKind::Translate { + 4 + } else { + 5 + }; + if rest.len() != want { + return Err(crate::error::type_error(format!( + "function takes exactly {} arguments ({} given)", + want, + rest.len() + ))); + } + let mut dict = inst_rc.dict.borrow_mut(); + set(&mut dict, "args", Object::new_tuple(rest.to_vec())); + let mut i = 0; + if kind != UnicodeErrorKind::Translate { + set(&mut dict, "encoding", rest[i].clone()); + i += 1; + } + set(&mut dict, "object", rest[i].clone()); + set(&mut dict, "start", rest[i + 1].clone()); + set(&mut dict, "end", rest[i + 2].clone()); + set(&mut dict, "reason", rest[i + 3].clone()); + Ok(Object::None) + }; + + let str_fn = move |args: &[Object]| -> Result { + let Some(Object::Instance(inst_rc)) = args.first() else { + return Ok(Object::from_static("")); + }; + let dict = inst_rc.dict.borrow(); + let get = |name: &'static str| dict.get(&DictKey(Object::from_static(name))).cloned(); + let as_i = |o: &Object| -> i64 { + match o { + Object::Int(n) => *n, + Object::Bool(b) => i64::from(*b), + _ => 0, + } + }; + let encoding = match get("encoding") { + Some(Object::Str(s)) => s.to_string(), + _ => String::new(), + }; + let reason = match get("reason") { + Some(Object::Str(s)) => s.to_string(), + _ => String::new(), + }; + let start = get("start").as_ref().map(as_i).unwrap_or(0); + let end = get("end").as_ref().map(as_i).unwrap_or(0); + let obj = get("object").unwrap_or(Object::None); + + // Escape a single offending scalar exactly as CPython does. + let escape = |c: u32| -> String { + if c < 0x100 { + format!("\\x{c:02x}") + } else if c < 0x10000 { + format!("\\u{c:04x}") + } else { + format!("\\U{c:08x}") + } + }; + + let msg = match kind { + UnicodeErrorKind::Encode => { + let s: Vec = match &obj { + Object::Str(s) => s.chars().collect(), + _ => Vec::new(), + }; + if start >= 0 && (start as usize) < s.len() && end == start + 1 { + let c = s[start as usize] as u32; + format!( + "'{encoding}' codec can't encode character '{}' in position {start}: {reason}", + escape(c) + ) + } else { + format!( + "'{encoding}' codec can't encode characters in position {start}-{}: {reason}", + end - 1 + ) + } + } + UnicodeErrorKind::Decode => { + let b: &[u8] = match &obj { + Object::Bytes(b) => b, + _ => &[], + }; + if start >= 0 && (start as usize) < b.len() && end == start + 1 { + format!( + "'{encoding}' codec can't decode byte 0x{:02x} in position {start}: {reason}", + b[start as usize] + ) + } else { + format!( + "'{encoding}' codec can't decode bytes in position {start}-{}: {reason}", + end - 1 + ) + } + } + UnicodeErrorKind::Translate => { + let s: Vec = match &obj { + Object::Str(s) => s.chars().collect(), + _ => Vec::new(), + }; + if start >= 0 && (start as usize) < s.len() && end == start + 1 { + let c = s[start as usize] as u32; + format!( + "can't translate character '{}' in position {start}: {reason}", + escape(c) + ) + } else { + format!( + "can't translate characters in position {start}-{}: {reason}", + end - 1 + ) + } + } + }; + Ok(Object::from_str(msg)) + }; + + let mut dict = ty.dict.borrow_mut(); + dict.insert( + DictKey(Object::from_static("__init__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__init__", + call: Box::new(init), + call_kw: None, + })), + ); + dict.insert( + DictKey(Object::from_static("__str__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__str__", + call: Box::new(str_fn), + call_kw: None, + })), + ); +} + +/// CPython's `SyntaxError.__init__` / `__str__`. +/// +/// `__init__(self, *args)` stores `args` like `BaseException`, then — when +/// called as `SyntaxError(msg, (filename, lineno, offset, text[, end_lineno, +/// end_offset]))` — unpacks the detail sequence into named attributes. +/// `__str__` reproduces CPython's `SyntaxError_str`: bare `msg` unless a +/// filename and/or line are present, in which case it appends +/// `" (, line N)"` / `" ()"` / `" (line N)"`. +fn install_syntax_error_dunders(syntax_error: &Rc) { + use crate::object::BuiltinFn; + + fn set(dict: &mut crate::object::DictData, name: &'static str, value: Object) { + dict.insert(DictKey(Object::from_static(name)), value); + } + + fn syntaxerror_init(args: &[Object]) -> Result { + let inst = args + .first() + .ok_or_else(|| crate::error::type_error("expected exception instance".to_owned()))?; + let Object::Instance(inst_rc) = inst else { + return Ok(Object::None); + }; + let rest = if args.len() > 1 { &args[1..] } else { &[][..] }; + let mut dict = inst_rc.dict.borrow_mut(); + set(&mut dict, "args", Object::new_tuple(rest.to_vec())); + // Defaults — CPython always defines these slots. + for name in [ + "msg", + "filename", + "lineno", + "offset", + "text", + "end_lineno", + "end_offset", + ] { + set(&mut dict, name, Object::None); + } + if let Some(msg) = rest.first() { + set(&mut dict, "msg", msg.clone()); + } + // `SyntaxError(msg, detail)` — `detail` is a 2-to-6 element + // sequence `(filename, lineno, offset, text[, end_lineno, + // end_offset])`. + if rest.len() == 2 { + let info: Option<&[Object]> = match &rest[1] { + Object::Tuple(items) => Some(items.as_ref()), + Object::List(items) => { + // Borrow can't outlive the match arm; handle inline. + let v = items.borrow(); + let pick = |i: usize| v.get(i).cloned().unwrap_or(Object::None); + set(&mut dict, "filename", pick(0)); + set(&mut dict, "lineno", pick(1)); + set(&mut dict, "offset", pick(2)); + set(&mut dict, "text", pick(3)); + if v.len() > 4 { + set(&mut dict, "end_lineno", pick(4)); + set(&mut dict, "end_offset", pick(5)); + } + None + } + // Non-sequence second arg: CPython leaves the location + // attributes at their `None` defaults. + _ => None, + }; + if let Some(items) = info { + let pick = |i: usize| items.get(i).cloned().unwrap_or(Object::None); + set(&mut dict, "filename", pick(0)); + set(&mut dict, "lineno", pick(1)); + set(&mut dict, "offset", pick(2)); + set(&mut dict, "text", pick(3)); + if items.len() > 4 { + set(&mut dict, "end_lineno", pick(4)); + set(&mut dict, "end_offset", pick(5)); + } + } + } + Ok(Object::None) + } + + fn syntaxerror_str(args: &[Object]) -> Result { + let inst = args + .first() + .ok_or_else(|| crate::error::type_error("expected exception instance".to_owned()))?; + let Object::Instance(inst_rc) = inst else { + return Ok(Object::from_static("")); + }; + let dict = inst_rc.dict.borrow(); + let get = |name: &'static str| { + dict.get(&DictKey(Object::from_static(name))) + .cloned() + .unwrap_or(Object::None) + }; + let msg = get("msg"); + // CPython renders the message via `str(self.msg)`. + let msg_str = match &msg { + Object::None => "None".to_owned(), + other => other.to_str(), + }; + let filename = get("filename"); + let lineno = get("lineno"); + let have_filename = matches!(filename, Object::Str(_)); + let lineno_val = match &lineno { + Object::Int(n) => Some(*n), + Object::Bool(b) => Some(i64::from(*b)), + _ => None, + }; + let result = match (have_filename, lineno_val) { + (true, Some(n)) => { + format!("{msg_str} ({}, line {n})", syntax_basename(&filename)) + } + (true, None) => format!("{msg_str} ({})", syntax_basename(&filename)), + (false, Some(n)) => format!("{msg_str} (line {n})"), + (false, None) => msg_str, + }; + Ok(Object::from_str(result)) + } + + let mut dict = syntax_error.dict.borrow_mut(); + set( + &mut dict, + "__init__", + Object::Builtin(Rc::new(BuiltinFn { + name: "__init__", + call: Box::new(syntaxerror_init), + call_kw: None, + })), + ); + set( + &mut dict, + "__str__", + Object::Builtin(Rc::new(BuiltinFn { + name: "__str__", + call: Box::new(syntaxerror_str), + call_kw: None, + })), + ); +} + +/// Last path component of a `SyntaxError.filename`, mirroring CPython's +/// `my_basename` (split on `/` — and `\\` on the same footing so Windows +/// paths render the same). Non-string filenames yield an empty string. +fn syntax_basename(filename: &Object) -> String { + let Object::Str(s) = filename else { + return String::new(); + }; + let s = s.as_ref(); + let cut = s.rfind(['/', '\\']).map_or(0, |i| i + 1); + s[cut..].to_owned() +} + fn install_exception_str_repr(base_exception: &Rc) { use crate::object::BuiltinFn; fn exc_init(args: &[Object]) -> Result { @@ -935,6 +1480,21 @@ fn install_exception_str_repr(base_exception: &Rc) { } Ok(Object::None) } + // `e.with_traceback(tb)` sets `__traceback__` and returns `self`, so + // `raise e.with_traceback(tb)` and chained-exception helpers work. + fn exc_with_traceback(args: &[Object]) -> Result { + let inst = args + .first() + .ok_or_else(|| crate::error::type_error("expected exception instance".to_owned()))?; + let tb = args.get(1).cloned().unwrap_or(Object::None); + if let Object::Instance(inst_rc) = inst { + inst_rc + .dict + .borrow_mut() + .insert(DictKey(Object::from_static("__traceback__")), tb); + } + Ok(inst.clone()) + } let mut dict = base_exception.dict.borrow_mut(); dict.insert( DictKey(Object::from_static("__init__")), @@ -968,18 +1528,37 @@ fn install_exception_str_repr(base_exception: &Rc) { call_kw: None, })), ); + dict.insert( + DictKey(Object::from_static("with_traceback")), + Object::Builtin(Rc::new(BuiltinFn { + name: "with_traceback", + call: Box::new(exc_with_traceback), + call_kw: None, + })), + ); } pub fn make_exception_with_class(class: Rc, message: impl Into) -> Object { use crate::types::PyInstance; let is_os = is_subclass_by_name(&class, "OSError"); + let is_syntax = is_subclass_by_name(&class, "SyntaxError"); let inst = PyInstance::new(class); let msg = Object::from_str(message); let args = Object::new_tuple(vec![msg.clone()]); { let mut dict = inst.dict.borrow_mut(); dict.insert(DictKey(Object::from_static("args")), args); - dict.insert(DictKey(Object::from_static("message")), msg); + dict.insert(DictKey(Object::from_static("message")), msg.clone()); + // Always-present `BaseException` slots (see `build_exception_instance`): + // default None/None/False/None so attribute access and context-chain + // walks never `AttributeError`. + dict.insert(DictKey(Object::from_static("__context__")), Object::None); + dict.insert(DictKey(Object::from_static("__cause__")), Object::None); + dict.insert( + DictKey(Object::from_static("__suppress_context__")), + Object::Bool(false), + ); + dict.insert(DictKey(Object::from_static("__traceback__")), Object::None); if is_os { // OSError attributes — populated to None when we raise // from Rust so callers can still ask `exc.errno` without @@ -989,6 +1568,18 @@ pub fn make_exception_with_class(class: Rc, message: impl Into) { + fn value_hash(args: &[Object]) -> Result { + crate::builtins::hash_object(args.first().unwrap_or(&Object::None)) + } + ty.dict.borrow_mut().insert( + DictKey(Object::from_static("__hash__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__hash__", + call: Box::new(value_hash), + call_kw: None, + })), + ); + } + for ty in [ + &bt.int_, + &bt.float_, + &bt.complex_, + &bt.str_, + &bt.bytes_, + &bt.tuple_, + &bt.frozenset_, + ] { + install_hash(ty); + } + + // Expose the inherited numeric coercion dunders so a subclass that does + // *not* override them (`class C(int)` with only `__index__`) still + // resolves the base type's value-returning `__int__`/`__index__`/ + // `__float__` through the MRO — matching CPython, where `int(C())` uses + // the wrapped value rather than the overriding hook. + fn install_method( + ty: &Rc, + name: &'static str, + f: fn(&[Object]) -> Result, + ) { + ty.dict.borrow_mut().insert( + DictKey(Object::from_static(name)), + Object::Builtin(Rc::new(BuiltinFn { + name, + call: Box::new(f), + call_kw: None, + })), + ); + } + fn self_as_int(args: &[Object]) -> Result { + let o = args + .first() + .ok_or_else(|| crate::error::type_error("__int__ requires an argument"))?; + let native = o.native_value(); + match native.as_ref().unwrap_or(o) { + Object::Int(i) => Ok(Object::Int(*i)), + Object::Long(b) => Ok(Object::Long(b.clone())), + Object::Bool(b) => Ok(Object::Int(i64::from(*b))), + other => Err(crate::error::type_error(format!( + "descriptor '__int__' requires a 'int' object but received a '{}'", + other.type_name() + ))), + } + } + fn self_as_float(args: &[Object]) -> Result { + let o = args + .first() + .ok_or_else(|| crate::error::type_error("__float__ requires an argument"))?; + let native = o.native_value(); + match native.as_ref().unwrap_or(o) { + Object::Float(f) => Ok(Object::Float(*f)), + other => Err(crate::error::type_error(format!( + "descriptor '__float__' requires a 'float' object but received a '{}'", + other.type_name() + ))), + } + } + install_method(&bt.int_, "__int__", self_as_int); + install_method(&bt.int_, "__index__", self_as_int); + install_method(&bt.float_, "__float__", self_as_float); } diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index ccdc522..67284ff 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -19,7 +19,7 @@ use crate::sync::Rc; use crate::sync::RefCell; use num_bigint::BigInt; -use num_traits::{Signed, ToPrimitive, Zero}; +use num_traits::{FromPrimitive, Signed, ToPrimitive, Zero}; use crate::builtin_types::{builtin_types, instance_is_subclass}; use crate::error::{ @@ -46,6 +46,69 @@ pub fn build_class_builtin() -> BuiltinFn { } } +/// Resolve the native constructor function for a built-in *type* by name. +/// +/// The VM's instantiation fallback (`builtin_constructor_for`) needs the +/// `b_*` constructor (e.g. `b_set`) even though the user-visible +/// `__builtins__` now maps these names to the real `type` objects. Keeping +/// this lookup independent of the `__builtins__` dict lets both coexist: +/// `builtins.set is set` (a type) while `set(...)` still constructs through +/// the native helper. +pub(crate) fn builtin_type_constructor(name: &str) -> Option> { + macro_rules! ctor { + ($n:literal, $body:expr) => { + Some(Rc::new(BuiltinFn { + name: $n, + call: Box::new($body), + call_kw: None, + })) + }; + } + match name { + "str" => ctor!("str", b_str), + "int" => ctor!("int", b_int), + "float" => ctor!("float", b_float), + "complex" => ctor!("complex", b_complex), + "bool" => ctor!("bool", b_bool), + "list" => ctor!("list", b_list), + "tuple" => ctor!("tuple", b_tuple), + "dict" => ctor!("dict", b_dict), + "set" => ctor!("set", b_set), + "frozenset" => ctor!("frozenset", b_frozenset), + "bytes" => ctor!("bytes", b_bytes), + "bytearray" => ctor!("bytearray", b_bytearray), + "object" => ctor!("object", b_object), + "type" => ctor!("type", b_type), + "range" => ctor!("range", b_range), + "slice" => ctor!("slice", b_slice), + "memoryview" => ctor!("memoryview", b_memoryview), + _ => None, + } +} + +/// `slice(stop)` / `slice(start, stop[, step])` → a real `Object::Slice`, +/// the same representation the `BUILD_SLICE` opcode produces for `a:b:c`. +/// Without this the type's generic instantiation path made a bare +/// `object` instance that the subscript handlers (which match +/// `Object::Slice`) rejected. Missing positions default to `None`, +/// matching CPython's `slice` type. +pub(crate) fn b_slice(args: &[Object]) -> Result { + let (start, stop, step) = match args.len() { + 0 => { + return Err(type_error("slice expected at least 1 argument, got 0")); + } + 1 => (Object::None, args[0].clone(), Object::None), + 2 => (args[0].clone(), args[1].clone(), Object::None), + 3 => (args[0].clone(), args[1].clone(), args[2].clone()), + n => { + return Err(type_error(format!( + "slice expected at most 3 arguments, got {n}" + ))); + } + }; + Ok(Object::Slice(Rc::new(crate::object::PySlice { start, stop, step }))) +} + /// Build the dict that backs the `builtins` module. pub fn default_builtins() -> DictData { let mut d = DictData::new(); @@ -332,9 +395,9 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "strip" => Some(method("strip", str_strip)), "lstrip" => Some(method("lstrip", str_lstrip)), "rstrip" => Some(method("rstrip", str_rstrip)), - "split" => Some(method("split", str_split)), - "rsplit" => Some(method("rsplit", str_rsplit)), - "splitlines" => Some(method("splitlines", str_splitlines)), + "split" => Some(method_kw("split", str_split)), + "rsplit" => Some(method_kw("rsplit", str_rsplit)), + "splitlines" => Some(method_kw("splitlines", str_splitlines)), "join" => Some(method("join", str_join)), "startswith" => Some(method("startswith", str_startswith)), "endswith" => Some(method("endswith", str_endswith)), @@ -364,10 +427,16 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "encode" => Some(method("encode", str_encode)), "removeprefix" => Some(method("removeprefix", str_removeprefix)), "removesuffix" => Some(method("removesuffix", str_removesuffix)), - "format" => Some(method("format", str_format)), - "format_map" => Some(method("format_map", str_format_map)), + "format" => Some(method(".format", str_format)), + "format_map" => Some(method(".format_map", str_format_map)), "translate" => Some(method("translate", str_translate)), "maketrans" => Some(method("maketrans", str_maketrans)), + // Sequence dunders so `hasattr(s, '__getitem__')` and direct + // `str.__getitem__(s, i)` calls work (CPython exposes these as + // slot wrappers; `operator.concat` probes `__getitem__`). + "__getitem__" => Some(method("__getitem__", seq_getitem)), + "__len__" => Some(method("__len__", obj_len)), + "__contains__" => Some(method("__contains__", obj_contains)), _ => None, }, Object::List(_) => match name { @@ -382,6 +451,13 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "reverse" => Some(method("reverse", list_reverse)), "clear" => Some(method("clear", list_clear)), "copy" => Some(method("copy", list_copy)), + // Dunders so `list.__setitem__` / `super().__getitem__` resolve + // for `list` subclasses (`class C(list)`). + "__getitem__" => Some(method("__getitem__", list_getitem)), + "__setitem__" => Some(method("__setitem__", list_setitem)), + "__delitem__" => Some(method("__delitem__", list_delitem)), + "__len__" => Some(method("__len__", obj_len)), + "__contains__" => Some(method("__contains__", obj_contains)), _ => None, }, Object::Dict(_) => match name { @@ -396,11 +472,20 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "copy" => Some(method("copy", dict_copy)), "fromkeys" => Some(method("fromkeys", dict_fromkeys)), "popitem" => Some(method("popitem", dict_popitem)), + // Dunders so `dict.__setitem__` / `super().__setitem__` resolve + // for `dict` subclasses (`class C(dict)`). + "__setitem__" => Some(method("__setitem__", dict_setitem)), + "__getitem__" => Some(method("__getitem__", dict_getitem)), + "__delitem__" => Some(method("__delitem__", dict_delitem)), + "__init__" => Some(method("__init__", dict_update)), _ => None, }, Object::Tuple(_) => match name { "count" => Some(method("count", tuple_count)), "index" => Some(method("index", tuple_index)), + "__getitem__" => Some(method("__getitem__", seq_getitem)), + "__len__" => Some(method("__len__", obj_len)), + "__contains__" => Some(method("__contains__", obj_contains)), _ => None, }, Object::Set(_) | Object::FrozenSet(_) => match name { @@ -536,7 +621,7 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "__trunc__" => Some(method("__trunc__", int_conjugate)), "__floor__" => Some(method("__floor__", int_conjugate)), "__ceil__" => Some(method("__ceil__", int_conjugate)), - _ => None, + _ => numeric_dunder(obj, name), }, Object::Float(_) => match name { "is_integer" => Some(method("is_integer", float_is_integer)), @@ -545,11 +630,39 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "as_integer_ratio" => Some(method("as_integer_ratio", float_as_integer_ratio)), "conjugate" => Some(method("conjugate", float_conjugate)), "__trunc__" => Some(method("__trunc__", float_trunc)), + "__floor__" => Some(method("__floor__", float_floor)), + "__ceil__" => Some(method("__ceil__", float_ceil)), "__round__" => Some(method("__round__", float_round)), - _ => None, + _ => numeric_dunder(obj, name), }, Object::Complex(_) => match name { "conjugate" => Some(method("conjugate", complex_conjugate)), + // `complex.__complex__(self)` returns the value unchanged, so + // `complex(x)` / the numeric-tower probes accept a complex. + "__complex__" => Some(method("__complex__", |args| { + args.first() + .cloned() + .ok_or_else(|| crate::error::type_error("__complex__() missing self")) + })), + "__abs__" => Some(method("__abs__", |args| { + b_abs(std::slice::from_ref(args.first().unwrap_or(&Object::None))) + })), + _ => numeric_dunder(obj, name), + }, + Object::Slice(_) => match name { + "indices" => Some(method("indices", slice_indices_method)), + _ => None, + }, + // Built-in iterators expose `__length_hint__` (PEP 424) so + // `operator.length_hint`, `list()` pre-sizing, and friends can + // query the remaining count without consuming the iterator. + Object::Iter(_) => match name { + "__length_hint__" => Some(method("__length_hint__", iter_length_hint)), + "__iter__" => Some(method("__iter__", |args| { + args.first() + .cloned() + .ok_or_else(|| type_error("__iter__() missing self")) + })), _ => None, }, _ => None, @@ -557,6 +670,158 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { f.map(|f| Object::Builtin(Rc::new(f))) } +/// `.__length_hint__()` — the number of items the iterator +/// will still yield, when cheaply known (PEP 424). Returns `0` for +/// exhausted/unknown-length sources, matching CPython's contract that +/// the hint is advisory and never raises. +fn iter_length_hint(args: &[Object]) -> Result { + match args.first() { + Some(Object::Iter(it)) => { + let n = it.borrow().remaining().unwrap_or(0); + Ok(Object::Int(n as i64)) + } + _ => Err(type_error("__length_hint__() requires an iterator")), + } +} + +/// `seq.__getitem__(self, index)` for built-in sequences — int (incl. +/// negatives) and `slice` indexing for `str`/`list`/`tuple`/`bytes`/ +/// `bytearray`. CPython exposes these as slot wrappers; this lets +/// `hasattr(s, '__getitem__')` succeed and direct `str.__getitem__` +/// calls work. +fn seq_getitem(args: &[Object]) -> Result { + let recv = args + .first() + .ok_or_else(|| type_error("__getitem__() missing self"))?; + let index = args + .get(1) + .ok_or_else(|| type_error("__getitem__() takes exactly one argument (0 given)"))?; + let as_seq = |v: &Object| -> Vec { + match v { + Object::List(items) => items.borrow().clone(), + Object::Tuple(items) => items.to_vec(), + Object::Str(s) => s.chars().map(|c| Object::from_str(c.to_string())).collect(), + Object::Bytes(b) => b.iter().map(|x| Object::Int(i64::from(*x))).collect(), + Object::ByteArray(b) => b.borrow().iter().map(|x| Object::Int(i64::from(*x))).collect(), + _ => Vec::new(), + } + }; + match index { + Object::Slice(s) => { + let seq = as_seq(recv); + let sliced = crate::slice_seq(&seq, s)?; + Ok(match recv { + Object::Str(_) => Object::from_str(sliced.iter().map(Object::to_str).collect::()), + Object::Tuple(_) => Object::new_tuple(sliced), + Object::Bytes(_) => { + let bytes: Vec = sliced.iter().filter_map(|o| o.as_i64()).map(|i| i as u8).collect(); + Object::new_bytes(bytes) + } + Object::ByteArray(_) => { + let bytes: Vec = sliced.iter().filter_map(|o| o.as_i64()).map(|i| i as u8).collect(); + Object::new_bytearray(bytes) + } + _ => Object::new_list(sliced), + }) + } + _ => { + let i = coerce_index_i64(index)?; + let seq = as_seq(recv); + let idx = crate::normalize_index(i, seq.len())?; + Ok(seq[idx].clone()) + } + } +} + +/// `obj.__len__(self)` for built-in containers. +fn obj_len(args: &[Object]) -> Result { + let recv = args + .first() + .ok_or_else(|| type_error("__len__() missing self"))?; + Ok(Object::Int(recv.len()? as i64)) +} + +/// `obj.__contains__(self, item)` for built-in containers. +fn obj_contains(args: &[Object]) -> Result { + let recv = args + .first() + .ok_or_else(|| type_error("__contains__() missing self"))?; + let item = args + .get(1) + .ok_or_else(|| type_error("__contains__() takes exactly one argument (0 given)"))?; + Ok(Object::Bool(recv.contains(item)?)) +} + +/// `slice.indices(length)` → the `(start, stop, step)` triple a sequence +/// of `length` items would use, mirroring CPython's `PySlice_Unpack` + +/// `PySlice_AdjustIndices` (`Objects/sliceobject.c`). `length` must be a +/// non-negative integer (or `__index__`-able); `step` of 0 is a +/// `ValueError`. +fn slice_indices_method(args: &[Object]) -> Result { + let s = match args.first() { + Some(Object::Slice(s)) => s.clone(), + _ => return Err(type_error("descriptor 'indices' requires a 'slice' object")), + }; + let length = match args.get(1) { + Some(o) => coerce_index_i64(o)?, + None => { + return Err(type_error( + "indices() takes exactly one argument (0 given)", + )) + } + }; + if length < 0 { + return Err(value_error("length should not be negative")); + } + let step = match &s.step { + Object::None => 1, + o => { + let st = coerce_index_i64(o)?; + if st == 0 { + return Err(value_error("slice step cannot be zero")); + } + st + } + }; + let (lower, upper) = if step < 0 { + (-1i64, length - 1) + } else { + (0i64, length) + }; + let clamp = |v: i64| -> i64 { + if v < 0 { + (v + length).max(lower) + } else { + v.min(upper) + } + }; + let start = match &s.start { + Object::None => { + if step < 0 { + upper + } else { + lower + } + } + o => clamp(coerce_index_i64(o)?), + }; + let stop = match &s.stop { + Object::None => { + if step < 0 { + lower + } else { + upper + } + } + o => clamp(coerce_index_i64(o)?), + }; + Ok(Object::new_tuple(vec![ + Object::Int(start), + Object::Int(stop), + Object::Int(step), + ])) +} + fn method( name: &'static str, body: impl Fn(&[Object]) -> Result + Send + Sync + 'static, @@ -568,6 +833,284 @@ fn method( } } +// ---- numeric slot-wrapper dunders (`int.__add__`, `complex.__eq__`, …) ---- +// +// CPython exposes every numeric operator as a method on its type +// (`int.__add__`, `(1+2j).__truediv__`, …) that follows the binary-op +// protocol: when the *other* operand isn't a type the forward operation +// accepts, the wrapper returns `NotImplemented` instead of raising. These +// wrappers reproduce that so explicit dunder calls match CPython. +// +// They are reached only through *attribute access* — `type.__op__` (via +// [`unbound_method`]) and `value.__op__` (via [`lookup_method`]). The hot +// `a + b` operator path dispatches through `instance_method`, which only +// matches user `Object::Instance`, so primitives never route their `+` +// through here and there is neither extra overhead nor recursion risk. + +#[derive(Clone, Copy)] +enum NumSelf { + Int, + Float, + Complex, +} + +/// Classify a numeric receiver (unwrapping a built-in subclass to its +/// native payload). Non-numerics return `None`. +fn num_self_of(o: &Object) -> Option { + let native = o.native_value(); + match native.as_ref().unwrap_or(o) { + Object::Int(_) | Object::Long(_) | Object::Bool(_) => Some(NumSelf::Int), + Object::Float(_) => Some(NumSelf::Float), + Object::Complex(_) => Some(NumSelf::Complex), + _ => None, + } +} + +/// Does the forward dunder of `kind` accept `other`? Mirrors CPython's +/// numeric coercion ladder: `int` accepts only ints, `float` also accepts +/// floats, `complex` also accepts complexes. +fn num_accepts(kind: NumSelf, other: &Object) -> bool { + let native = other.native_value(); + let o = native.as_ref().unwrap_or(other); + let is_int = matches!(o, Object::Int(_) | Object::Long(_) | Object::Bool(_)); + let is_float = matches!(o, Object::Float(_)); + let is_complex = matches!(o, Object::Complex(_)); + match kind { + NumSelf::Int => is_int, + NumSelf::Float => is_int || is_float, + NumSelf::Complex => is_int || is_float || is_complex, + } +} + +#[derive(Clone, Copy)] +enum CmpDun { + Eq, + Ne, + Lt, + Le, + Gt, + Ge, +} + +/// Build a binary-arithmetic dunder (`__add__`, `__rmul__`, …). +fn num_binop_method( + nm: &'static str, + kind: NumSelf, + op: weavepy_compiler::BinOpKind, + reflected: bool, +) -> BuiltinFn { + method(nm, move |args| { + let s = args + .first() + .cloned() + .ok_or_else(|| type_error(format!("unbound method {nm}() needs an argument")))?; + let o = match args.get(1) { + Some(o) => o.clone(), + None => return Err(type_error(format!("{nm}() takes exactly one argument"))), + }; + if !num_accepts(kind, &o) { + return Ok(crate::vm_singletons::not_implemented()); + } + let (l, r) = if reflected { (&o, &s) } else { (&s, &o) }; + crate::binary_op(l, r, op) + }) +} + +/// Build a rich-comparison dunder (`__eq__`, `__lt__`, …). +fn num_cmp_method(nm: &'static str, kind: NumSelf, which: CmpDun) -> BuiltinFn { + method(nm, move |args| { + let s = args + .first() + .cloned() + .ok_or_else(|| type_error(format!("unbound method {nm}() needs an argument")))?; + let o = match args.get(1) { + Some(o) => o.clone(), + None => return Err(type_error(format!("{nm}() takes exactly one argument"))), + }; + let ordering = matches!(which, CmpDun::Lt | CmpDun::Le | CmpDun::Gt | CmpDun::Ge); + // `complex` has no ordering: `<`/`<=`/`>`/`>=` always decline. + if ordering && matches!(kind, NumSelf::Complex) { + return Ok(crate::vm_singletons::not_implemented()); + } + if !num_accepts(kind, &o) { + return Ok(crate::vm_singletons::not_implemented()); + } + let result = match which { + CmpDun::Eq => s.eq_value(&o), + CmpDun::Ne => !s.eq_value(&o), + CmpDun::Lt | CmpDun::Le | CmpDun::Gt | CmpDun::Ge => match s.cmp(&o) { + Ok(ord) => match which { + CmpDun::Lt => ord.is_lt(), + CmpDun::Le => ord.is_le(), + CmpDun::Gt => ord.is_gt(), + CmpDun::Ge => ord.is_ge(), + _ => unreachable!(), + }, + // Unorderable (NaN) → CPython yields `False`, not an error. + Err(_) => false, + }, + }; + Ok(Object::Bool(result)) + }) +} + +/// Build a unary dunder (`__neg__`, `__pos__`, `__abs__`). +fn num_unary_method(nm: &'static str, op: weavepy_compiler::UnaryKind) -> BuiltinFn { + method(nm, move |args| { + let s = args + .first() + .cloned() + .ok_or_else(|| type_error(format!("unbound method {nm}() needs an argument")))?; + crate::unary_op(&s, op) + }) +} + +/// `(value).__getnewargs__()` for the built-in numerics: `complex` +/// reconstructs from `(real, imag)`, the rest from `(value,)`. +fn num_getnewargs(self_o: &Object) -> Object { + let native = self_o.native_value(); + let v = native.as_ref().unwrap_or(self_o); + match v { + Object::Complex(c) => { + Object::new_tuple(vec![Object::Float(c.real), Object::Float(c.imag)]) + } + other => Object::new_tuple(vec![other.clone()]), + } +} + +/// Resolve a numeric slot-wrapper dunder by name for receiver `self_repr`. +/// Returns `None` for anything that isn't a numeric dunder so the caller +/// falls through to its other attribute paths. +fn numeric_dunder(self_repr: &Object, name: &str) -> Option { + use weavepy_compiler::BinOpKind as B; + use weavepy_compiler::UnaryKind as U; + let kind = num_self_of(self_repr)?; + let not_complex = !matches!(kind, NumSelf::Complex); + let m = match name { + "__add__" => num_binop_method("__add__", kind, B::Add, false), + "__radd__" => num_binop_method("__radd__", kind, B::Add, true), + "__sub__" => num_binop_method("__sub__", kind, B::Sub, false), + "__rsub__" => num_binop_method("__rsub__", kind, B::Sub, true), + "__mul__" => num_binop_method("__mul__", kind, B::Mult, false), + "__rmul__" => num_binop_method("__rmul__", kind, B::Mult, true), + "__truediv__" => num_binop_method("__truediv__", kind, B::Div, false), + "__rtruediv__" => num_binop_method("__rtruediv__", kind, B::Div, true), + "__pow__" => num_binop_method("__pow__", kind, B::Pow, false), + "__rpow__" => num_binop_method("__rpow__", kind, B::Pow, true), + // `floordiv`/`mod` are undefined on `complex`. + "__floordiv__" if not_complex => num_binop_method("__floordiv__", kind, B::FloorDiv, false), + "__rfloordiv__" if not_complex => { + num_binop_method("__rfloordiv__", kind, B::FloorDiv, true) + } + "__mod__" if not_complex => num_binop_method("__mod__", kind, B::Mod, false), + "__rmod__" if not_complex => num_binop_method("__rmod__", kind, B::Mod, true), + "__eq__" => num_cmp_method("__eq__", kind, CmpDun::Eq), + "__ne__" => num_cmp_method("__ne__", kind, CmpDun::Ne), + "__lt__" => num_cmp_method("__lt__", kind, CmpDun::Lt), + "__le__" => num_cmp_method("__le__", kind, CmpDun::Le), + "__gt__" => num_cmp_method("__gt__", kind, CmpDun::Gt), + "__ge__" => num_cmp_method("__ge__", kind, CmpDun::Ge), + "__neg__" => num_unary_method("__neg__", U::Neg), + "__pos__" => num_unary_method("__pos__", U::Pos), + "__getnewargs__" => method("__getnewargs__", |args| { + Ok(num_getnewargs(args.first().unwrap_or(&Object::None))) + }), + "__format__" => method("__format__", |args| { + let value = args.first().cloned().unwrap_or(Object::None); + let spec = match args.get(1) { + Some(Object::Str(s)) => s.to_string(), + Some(other) => { + return Err(type_error(format!( + "__format__() argument 1 must be str, not {}", + other.type_name() + ))) + } + None => String::new(), + }; + crate::format_via_spec(&value, &spec).map(Object::from_str) + }), + // Exposing the numeric `__hash__` puts it in the type's MRO so a + // mixin like `class F(float, H)` resolves `float.__hash__` (not + // `H.__hash__`), matching CPython's method resolution. + "__hash__" => method("__hash__", |args| { + hash_object(args.first().unwrap_or(&Object::None)) + }), + _ => return None, + }; + Some(m) +} + +/// `value.__getnewargs__()` for an immutable built-in subclass instance: +/// returns `(value,)` so `copy`/`pickle` reconstruct it as +/// `cls.__new__(cls, value)`. The receiver (`args[0]`) is the subclass +/// instance; its wrapped native payload is the base-type value. +fn instance_getnewargs(args: &[Object]) -> Result { + let native = match args.first() { + Some(Object::Instance(inst)) => inst.native.clone(), + other => other.cloned(), + }; + match native { + Some(v) => Ok(Object::new_tuple(vec![v])), + None => Ok(Object::new_tuple(Vec::new())), + } +} + +/// `__getnewargs__` for a subclass of an immutable built-in whose +/// reconstruction takes a single positional value (`int`/`float`/`str`/ +/// `bytes`/`tuple`/`bool`). Returns `None` for everything else: mutable +/// containers rebuild from items/state, `frozenset`/`set` have no +/// `__getnewargs__` in CPython, and `complex` uses a two-arg `(re, im)` +/// form handled separately. +pub fn immutable_subclass_getnewargs(native: &Object) -> Option { + let single_value = matches!( + native, + Object::Int(_) + | Object::Long(_) + | Object::Bool(_) + | Object::Float(_) + | Object::Str(_) + | Object::Bytes(_) + | Object::Tuple(_) + ); + single_value.then(|| Object::Builtin(Rc::new(method("__getnewargs__", instance_getnewargs)))) +} + +/// Like [`method`] but for builtins that accept keyword arguments. The +/// body receives the positional args (with the bound receiver at index +/// 0) *and* the keyword pairs, so it can implement CPython's mixed +/// positional/keyword signatures (e.g. `str.split(sep=None, maxsplit=-1)`, +/// `str.splitlines(keepends=False)`). +fn method_kw( + name: &'static str, + body: impl Fn(&[Object], &[(String, Object)]) -> Result + + Send + + Sync + + 'static, +) -> BuiltinFn { + let body = std::sync::Arc::new(body); + let positional = body.clone(); + BuiltinFn { + name, + call: Box::new(move |args| positional(args, &[])), + call_kw: Some(Box::new(move |args, kwargs| body(args, kwargs))), + } +} + +/// Resolve a parameter that may be passed positionally (`args[pos]`) or +/// by keyword (`kwargs[name]`). Positional wins; returns `None` when the +/// argument is absent so the caller can apply its default. +fn arg_or_kw<'a>( + args: &'a [Object], + pos: usize, + kwargs: &'a [(String, Object)], + name: &str, +) -> Option<&'a Object> { + if let Some(v) = args.get(pos) { + return Some(v); + } + kwargs.iter().find(|(k, _)| k == name).map(|(_, v)| v) +} + /// Built-in classmethod / staticmethod table: `Type.name` access for /// names not stored in the type's ``dict`` (e.g. `str.maketrans`, /// `bytes.fromhex`, `int.from_bytes`, `dict.fromkeys`, @@ -585,6 +1128,40 @@ pub fn builtin_classmethod(type_name: &str, attr: &str) -> Option { f.map(|f| Object::Builtin(Rc::new(f))) } +/// Unbound-method access on a built-in type, e.g. `str.upper`, `float.hex`, +/// `list.append`. CPython exposes every instance method as an attribute of +/// its type that takes the receiver as an explicit first argument; the +/// `BuiltinFn`s in [`lookup_method`] already treat `args[0]` as `self`, so +/// the same function object serves both bound (`x.upper()`) and unbound +/// (`str.upper(x)`) call forms. We synthesise a throw-away representative of +/// the type purely so the variant-based dispatch in [`lookup_method`] can +/// pick the right table — the value is never inspected. +pub fn unbound_method(type_name: &str, name: &str) -> Option { + let rep: Object = match type_name { + "str" => Object::from_static(""), + "float" => Object::Float(0.0), + "int" => Object::Int(0), + "bool" => Object::Bool(false), + "complex" => Object::new_complex(0.0, 0.0), + "bytes" => Object::new_bytes(Vec::::new()), + "bytearray" => Object::new_bytearray(Vec::::new()), + "list" => Object::new_list(Vec::new()), + "tuple" => Object::new_tuple(Vec::new()), + "dict" => Object::new_dict(), + "set" => Object::new_set(), + "frozenset" => Object::new_frozenset_from(std::iter::empty::()), + // A representative (empty) iterator so `type(it).__length_hint__` + // resolves to the unbound slot wrapper; the actual call receives the + // real iterator as `self`. `operator.length_hint` reaches it this way. + "iterator" => Object::Iter(Rc::new(RefCell::new(crate::object::PyIterator::Tuple { + items: Rc::from(Vec::::new()), + index: 0, + }))), + _ => return None, + }; + lookup_method(&rep, name) +} + // ---------- free builtins ---------- fn one<'a>(args: &'a [Object], name: &str) -> Result<&'a Object, RuntimeError> { @@ -597,17 +1174,78 @@ fn b_len(args: &[Object]) -> Result { Ok(Object::Int(v.len()? as i64)) } -fn b_range(args: &[Object]) -> Result { - let to_int = |o: &Object| -> Result { - match o { - Object::Int(i) => Ok(*i), - Object::Bool(b) => Ok(i64::from(*b)), - _ => Err(type_error(format!( - "'{}' object cannot be interpreted as an integer", - o.type_name() - ))), +/// Coerce `o` to an `i64` index the way CPython's `__index__` protocol does: +/// accept ints/bools directly, unwrap integer-backed subclass instances +/// (e.g. `IntEnum` members), and otherwise invoke a Python-level `__index__` +/// via reentry into the running interpreter. Shared by the integer-position +/// builtins (`range`, slicing helpers, …) so they all honour `__index__`. +pub(crate) fn coerce_index_i64(o: &Object) -> Result { + if let Some(v) = o.as_i64() { + return Ok(v); + } + if let Object::Instance(_) = o { + if let Some(method) = crate::instance_method(o, "__index__") { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: the pointer was published by an enclosing VM frame + // still live on this thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + let r = interp.call_object_with_globals(&method, &[], &[], &globals)?; + if let Some(v) = r.as_i64() { + return Ok(v); + } + } } - }; + } + Err(type_error(format!( + "'{}' object cannot be interpreted as an integer", + o.type_name() + ))) +} + +/// Coerce `o` to an `f64` the way CPython's float-accepting C functions +/// (`math.*`, etc.) do: floats/ints/bools/big ints directly, built-in +/// numeric subclass payloads by unwrapping, and otherwise via the Python +/// `__float__` then `__index__` protocol through interpreter reentry. +/// +/// `Ok(None)` means "not coercible" — the caller raises its own +/// function-specific `TypeError`. `Err` propagates an exception raised +/// inside a user `__float__`/`__index__`. +pub(crate) fn coerce_f64_opt(o: &Object) -> Result, RuntimeError> { + match o { + Object::Float(f) => Ok(Some(*f)), + Object::Int(i) => Ok(Some(*i as f64)), + Object::Bool(b) => Ok(Some(if *b { 1.0 } else { 0.0 })), + Object::Long(b) => { + use num_traits::ToPrimitive; + Ok(Some(b.to_f64().unwrap_or(f64::INFINITY))) + } + Object::Instance(inst) => { + if let Some(native) = &inst.native { + let native = native.clone(); + return coerce_f64_opt(&native); + } + for dunder in ["__float__", "__index__"] { + if let Some(method) = crate::instance_method(o, dunder) { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still live + // on this thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + let r = + interp.call_object_with_globals(&method, &[], &[], &globals)?; + return coerce_f64_opt(&r); + } + } + } + Ok(None) + } + _ => Ok(None), + } +} + +fn b_range(args: &[Object]) -> Result { + let to_int = |o: &Object| -> Result { coerce_index_i64(o) }; let (start, stop, step) = match args.len() { 1 => (0, to_int(&args[0])?, 1), 2 => (to_int(&args[0])?, to_int(&args[1])?, 1), @@ -624,10 +1262,54 @@ fn b_range(args: &[Object]) -> Result { Ok(Object::Range(Rc::new(Range { start, stop, step }))) } +/// PEP 0467 int→str conversion cap. Raises `ValueError` when the decimal +/// expansion of `b` would exceed `sys.get_int_max_str_digits()` (0 = off). +/// +/// The expensive base-10 conversion is avoided for pathological inputs: the +/// digit count is first bounded from the bit length, and the exact string is +/// only materialised when the magnitude sits right at the limit (in which +/// case it is small and cheap to convert). +pub(crate) fn long_str_limit_check(b: &num_bigint::BigInt) -> Result<(), RuntimeError> { + let max_digits = crate::stdlib::sys::int_max_str_digits(); + if max_digits <= 0 { + return Ok(()); + } + let limit = max_digits as u64; + let bits = b.bits(); + if bits == 0 { + return Ok(()); // "0" — a single digit, never exceeds the (>=640) cap. + } + const LOG10_2: f64 = std::f64::consts::LOG10_2; + let lower = (((bits - 1) as f64) * LOG10_2).floor() as u64 + 1; + if lower > limit { + return Err(int_to_str_limit_error(max_digits)); + } + let upper = ((bits as f64) * LOG10_2).floor() as u64 + 1; + if upper <= limit { + return Ok(()); + } + // Boundary case: the value is within ~1 digit of the cap, so it is small + // enough to expand exactly without performance risk. + if b.magnitude().to_str_radix(10).len() as u64 > limit { + return Err(int_to_str_limit_error(max_digits)); + } + Ok(()) +} + +fn int_to_str_limit_error(max_digits: i64) -> RuntimeError { + value_error(format!( + "Exceeds the limit ({max_digits} digits) for integer string conversion; \ + use sys.set_int_max_str_digits() to increase the limit" + )) +} + fn b_str(args: &[Object]) -> Result { if args.is_empty() { return Ok(Object::from_static("")); } + if let Object::Long(b) = &args[0] { + long_str_limit_check(b)?; + } // `str(object, encoding[, errors])` decodes a bytes-like object, // equivalent to `object.decode(encoding, errors)`. CPython's // `re._parser.Tokenizer` relies on `str(pattern, 'latin1')` to @@ -661,7 +1343,11 @@ fn b_str(args: &[Object]) -> Result { } fn b_repr(args: &[Object]) -> Result { - Ok(Object::from_str(one(args, "repr")?.repr())) + let v = one(args, "repr")?; + if let Object::Long(b) = v { + long_str_limit_check(b)?; + } + Ok(Object::from_str(v.repr())) } fn b_format(args: &[Object]) -> Result { @@ -1458,37 +2144,40 @@ pub(crate) fn b_int_compat(args: &[Object]) -> Result { crate::object::bigint_from_f64_trunc(truncated), )) } - Object::Str(s) => parse_int_string(s.trim(), &args[1..]), + Object::Str(s) => parse_int_string(&args[0], s, &args[1..]), + // bytes-like: each byte maps to one Latin-1 code point so non-ASCII + // bytes (and embedded NULs) become non-digit characters that fail to + // parse — with the original `b'…'` repr in the error, like CPython. Object::Bytes(b) => { - let s = std::str::from_utf8(b) - .map_err(|_| value_error("int() can't convert non-string with explicit base"))?; - parse_int_string(s.trim(), &args[1..]) + let text: String = b.iter().map(|&c| c as char).collect(); + parse_int_string(&args[0], &text, &args[1..]) } Object::ByteArray(b) => { - let bytes = b.borrow(); - let s = std::str::from_utf8(&bytes) - .map_err(|_| value_error("int() can't convert non-string with explicit base"))?; - parse_int_string(s.trim(), &args[1..]) + let text: String = b.borrow().iter().map(|&c| c as char).collect(); + parse_int_string(&args[0], &text, &args[1..]) } _ => Err(type_error(format!( - "int() argument must be a string or a real number, not '{}'", + "int() argument must be a string, a bytes-like object or a real number, not '{}'", args[0].type_name() ))), } } -fn parse_int_string(s: &str, base_arg: &[Object]) -> Result { +/// Parse the text of an `int(x, base)` call. `original` is the *original* +/// argument object; its `repr()` is computed lazily and only when an +/// `invalid literal` error is actually raised (so surrounding whitespace and +/// `b'…'` framing are preserved, matching CPython, without paying the O(N) +/// repr cost on the success / digit-limit paths). Unicode decimal digits and +/// whitespace are normalised to ASCII first. +fn parse_int_string( + original: &Object, + raw: &str, + base_arg: &[Object], +) -> Result { use num_bigint::BigInt; - let mut s = s; - let mut sign = 1i32; - if let Some(stripped) = s.strip_prefix('+') { - s = stripped; - } else if let Some(stripped) = s.strip_prefix('-') { - s = stripped; - sign = -1; - } - + // Resolve the base argument up front: the error message reports it + // verbatim (`base 0`, `base 20`, …), not the prefix-resolved radix. let base = if base_arg.is_empty() { 10u32 } else { @@ -1496,6 +2185,9 @@ fn parse_int_string(s: &str, base_arg: &[Object]) -> Result u32::try_from(*i) .map_err(|_| value_error("int() base must be >= 2 and <= 36, or 0"))?, Object::Bool(b) => u32::from(*b), + Object::Long(_) => { + return Err(value_error("int() base must be >= 2 and <= 36, or 0")) + } _ => return Err(type_error("int() base must be an integer".to_owned())), } }; @@ -1503,6 +2195,65 @@ fn parse_int_string(s: &str, base_arg: &[Object]) -> Result= 2 and <= 36, or 0")); } + // Fast DoS guard (PEP 0467): reject a pathologically long input *before* + // the O(N) Unicode-normalisation and underscore-stripping passes. A raw + // string of length L yields at least ceil((L+1)/2) digits once the only + // legal underscores (between two digits) are removed, so when that lower + // bound already exceeds the cap the value is over the limit regardless of + // its exact contents. Power-of-two radices parse in linear time and are + // exempt, matching CPython. + let max_digits = crate::stdlib::sys::int_max_str_digits(); + if max_digits > 0 { + let radix_is_pow2 = base.is_power_of_two() + || (base == 0 && { + let t = raw.trim_start(); + let t = t.strip_prefix(['+', '-']).unwrap_or(t); + let tb = t.as_bytes(); + tb.len() >= 2 + && tb[0] == b'0' + && matches!(tb[1], b'x' | b'X' | b'o' | b'O' | b'b' | b'B') + }); + if !radix_is_pow2 && (raw.len() + 1) / 2 > max_digits as usize { + return Err(value_error(format!( + "Exceeds the limit ({max_digits} digits) for integer string conversion; \ + use sys.set_int_max_str_digits() to increase the limit" + ))); + } + } + + let invalid = + || value_error(format!("invalid literal for int() with base {base}: {}", original.repr())); + + // Normalise Unicode decimal digits / whitespace to ASCII, then strip the + // surrounding whitespace CPython ignores. + let transformed = transform_decimal_and_space(raw); + let mut s = transformed.trim(); + let mut sign = 1i32; + if let Some(stripped) = s.strip_prefix('+') { + s = stripped; + } else if let Some(stripped) = s.strip_prefix('-') { + s = stripped; + sign = -1; + } + + // Validate underscore placement up front: CPython only accepts a single + // underscore between two "digit" characters (or right after a base + // prefix, e.g. `0x_ff`). Leading/trailing/doubled underscores such as + // `_1`, `1_`, `1__2` are `ValueError`s rather than silently stripped. + if s.contains('_') { + let b = s.as_bytes(); + for (i, &c) in b.iter().enumerate() { + if c == b'_' + && !(i > 0 + && i + 1 < b.len() + && b[i - 1].is_ascii_alphanumeric() + && b[i + 1].is_ascii_alphanumeric()) + { + return Err(invalid()); + } + } + } + // Strip a 0x/0o/0b prefix when it matches the base, or pick the // base from the prefix when `base == 0`. let (radix, digits): (u32, &str) = @@ -1532,19 +2283,37 @@ fn parse_int_string(s: &str, base_arg: &[Object]) -> Result 0 && !radix.is_power_of_two() && cleaned.len() > max_digits as usize { return Err(value_error(format!( - "invalid literal for int() with base {radix}: '{s}'" + "Exceeds the limit ({max_digits} digits) for integer string conversion: \ + value has {} digits; use sys.set_int_max_str_digits() to increase the limit", + cleaned.len() ))); } if let Ok(small) = i64::from_str_radix(&cleaned, radix) { return Ok(Object::Int(if sign < 0 { -small } else { small })); } - let big = BigInt::parse_bytes(cleaned.as_bytes(), radix).ok_or_else(|| { - value_error(format!( - "invalid literal for int() with base {radix}: '{s}'" - )) - })?; + let big = BigInt::parse_bytes(cleaned.as_bytes(), radix).ok_or_else(invalid)?; let big = if sign < 0 { -big } else { big }; Ok(Object::int_from_bigint(big)) } @@ -1762,16 +2531,38 @@ fn float_hex(args: &[Object]) -> Result { fn float_fromhex(args: &[Object]) -> Result { // First arg is the class (float) for classmethod-style; tolerate // either form. - let s_obj = if matches!(args.first(), Some(Object::Type(_))) { - args.get(1) + let (cls, s_obj) = if matches!(args.first(), Some(Object::Type(_))) { + (args.first(), args.get(1)) } else { - args.first() + (None, args.first()) }; let s = match s_obj { Some(Object::Str(s)) => s.to_string(), _ => return Err(type_error("fromhex() requires a string")), }; - parse_float_hex(&s).map(Object::Float) + let x = parse_float_hex(&s)?; + float_fromhex_wrap(cls, x) +} + +/// Wrap a parsed `fromhex` value in the requested class. For the plain +/// `float` type that's just `Object::Float`; for a subclass we re-enter the +/// interpreter and call `cls(x)` so the subclass's `__new__`/`__init__` +/// run (CPython does `PyObject_CallOneArg(type, result)`). +fn float_fromhex_wrap(cls: Option<&Object>, x: f64) -> Result { + if let Some(Object::Type(t)) = cls { + let bt = crate::builtin_types::builtin_types(); + if !crate::sync::Rc::ptr_eq(t, &bt.float_) { + let ptr = crate::vm_singletons::current_interpreter_ptr().ok_or_else(|| { + type_error("float.fromhex() subclass construction requires a running interpreter") + })?; + // SAFETY: pointer published by the running dispatch loop for this + // thread; re-entered synchronously like the other reentrant + // callbacks (`__hash__`, `__eq__`). + let interp = unsafe { &mut *ptr }; + return interp.call_object(Object::Type(t.clone()), &[Object::Float(x)], &[]); + } + } + Ok(Object::Float(x)) } fn float_as_integer_ratio(args: &[Object]) -> Result { @@ -1780,8 +2571,13 @@ fn float_as_integer_ratio(args: &[Object]) -> Result { Object::Float(f) => *f, _ => return Err(type_error("as_integer_ratio: float expected")), }; - if !f.is_finite() { - return Err(value_error("cannot convert non-finite float")); + if f.is_nan() { + return Err(value_error("cannot convert NaN to integer ratio")); + } + if f.is_infinite() { + return Err(crate::error::overflow_error( + "cannot convert Infinity to integer ratio", + )); } let bits = f.to_bits(); let sign = if (bits >> 63) & 1 == 1 { -1i32 } else { 1 }; @@ -1828,6 +2624,36 @@ fn float_trunc(args: &[Object]) -> Result { } } +fn float_floor(args: &[Object]) -> Result { + match one(args, "__floor__")? { + Object::Float(f) => float_int_part(f.floor()), + _ => Err(type_error("__floor__: float expected")), + } +} + +fn float_ceil(args: &[Object]) -> Result { + match one(args, "__ceil__")? { + Object::Float(f) => float_int_part(f.ceil()), + _ => Err(type_error("__ceil__: float expected")), + } +} + +/// Convert an already-floored/ceiled `f64` to an `int`, raising the same +/// errors CPython's `float.__floor__`/`__ceil__` do for non-finite values. +fn float_int_part(f: f64) -> Result { + if f.is_nan() { + return Err(value_error("cannot convert float NaN to integer")); + } + if f.is_infinite() { + return Err(crate::error::overflow_error( + "cannot convert float infinity to integer", + )); + } + Ok(Object::int_from_bigint(crate::object::bigint_from_f64_trunc( + f, + ))) +} + fn float_round(args: &[Object]) -> Result { let v = one(args, "__round__")?; let f = match v { @@ -1892,59 +2718,264 @@ fn format_float_hex(f: f64) -> String { format!("{sign_str}{m_hex}p{exp_sign}{exponent}") } +/// `float.fromhex` string parser, a faithful port of CPython's +/// `float_fromhex` (`Objects/floatobject.c`). Returns the parsed value +/// (with correct round-half-even in the subnormal range), a `ValueError` +/// for malformed input, or an `OverflowError` for values too large to +/// represent. Works on raw bytes so embedded NULs and multibyte +/// (fullwidth) digits are rejected exactly as CPython rejects them. fn parse_float_hex(s: &str) -> Result { - let s = s.trim(); - let lower = s.to_ascii_lowercase(); - match lower.as_str() { - "nan" | "+nan" | "-nan" => return Ok(f64::NAN), - "inf" | "+inf" | "infinity" | "+infinity" => return Ok(f64::INFINITY), - "-inf" | "-infinity" => return Ok(f64::NEG_INFINITY), - _ => {} + const DBL_MANT_DIG: i64 = 53; + const DBL_MIN_EXP: i64 = -1021; + const DBL_MAX_EXP: i64 = 1024; + let parse_err = || value_error("invalid hexadecimal floating-point string"); + let overflow = || crate::error::overflow_error("hexadecimal value too large to represent as a float"); + + let bytes = s.as_bytes(); + let n = bytes.len(); + let mut i = 0usize; + + // Leading whitespace. + while i < n && is_py_space(bytes[i]) { + i += 1; + } + + // Infinities and nans (consume their own optional sign). + if let Some((val, end)) = parse_inf_or_nan(bytes, i) { + return finish_hex_tail(bytes, end, val); } + // Optional sign. - let mut idx = 0usize; - let bytes = s.as_bytes(); - let sign = if bytes.first() == Some(&b'-') { - idx += 1; - -1.0 + let mut negate = false; + if i < n && bytes[i] == b'-' { + negate = true; + i += 1; + } else if i < n && bytes[i] == b'+' { + i += 1; + } + + // Optional `0x` / `0X` prefix. + let s_store = i; + if i < n && bytes[i] == b'0' { + i += 1; + if i < n && (bytes[i] == b'x' || bytes[i] == b'X') { + i += 1; + } else { + i = s_store; + } + } + + // Coefficient: [. ]. + let coeff_start = i; + while i < n && hex_from_byte(bytes[i]) >= 0 { + i += 1; + } + let dot_store = i; + let coeff_end: usize; + if i < n && bytes[i] == b'.' { + i += 1; + while i < n && hex_from_byte(bytes[i]) >= 0 { + i += 1; + } + coeff_end = i - 1; } else { - if bytes.first() == Some(&b'+') { - idx += 1; + coeff_end = i; + } + + let mut ndigits = coeff_end as i64 - coeff_start as i64; + let fdigits = coeff_end as i64 - dot_store as i64; + if ndigits == 0 { + return Err(parse_err()); + } + let length_limit = core::cmp::min( + DBL_MIN_EXP - DBL_MANT_DIG - i64::MIN / 2, + i64::MAX / 2 + 1 - DBL_MAX_EXP, + ) / 4; + if ndigits > length_limit { + return Err(value_error("hexadecimal string too long to convert")); + } + + // Optional `p `. + let mut exp: i64 = 0; + if i < n && (bytes[i] == b'p' || bytes[i] == b'P') { + i += 1; + let exp_start = i; + if i < n && (bytes[i] == b'-' || bytes[i] == b'+') { + i += 1; } - 1.0 - }; - let rest = &s[idx..]; - let rest = rest - .strip_prefix("0x") - .or_else(|| rest.strip_prefix("0X")) - .ok_or_else(|| value_error("invalid hexadecimal float"))?; - // Split on 'p' / 'P'. - let (mantissa_part, exp_part) = match rest.find(['p', 'P']) { - Some(i) => (&rest[..i], &rest[i + 1..]), - None => return Err(value_error("invalid hexadecimal float")), - }; - let exponent: i32 = exp_part - .parse() - .map_err(|_| value_error("invalid hexadecimal float exponent"))?; - let (int_part, frac_part) = match mantissa_part.find('.') { - Some(i) => (&mantissa_part[..i], &mantissa_part[i + 1..]), - None => (mantissa_part, ""), + if !(i < n && bytes[i].is_ascii_digit()) { + return Err(parse_err()); + } + i += 1; + while i < n && bytes[i].is_ascii_digit() { + i += 1; + } + // `strtol` saturates to LONG_MIN/MAX on overflow; mirror that so a + // gigantic exponent funnels into the overflow/zero branches below. + let exp_text = std::str::from_utf8(&bytes[exp_start..i]).unwrap_or("0"); + exp = exp_text.parse::().unwrap_or(if bytes[exp_start] == b'-' { + i64::MIN + } else { + i64::MAX + }); + } + + // `HEX_DIGIT(j)` — the j'th least-significant hex digit, hopping over the + // radix point for digits in the integer part. + let hex_digit = |j: i64| -> i32 { + let idx = if j < fdigits { + coeff_end as i64 - j + } else { + coeff_end as i64 - 1 - j + }; + hex_from_byte(bytes[idx as usize]) }; - let mut value: f64 = 0.0; - for c in int_part.chars() { - value = value * 16.0 + f64::from(hex_digit(c)?); + + // Discard leading zeros; catch extreme over/underflow. + while ndigits > 0 && hex_digit(ndigits - 1) == 0 { + ndigits -= 1; + } + if ndigits == 0 || exp < i64::MIN / 2 { + return finish_hex_tail(bytes, i, if negate { -0.0 } else { 0.0 }); + } + if exp > i64::MAX / 2 { + return Err(overflow()); + } + + // Adjust exponent for the fractional part. + exp -= 4 * fdigits; + + // `top_exp` = one more than the exponent of the most-significant bit. + let mut top_exp = exp + 4 * (ndigits - 1); + let mut msd = hex_digit(ndigits - 1); + while msd != 0 { + top_exp += 1; + msd /= 2; + } + + if top_exp < DBL_MIN_EXP - DBL_MANT_DIG { + return finish_hex_tail(bytes, i, if negate { -0.0 } else { 0.0 }); + } + if top_exp > DBL_MAX_EXP { + return Err(overflow()); + } + + let lsb = core::cmp::max(top_exp, DBL_MIN_EXP) - DBL_MANT_DIG; + let mut x: f64 = 0.0; + if exp >= lsb { + // No rounding required. + let mut j = ndigits - 1; + while j >= 0 { + x = 16.0 * x + f64::from(hex_digit(j)); + j -= 1; + } + x = crate::stdlib::math::ldexp(x, exp as i32); + return finish_hex_tail(bytes, i, if negate { -x } else { x }); } - let mut frac_factor = 1.0 / 16.0; - for c in frac_part.chars() { - value += f64::from(hex_digit(c)?) * frac_factor; - frac_factor /= 16.0; + + // Rounding required. `key_digit` holds the first bit to round away. + let half_eps = 1i32 << ((lsb - exp - 1) % 4) as u32; + let key_digit = (lsb - exp - 1) / 4; + let mut j = ndigits - 1; + while j > key_digit { + x = 16.0 * x + f64::from(hex_digit(j)); + j -= 1; + } + let digit = hex_digit(key_digit); + x = 16.0 * x + f64::from(digit & (16 - 2 * half_eps)); + + // Round half to even. + if (digit & half_eps) != 0 { + let mut round_up = false; + if (digit & (3 * half_eps - 1)) != 0 + || (half_eps == 8 && key_digit + 1 < ndigits && (hex_digit(key_digit + 1) & 1) != 0) + { + round_up = true; + } else { + let mut k = key_digit - 1; + while k >= 0 { + if hex_digit(k) != 0 { + round_up = true; + break; + } + k -= 1; + } + } + if round_up { + x += f64::from(2 * half_eps); + if top_exp == DBL_MAX_EXP + && x == crate::stdlib::math::ldexp(f64::from(2 * half_eps), DBL_MANT_DIG as i32) + { + // Pre-rounding value was < DBL_MAX, post-rounding == DBL_MAX. + return Err(overflow()); + } + } } - Ok(sign * value * 2f64.powi(exponent)) + x = crate::stdlib::math::ldexp(x, (exp + 4 * key_digit) as i32); + finish_hex_tail(bytes, i, if negate { -x } else { x }) } -fn hex_digit(c: char) -> Result { - c.to_digit(16) - .ok_or_else(|| value_error("invalid hex digit")) +/// CPython `Py_ISSPACE` for the ASCII range (space, tab, newline, vtab, +/// formfeed, carriage return). +fn is_py_space(b: u8) -> bool { + matches!(b, b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r') +} + +/// Value of an ASCII hex digit, or `-1` for anything else (including +/// multibyte UTF-8 lead bytes, so fullwidth digits are rejected). +fn hex_from_byte(b: u8) -> i32 { + match b { + b'0'..=b'9' => (b - b'0') as i32, + b'a'..=b'f' => (b - b'a' + 10) as i32, + b'A'..=b'F' => (b - b'A' + 10) as i32, + _ => -1, + } +} + +/// ASCII case-insensitive match of `pat` at `s[i..]`. +fn ci_match(s: &[u8], i: usize, pat: &[u8]) -> bool { + s.len() >= i + pat.len() && s[i..i + pat.len()].eq_ignore_ascii_case(pat) +} + +/// CPython `_Py_parse_inf_or_nan`: parse an optional sign followed by +/// `inf`/`infinity`/`nan` (case-insensitive). Returns the value and the +/// index just past the match, or `None` if no match. +fn parse_inf_or_nan(s: &[u8], start: usize) -> Option<(f64, usize)> { + let n = s.len(); + let mut i = start; + let mut negate = false; + if i < n && s[i] == b'-' { + negate = true; + i += 1; + } else if i < n && s[i] == b'+' { + i += 1; + } + if ci_match(s, i, b"inf") { + i += 3; + if ci_match(s, i, b"inity") { + i += 5; + } + Some((if negate { f64::NEG_INFINITY } else { f64::INFINITY }, i)) + } else if ci_match(s, i, b"nan") { + i += 3; + Some((if negate { -f64::NAN } else { f64::NAN }, i)) + } else { + None + } +} + +/// Skip trailing ASCII whitespace and require we've reached the end of the +/// string (CPython rejects trailing junk, including bytes past an embedded +/// NUL). +fn finish_hex_tail(s: &[u8], mut i: usize, val: f64) -> Result { + let n = s.len(); + while i < n && is_py_space(s[i]) { + i += 1; + } + if i != n { + return Err(value_error("invalid hexadecimal floating-point string")); + } + Ok(val) } // ---------- classmethod-shaped wrappers used by builtin_types ---------- @@ -1979,12 +3010,13 @@ pub(crate) fn b_bytearray_fromhex_cls(args: &[Object]) -> Result Result { - let _cls = args.first(); + let cls = args.first(); let s = match args.get(1) { Some(Object::Str(s)) => s.to_string(), _ => return Err(type_error("fromhex() argument must be str")), }; - parse_float_hex(&s).map(Object::Float) + let x = parse_float_hex(&s)?; + float_fromhex_wrap(cls, x) } fn parse_hex_bytes(s: &str) -> Result, RuntimeError> { @@ -2036,19 +3068,38 @@ fn b_float(args: &[Object]) -> Result { } match &args[0] { Object::Int(i) => Ok(Object::Float(*i as f64)), - Object::Long(b) => Ok(Object::Float(b.to_f64().unwrap_or(f64::INFINITY))), + Object::Long(b) => { + // CPython raises OverflowError when the magnitude exceeds the + // f64 range rather than silently producing `inf`. + match b.to_f64() { + Some(f) if f.is_finite() => Ok(Object::Float(f)), + _ => Err(crate::error::overflow_error( + "int too large to convert to float", + )), + } + } Object::Bool(b) => Ok(Object::Float(f64::from(*b))), Object::Float(f) => Ok(Object::Float(*f)), - Object::Str(s) => parse_float_str(s.trim()).map(Object::Float), - Object::Bytes(b) => { - let s = std::str::from_utf8(b).map_err(|_| value_error("invalid bytes for float()"))?; - parse_float_str(s.trim()).map(Object::Float) - } - Object::ByteArray(b) => { - let bytes = b.borrow(); - let s = std::str::from_utf8(&bytes) - .map_err(|_| value_error("invalid bytes for float()"))?; - parse_float_str(s.trim()).map(Object::Float) + Object::Str(_) | Object::Bytes(_) | Object::ByteArray(_) | Object::MemoryView(_) => { + // str / bytes-like: bytes-like buffers are decoded as ASCII-ish + // text; non-UTF-8 input simply fails to parse (CPython raises the + // same ValueError). + let text: Option = match &args[0] { + Object::Str(s) => Some(s.to_string()), + Object::Bytes(b) => String::from_utf8(b.to_vec()).ok(), + Object::ByteArray(b) => String::from_utf8(b.borrow().to_vec()).ok(), + Object::MemoryView(mv) => String::from_utf8(mv.to_bytes()).ok(), + _ => unreachable!(), + }; + text.as_deref() + .and_then(parse_float_text) + .map(Object::Float) + .ok_or_else(|| { + value_error(format!( + "could not convert string to float: {}", + args[0].repr() + )) + }) } _ => Err(type_error(format!( "float() argument must be a string or a number, not '{}'", @@ -2057,19 +3108,102 @@ fn b_float(args: &[Object]) -> Result { } } -fn parse_float_str(s: &str) -> Result { - // Special tokens (case-insensitive). CPython accepts these forms. - let lower = s.to_ascii_lowercase(); - match lower.as_str() { - "inf" | "infinity" | "+inf" | "+infinity" => return Ok(f64::INFINITY), - "-inf" | "-infinity" => return Ok(f64::NEG_INFINITY), - "nan" | "+nan" | "-nan" => return Ok(f64::NAN), +/// Parse a `float()` string argument following CPython's grammar: surrounding +/// whitespace is stripped, `inf`/`nan` spellings are accepted, and PEP 515 +/// underscores are honoured only *between* digits. Returns `None` on any +/// malformed input (the caller renders the `could not convert` ValueError). +fn parse_float_text(raw: &str) -> Option { + let transformed = transform_decimal_and_space(raw); + let s = transformed.trim(); + if s.is_empty() || !valid_float_underscores(s) { + return None; + } + let cleaned: String = s.chars().filter(|&c| c != '_').collect(); + match cleaned.to_ascii_lowercase().as_str() { + "inf" | "infinity" | "+inf" | "+infinity" => return Some(f64::INFINITY), + "-inf" | "-infinity" => return Some(f64::NEG_INFINITY), + "nan" | "+nan" => return Some(f64::NAN), + // Preserve the sign bit so `copysign(1.0, float('-nan'))` is -1.0. + "-nan" => return Some(-f64::NAN), _ => {} } - let cleaned: String = s.chars().filter(|c| *c != '_').collect(); - cleaned - .parse() - .map_err(|e: std::num::ParseFloatError| value_error(e.to_string())) + // Reject the bare `inf`/`infinity`/`nan` tokens that Rust's parser also + // accepts (CPython only takes the spellings handled above); everything + // else Rust accepts matches CPython's float grammar closely enough. + if cleaned + .bytes() + .any(|b| b.eq_ignore_ascii_case(&b'i') || b.eq_ignore_ascii_case(&b'n')) + { + return None; + } + cleaned.parse::().ok() +} + +/// CPython's `_PyUnicode_TransformDecimalAndSpaceToASCII`: map Unicode +/// decimal digits to their ASCII value and any Unicode whitespace to a +/// plain space, so `float("\u0663.\u0661\u0664")` and +/// `float("\N{EM SPACE}3.14")` parse. Any other non-ASCII character becomes +/// `'?'` (and truncates), which makes the subsequent parse fail with the +/// same `ValueError` CPython raises. +fn transform_decimal_and_space(raw: &str) -> String { + if raw.is_ascii() { + return raw.to_string(); + } + let mut out = String::with_capacity(raw.len()); + for c in raw.chars() { + if (c as u32) < 127 { + out.push(c); + } else if c.is_whitespace() { + out.push(' '); + } else if let Some(v) = unicode_decimal_value(c) { + out.push((b'0' + v as u8) as char); + } else { + out.push('?'); + break; + } + } + out +} + +/// Decimal value (0–9) of a Unicode `Nd` (Decimal_Number) character, or +/// `None`. Each `Nd` block is exactly ten consecutive code points `0..=9`, +/// so the block's zero is found by walking down while still in category +/// `Nd` (bounded to nine steps). +fn unicode_decimal_value(c: char) -> Option { + use unicode_properties::{GeneralCategory, UnicodeGeneralCategory}; + if let Some(d) = c.to_digit(10) { + return Some(d); + } + if c.general_category() != GeneralCategory::DecimalNumber { + return None; + } + let cp = c as u32; + let mut zero = cp; + while cp - zero < 9 { + match char::from_u32(zero - 1) { + Some(p) if p.general_category() == GeneralCategory::DecimalNumber => zero -= 1, + _ => break, + } + } + Some(cp - zero) +} + +/// PEP 515 underscore rule for decimal float literals: every `_` must sit +/// directly between two ASCII digits (so `1_000` is fine but `_1`, `1_`, +/// `1__0`, `1_.0`, `1e_5` are not). +fn valid_float_underscores(s: &str) -> bool { + let b = s.as_bytes(); + for (i, &c) in b.iter().enumerate() { + if c == b'_' + && !(i > 0 + && b[i - 1].is_ascii_digit() + && i + 1 < b.len() + && b[i + 1].is_ascii_digit()) + { + return false; + } + } + true } fn b_bool(args: &[Object]) -> Result { @@ -2079,10 +3213,26 @@ fn b_bool(args: &[Object]) -> Result { Ok(Object::Bool(args[0].is_truthy())) } -fn b_complex(args: &[Object]) -> Result { +pub(crate) fn b_complex(args: &[Object]) -> Result { if args.is_empty() { return Ok(Object::new_complex(0.0, 0.0)); } + let has_second = args.len() >= 2; + // CPython's `complex_new` ordering: a string `real` is only valid as the + // sole argument; a string `imag` is never valid. Both checks precede the + // numeric coercion (so e.g. `complex({}, '1')` reports the string, not the + // dict). + if let Object::Str(s) = &args[0] { + if has_second { + return Err(type_error( + "complex() can't take second arg if first is a string", + )); + } + return parse_complex_string(s).map(|(r, i)| Object::new_complex(r, i)); + } + if has_second && matches!(&args[1], Object::Str(_)) { + return Err(type_error("complex() second arg can't be a string")); + } let real = match &args[0] { Object::Complex(c) => { return Ok(args.get(1).cloned().map_or_else( @@ -2093,16 +3243,13 @@ fn b_complex(args: &[Object]) -> Result { }, )) } - Object::Str(s) if args.len() == 1 => { - return parse_complex_string(s).map(|(r, i)| Object::new_complex(r, i)); - } Object::Int(_) | Object::Long(_) | Object::Bool(_) | Object::Float(_) => { args[0].as_f64().expect("numeric") } other => { return Err(type_error(format!( - "complex() argument must be a string or a number, not '{}'", - other.type_name() + "complex() first argument must be a string or a number, not '{}'", + other.type_name_owned() ))); } }; @@ -2115,7 +3262,7 @@ fn b_complex(args: &[Object]) -> Result { other => { return Err(type_error(format!( "complex() second argument must be a number, not '{}'", - other.type_name() + other.type_name_owned() ))); } } @@ -2125,63 +3272,187 @@ fn b_complex(args: &[Object]) -> Result { Ok(Object::new_complex(real, imag)) } +/// Parse a `complex(str)` argument, following CPython's +/// `complex_from_string_inner` grammar exactly: +/// +/// ```text +/// - real part only +/// j - imaginary part only +/// j - real and imaginary parts +/// j | j - bare ±1j +/// ``` +/// +/// with an optional pair of `repr()` parentheses, leading/trailing +/// whitespace, and PEP 515 underscores (only between digits). Anything +/// else — trailing garbage, a real part with no `j`, doubled signs, +/// embedded NULs — is a `ValueError`. fn parse_complex_string(s: &str) -> Result<(f64, f64), RuntimeError> { - // CPython accepts an optional pair of parens, then a complex - // number like `1+2j`, `1J`, `2.5e-1+3.4j`, with `j` or `J` - // suffix on the imaginary half. - let trimmed = s.trim(); - let s = trimmed - .strip_prefix('(') - .and_then(|s| s.strip_suffix(')')) - .map(str::trim) - .unwrap_or(trimmed); - if s.is_empty() { - return Err(value_error("complex() arg is an empty string")); - } - // Find a `+`/`-` that splits real and imag, skipping the - // exponent sign in `1e-3`. - let bytes = s.as_bytes(); - let mut split = None; - for i in (1..bytes.len()).rev() { - let c = bytes[i]; - if c == b'+' || c == b'-' { - let prev = bytes[i - 1]; - if prev != b'e' && prev != b'E' { - split = Some(i); - break; + let malformed = || value_error("complex() arg is a malformed string"); + // Fold Unicode whitespace to ASCII space (CPython's + // `_PyUnicode_TransformDecimalAndSpaceToASCII`); non-ASCII, non-space + // characters are left to fail the parse below, exactly as CPython does. + let transformed: String = s + .chars() + .map(|c| if c.is_whitespace() { ' ' } else { c }) + .collect(); + let cleaned = strip_number_underscores(&transformed).ok_or_else(malformed)?; + parse_complex_inner(&cleaned).ok_or_else(malformed) +} + +/// Remove PEP 515 underscores from a numeric literal, validating that +/// each `_` sits directly between two ASCII digits. Returns `None` for a +/// misplaced underscore (leading/trailing/doubled/adjacent to a sign, +/// dot, exponent, or `j`). +fn strip_number_underscores(s: &str) -> Option { + let chars: Vec = s.chars().collect(); + let mut out = String::with_capacity(s.len()); + for (i, &c) in chars.iter().enumerate() { + if c == '_' { + let prev = if i > 0 { chars[i - 1] } else { '\0' }; + let next = chars.get(i + 1).copied().unwrap_or('\0'); + if !(prev.is_ascii_digit() && next.is_ascii_digit()) { + return None; } + } else { + out.push(c); } } - let (real_str, imag_str) = if let Some(i) = split { - (&s[..i], &s[i..]) - } else if s.ends_with('j') || s.ends_with('J') { - ("0", s) - } else { - (s, "0") + Some(out) +} + +/// Scan the longest valid C-`double` prefix of `b` (CPython's +/// `PyOS_string_to_double`): optional sign, then `inf`/`infinity`/`nan` +/// or a decimal mantissa with optional fraction and exponent. Returns +/// `(value, bytes_consumed)`, or `None` when no float prefix is present. +fn parse_double_prefix(b: &[u8]) -> Option<(f64, usize)> { + let n = b.len(); + let mut i = 0; + if i < n && (b[i] == b'+' || b[i] == b'-') { + i += 1; + } + let rest = &b[i..]; + let starts = |word: &[u8]| rest.len() >= word.len() && rest[..word.len()].eq_ignore_ascii_case(word); + let finish = |end: usize| -> Option<(f64, usize)> { + let slice = std::str::from_utf8(&b[..end]).ok()?; + slice.parse::().ok().map(|v| (v, end)) }; - let parse_part = |t: &str| -> Result { - let stripped = t.strip_suffix(['j', 'J']).unwrap_or(t); - if stripped.is_empty() || stripped == "+" { - return Ok(1.0); + if starts(b"infinity") { + return finish(i + 8); + } + if starts(b"inf") { + return finish(i + 3); + } + if starts(b"nan") { + return finish(i + 3); + } + let mut has_digits = false; + while i < n && b[i].is_ascii_digit() { + i += 1; + has_digits = true; + } + if i < n && b[i] == b'.' { + i += 1; + while i < n && b[i].is_ascii_digit() { + i += 1; + has_digits = true; } - if stripped == "-" { - return Ok(-1.0); + } + if !has_digits { + return None; + } + if i < n && (b[i] == b'e' || b[i] == b'E') { + let mut j = i + 1; + if j < n && (b[j] == b'+' || b[j] == b'-') { + j += 1; } - stripped - .parse::() - .map_err(|_| value_error(format!("complex() arg is malformed: '{s}'"))) - }; - let imag_is_imag = imag_str.ends_with('j') || imag_str.ends_with('J'); - let real_is_imag = real_str.ends_with('j') || real_str.ends_with('J'); - if real_is_imag && !imag_is_imag { - // Single imaginary like "5j+0" — unusual; treat as 5j+0. - let real = parse_part(imag_str)?; - let imag = parse_part(real_str)?; - return Ok((real, imag)); + if j < n && b[j].is_ascii_digit() { + while j < n && b[j].is_ascii_digit() { + j += 1; + } + i = j; + } + // No exponent digits ⇒ stop before the `e` (e.g. "1e1ej"). + } + finish(i) +} + +/// The core of [`parse_complex_string`], operating on an +/// underscore-stripped, whitespace-normalized string. Mirrors CPython's +/// `complex_from_string_inner` state machine; returns `None` on any +/// malformed input. +fn parse_complex_inner(s: &str) -> Option<(f64, f64)> { + let b = s.as_bytes(); + let len = b.len(); + let mut i = 0; + let is_space = |c: u8| matches!(c, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c); + while i < len && is_space(b[i]) { + i += 1; + } + let mut got_bracket = false; + if i < len && b[i] == b'(' { + got_bracket = true; + i += 1; + while i < len && is_space(b[i]) { + i += 1; + } + } + let (mut x, mut y) = (0.0_f64, 0.0_f64); + match parse_double_prefix(&b[i..]) { + Some((z, consumed)) => { + i += consumed; + if i < len && (b[i] == b'+' || b[i] == b'-') { + x = z; + match parse_double_prefix(&b[i..]) { + Some((yy, c2)) => { + y = yy; + i += c2; + } + None => { + y = if b[i] == b'+' { 1.0 } else { -1.0 }; + i += 1; + } + } + if !(i < len && (b[i] == b'j' || b[i] == b'J')) { + return None; + } + i += 1; + } else if i < len && (b[i] == b'j' || b[i] == b'J') { + i += 1; + y = z; + } else { + x = z; + } + } + None => { + // No leading float ⇒ must be `j` or bare `j`. + if i < len && (b[i] == b'+' || b[i] == b'-') { + y = if b[i] == b'+' { 1.0 } else { -1.0 }; + i += 1; + } else { + y = 1.0; + } + if !(i < len && (b[i] == b'j' || b[i] == b'J')) { + return None; + } + i += 1; + } + } + while i < len && is_space(b[i]) { + i += 1; + } + if got_bracket { + if !(i < len && b[i] == b')') { + return None; + } + i += 1; + while i < len && is_space(b[i]) { + i += 1; + } + } + if i != len { + return None; } - let real = parse_part(real_str)?; - let imag = parse_part(imag_str)?; - Ok((real, imag)) + Some((x, y)) } fn b_list(args: &[Object]) -> Result { @@ -2445,7 +3716,7 @@ fn b_open(args: &[Object]) -> Result { )))) } -fn b_abs(args: &[Object]) -> Result { +pub(crate) fn b_abs(args: &[Object]) -> Result { match one(args, "abs")? { Object::Int(i) => match i.checked_abs() { Some(v) => Ok(Object::Int(v)), @@ -2454,7 +3725,17 @@ fn b_abs(args: &[Object]) -> Result { }, Object::Long(b) => Ok(Object::int_from_bigint(b.abs())), Object::Float(f) => Ok(Object::Float(f.abs())), - Object::Complex(c) => Ok(Object::Float((c.real * c.real + c.imag * c.imag).sqrt())), + Object::Complex(c) => { + // `hypot` (CPython's `_Py_c_abs`) avoids the spurious overflow + // of `sqrt(re²+im²)`; a non-finite result from finite parts is + // a genuine magnitude overflow → OverflowError, matching + // CPython's `complex___abs___impl`. + let m = c.real.hypot(c.imag); + if m.is_infinite() && c.real.is_finite() && c.imag.is_finite() { + return Err(crate::error::overflow_error("absolute value too large")); + } + Ok(Object::Float(m)) + } Object::Bool(b) => Ok(Object::Int(i64::from(*b))), other => Err(type_error(format!( "bad operand type for abs(): '{}'", @@ -2547,14 +3828,19 @@ fn b_enumerate(args: &[Object]) -> Result { } else { 0 }; - let mut it = iterable.make_iter()?; - let mut buf = Vec::new(); - let mut i = start; - while let Some(v) = it.next_value() { - buf.push(Object::new_tuple(vec![Object::Int(i), v])); - i += 1; - } - Ok(Object::new_list(buf)) + // CPython's `enumerate(x)` wraps `iter(x)` lazily. When `x` is already an + // iterator, `iter(x)` returns `x` itself, so consuming the enumerate must + // advance the *same* iterator (test_operator's `indexOf` relies on the + // source iterator being left at the position after the match). Share the + // handle for `Object::Iter`; otherwise build a fresh underlying iterator. + let inner = match iterable { + Object::Iter(rc) => rc.clone(), + other => Rc::new(RefCell::new(other.make_iter()?)), + }; + Ok(Object::Iter(Rc::new(RefCell::new(PyIterator::Enumerate { + inner, + count: start, + })))) } fn b_zip(args: &[Object]) -> Result { @@ -2646,8 +3932,23 @@ fn b_super(args: &[Object]) -> Result { /// zero-arg super objects. pub fn make_super(class: Rc, receiver: Object) -> Object { use crate::types::TypeObject; + // CPython's `super(C, obj_or_type)` (see `super_init_impl`) chooses + // which MRO to walk from the *second* argument: + // * `obj` is an instance → walk `type(obj)`'s MRO. + // * `obj` is a type & subclass → "bound-to-subclass" form, walk + // of `C` `obj`'s own MRO (classmethods and + // the implicit `super()` inside + // `__init_subclass__` / `__new__`). + // * `obj` is a type but NOT a → metaclass-method form (`obj` is an + // subclass of `C` *instance* of the metaclass `C`), + // walk `type(obj)`'s MRO. + // Collapsing the two type cases into one (always `obj`'s MRO, or + // always `C`'s MRO) breaks either diamond `__init_subclass__` or + // `super().__init__()` inside a metaclass, respectively. let receiver_class = match &receiver { Object::Instance(inst) => inst.class.clone(), + Object::Type(t) if t.is_subclass_of(&class) => t.clone(), + Object::Type(t) => t.metaclass_or_type(), _ => class.clone(), }; let mro = receiver_class.mro.borrow(); @@ -2666,6 +3967,8 @@ pub fn make_super(class: Rc, receiver: Object) -> Obje metaclass: RefCell::new(None), slot_names: RefCell::new(Vec::new()), forbids_dict: false, + subclasses: RefCell::new(Vec::new()), + getattribute_kind: crate::sync::Cell::new(0), }); let inst = crate::types::PyInstance { class: proxy, @@ -2786,7 +4089,10 @@ pub fn class_of(obj: &Object) -> crate::sync::Rc { }, Object::SimpleNamespace(_) => bt.simple_namespace_.clone(), Object::Type(t) => t.metaclass_or_type(), - Object::Function(_) | Object::Builtin(_) | Object::BoundMethod(_) => bt.function_.clone(), + Object::Function(_) | Object::Builtin(_) => bt.function_.clone(), + // A bound method is its own type in CPython (`type(o.m)` is `method`), + // which also makes `types.MethodType(func, obj)` construct one. + Object::BoundMethod(_) => bt.method_.clone(), Object::Property(_) => bt.property_.clone(), Object::StaticMethod(_) => bt.staticmethod_.clone(), Object::ClassMethod(_) => bt.classmethod_.clone(), @@ -2907,12 +4213,44 @@ fn object_identity(obj: &Object) -> i64 { /// Structural hash for primitives. Mirrors CPython's "hash by value" /// semantics for the built-in immutable types we support. +/// Reject values that cannot serve as a dict/set key, matching CPython: +/// `list`/`dict`/`set`/`bytearray`/`slice` are unhashable, and a `tuple` +/// is unhashable iff any element is (the hash recurses). `frozenset` is +/// hashable by construction. Instances carry their own `__hash__`/`None` +/// marker handled by the VM's `do_hash_call`, so they pass here. +pub fn ensure_hashable(obj: &Object) -> Result<(), RuntimeError> { + let name = match obj { + Object::List(_) => "list", + Object::Dict(_) => "dict", + Object::Set(_) => "set", + Object::ByteArray(_) => "bytearray", + Object::Slice(_) => "slice", + Object::Tuple(items) => { + for it in items.iter() { + ensure_hashable(it)?; + } + return Ok(()); + } + _ => return Ok(()), + }; + Err(type_error(format!("unhashable type: '{name}'"))) +} + pub fn hash_object(obj: &Object) -> Result { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - let mut h = DefaultHasher::new(); - crate::object::DictKey(obj.clone()).hash(&mut h); - Ok(Object::Int(h.finish() as i64)) + ensure_hashable(obj)?; + // Single source of truth shared with `DictKey`'s hasher: the numeric + // tower uses CPython's exact reduction modulo 2**61-1 (so equal values of + // different numeric types hash identically and specials match + // `sys.hash_info`); `str`/`bytes`/`tuple`/`frozenset` get a stable + // value hash; an int/str/… subclass hashes as its wrapped value; a custom + // `__hash__` is dispatched through the interpreter. Everything else hashes + // by allocation identity. Keeping `hash()` and dict bucketing in lockstep + // is what makes custom `__hash__`/`__eq__` keys interoperate with built-in + // values in a `set`/`dict`. + if let Some(h) = crate::object::py_hash_value(obj) { + return Ok(Object::Int(h)); + } + Ok(Object::Int(crate::object::identity_hash(obj))) } fn b_hash(args: &[Object]) -> Result { @@ -3077,7 +4415,7 @@ fn b_input_unsupported(_args: &[Object]) -> Result { /// given, otherwise `base ** exp`. Mirrors CPython's three-arg /// `pow` including the negative-exponent + mod case (the modular /// inverse). -fn b_pow(args: &[Object]) -> Result { +pub(crate) fn b_pow(args: &[Object]) -> Result { if args.len() < 2 || args.len() > 3 { return Err(type_error("pow() takes 2 or 3 arguments")); } @@ -3096,12 +4434,30 @@ fn b_pow(args: &[Object]) -> Result { /// covers ints, floats, complex, and bool. Mirrors the /// integer/float/complex arithmetic the VM's `BinaryOp::Pow` does /// inline. +/// `float ** float` shared by `pow()` and the `**` operator: a finite +/// negative power of zero is a `ZeroDivisionError`, a fractional power of a +/// negative base yields a `complex` (CPython promotes rather than NaN-ing). +fn float_pow_value(x: f64, y: f64) -> Result { + if x == 0.0 && y < 0.0 && y.is_finite() { + return Err(crate::error::zero_division_error( + "0.0 cannot be raised to a negative power", + )); + } + if x < 0.0 && y.fract() != 0.0 && x.is_finite() && y.is_finite() { + let magnitude = (-x).powf(y); + let theta = std::f64::consts::PI * y; + Ok(Object::new_complex(magnitude * theta.cos(), magnitude * theta.sin())) + } else { + Ok(Object::Float(x.powf(y))) + } +} + fn pow_simple(base: &Object, exp: &Object) -> Result { use num_traits::ToPrimitive; match (base, exp) { (Object::Int(x), Object::Int(y)) => { if *y < 0 { - Ok(Object::Float((*x as f64).powf(*y as f64))) + float_pow_value(*x as f64, *y as f64) } else if let Ok(e) = u32::try_from(*y) { if let Some(r) = x.checked_pow(e) { Ok(Object::Int(r)) @@ -3113,15 +4469,15 @@ fn pow_simple(base: &Object, exp: &Object) -> Result { Err(value_error("pow() exponent too large")) } } - (Object::Int(x), Object::Float(y)) => Ok(Object::Float((*x as f64).powf(*y))), - (Object::Float(x), Object::Int(y)) => Ok(Object::Float(x.powf(*y as f64))), - (Object::Float(x), Object::Float(y)) => Ok(Object::Float(x.powf(*y))), + (Object::Int(x), Object::Float(y)) => float_pow_value(*x as f64, *y), + (Object::Float(x), Object::Int(y)) => float_pow_value(*x, *y as f64), + (Object::Float(x), Object::Float(y)) => float_pow_value(*x, *y), (Object::Bool(b), other) => pow_simple(&Object::Int(i64::from(*b)), other), (other, Object::Bool(b)) => pow_simple(other, &Object::Int(i64::from(*b))), (Object::Long(x), Object::Int(y)) => { if *y < 0 { let xf = x.to_f64().ok_or_else(|| value_error("int too large"))?; - Ok(Object::Float(xf.powf(*y as f64))) + float_pow_value(xf, *y as f64) } else if let Ok(e) = u32::try_from(*y) { Ok(Object::int_from_bigint(x.as_ref().pow(e))) } else { @@ -3159,27 +4515,67 @@ fn pow_modular(base: &Object, exp: &Object, m: &Object) -> Result { + base_mod = inv; + exp_val = -e; + } + None => { + return Err(value_error( + "base is not invertible for the given modulus", + )) + } + } + } let mut result: BigInt = BigInt::one(); let zero: BigInt = BigInt::from(0i64); while exp_val > zero { if &exp_val % 2i64 == BigInt::one() { - result = (&result * &base_mod) % &mm; + result = (&result * &base_mod) % &m_abs; } exp_val >>= 1; - base_mod = (&base_mod * &base_mod) % &mm; + base_mod = (&base_mod * &base_mod) % &m_abs; + } + // `result` is in [0, |m|); shift into (m, 0] when the modulus is negative + // so the sign matches CPython's `int.__mod__` convention. + if mm.is_negative() && !result.is_zero() { + result += &mm; } Ok(Object::int_from_bigint(result)) } +/// Modular inverse of `a` (already reduced into `[0, m)`) modulo `m > 0`, via +/// the iterative extended Euclidean algorithm. Returns `None` when `a` and `m` +/// are not coprime (no inverse exists). Result is normalised into `[0, m)`. +fn mod_inverse(a: &num_bigint::BigInt, m: &num_bigint::BigInt) -> Option { + use num_bigint::BigInt; + use num_traits::{One, Zero}; + let (mut old_r, mut r) = (a.clone(), m.clone()); + let (mut old_s, mut s) = (BigInt::one(), BigInt::zero()); + while !r.is_zero() { + let q = &old_r / &r; + let new_r = &old_r - &q * &r; + old_r = std::mem::replace(&mut r, new_r); + let new_s = &old_s - &q * &s; + old_s = std::mem::replace(&mut s, new_s); + } + if !old_r.is_one() { + return None; + } + Some(((old_s % m) + m) % m) +} + fn bigint_from(o: &Object, fn_name: &str) -> Result { match o { Object::Int(i) => Ok(BigInt::from(*i)), @@ -3292,7 +4688,7 @@ fn b_iter(args: &[Object]) -> Result { Ok(Object::Iter(Rc::new(RefCell::new(it)))) } -fn b_divmod(args: &[Object]) -> Result { +pub(crate) fn b_divmod(args: &[Object]) -> Result { if args.len() != 2 { return Err(type_error("divmod expected 2 arguments")); } @@ -3301,14 +4697,23 @@ fn b_divmod(args: &[Object]) -> Result { Ok(Object::new_tuple(vec![q, r])) } -fn b_round(args: &[Object]) -> Result { +pub(crate) fn b_round(args: &[Object]) -> Result { let value = args .first() .ok_or_else(|| type_error("round() takes at least one argument"))?; + // `ndigits` must be an integer (or omitted); a `Long` is saturated to + // `i64` (anything beyond ±323 short-circuits anyway). let ndigits = match args.get(1) { None | Some(Object::None) => None, Some(Object::Int(i)) => Some(*i), Some(Object::Bool(b)) => Some(i64::from(*b)), + Some(Object::Long(b)) => { + Some(b.to_i64().unwrap_or(if b.is_negative() { + i64::MIN + } else { + i64::MAX + })) + } Some(other) => { return Err(type_error(format!( "'{}' object cannot be interpreted as an integer", @@ -3317,26 +4722,124 @@ fn b_round(args: &[Object]) -> Result { } }; match value { - Object::Int(i) => match ndigits { - None | Some(0) => Ok(Object::Int(*i)), - Some(n) if n > 0 => Ok(Object::Int(*i)), - Some(n) => { - let scale = 10i64.pow(n.unsigned_abs() as u32); - let rounded = ((*i as f64) / scale as f64).round() as i64 * scale; - Ok(Object::Int(rounded)) - } - }, + Object::Int(_) | Object::Long(_) | Object::Bool(_) => round_int(value, ndigits), Object::Float(f) => match ndigits { - None => Ok(Object::Float(f.round())), - Some(n) => { - let factor = 10f64.powi(n as i32); - Ok(Object::Float((f * factor).round() / factor)) + // `round(x)` (no ndigits) rounds to the nearest integer + // (ties-to-even) and returns an `int`. + None => { + if f.is_nan() { + return Err(value_error("cannot convert float NaN to integer")); + } + if f.is_infinite() { + return Err(crate::error::overflow_error( + "cannot convert float infinity to integer", + )); + } + Ok(float_to_int_obj(round_ties_even(*f))) } + // `round(x, n)` returns a `float`, correctly rounded (ties-to-even) + // to `n` decimal places. + Some(n) => double_round(*f, n).map(Object::Float), }, _ => Err(type_error("round() argument must be int or float")), } } +/// Round a finite `f64` to the nearest integer, ties to even. +fn round_ties_even(x: f64) -> f64 { + let r = x.round(); + if (x - x.trunc()).abs() == 0.5 && (r / 2.0).fract() != 0.0 { + // `x` was a half-integer and `round()` (ties-away) landed on an odd + // integer; step toward the even neighbour. + r - x.signum() + } else { + r + } +} + +/// Convert an integral `f64` to `int`/`Long`, used by `round(x)`. +fn float_to_int_obj(r: f64) -> Object { + if r >= -(9.223_372_036_854_776e18) && r < 9.223_372_036_854_776e18 { + Object::Int(r as i64) + } else { + BigInt::from_f64(r).map_or(Object::Int(0), |b| Object::Long(Rc::new(b))) + } +} + +/// `round(int_like, ndigits)` — non-negative `ndigits` leave the value +/// unchanged; negative `ndigits` round to a power of ten (ties-to-even). +fn round_int(value: &Object, ndigits: Option) -> Result { + let n = match ndigits { + None => return Ok(value.clone()), + Some(n) if n >= 0 => return Ok(value.clone()), + Some(n) => n, + }; + // Negative ndigits: round to 10^(-n) via BigInt to stay exact. + let v = match value { + Object::Int(i) => BigInt::from(*i), + Object::Bool(b) => BigInt::from(i64::from(*b)), + Object::Long(b) => (**b).clone(), + _ => unreachable!(), + }; + let pow = (-n) as u32; + let scale = BigInt::from(10).pow(pow); + let q = &v / &scale; + let r = &v - &q * &scale; + let mut result = q.clone(); + let two = BigInt::from(2); + // Compare |remainder|*2 to the scale to decide rounding, breaking exact + // ties toward the even quotient (CPython's round-half-to-even). + let round_up = match (r.abs() * &two).cmp(&scale) { + std::cmp::Ordering::Greater => true, + std::cmp::Ordering::Less => false, + std::cmp::Ordering::Equal => (&q % &two) != BigInt::from(0), + }; + if round_up { + if v.is_negative() { + result -= 1; + } else { + result += 1; + } + } + let scaled = result * &scale; + Ok(Object::int_from_bigint(scaled)) +} + +/// CPython's `double_round`: round `x` to `ndigits` decimal places with +/// round-half-to-even, returning a `float`. Uses round-trip decimal +/// formatting (Rust's formatter rounds ties-to-even, matching dtoa). +fn double_round(x: f64, ndigits: i64) -> Result { + if !x.is_finite() || x == 0.0 { + return Ok(x); + } + // Outside the representable decimal range nothing changes / underflows. + if ndigits > 323 { + return Ok(x); + } + if ndigits < -308 { + return Ok(0.0 * x); + } + if ndigits >= 0 { + let s = format!("{:.*}", ndigits as usize, x); + let r: f64 = s.parse().unwrap_or(x); + if r.is_infinite() { + return Err(crate::error::overflow_error( + "rounded value too large to represent", + )); + } + Ok(r) + } else { + let scale = 10f64.powi((-ndigits) as i32); + let r = round_ties_even(x / scale) * scale; + if r.is_infinite() { + return Err(crate::error::overflow_error( + "rounded value too large to represent", + )); + } + Ok(r) + } +} + // ---------- str methods ---------- fn str_self(args: &[Object]) -> Result<&str, RuntimeError> { @@ -3358,15 +4861,68 @@ fn str_strip(args: &[Object]) -> Result { Ok(Object::from_str(str_self(args)?.trim().to_owned())) } -fn str_split(args: &[Object]) -> Result { +fn split_maxsplit(o: Option<&Object>) -> Result { + match o { + None | Some(Object::None) => Ok(-1), + Some(Object::Int(n)) => Ok(*n), + Some(Object::Bool(b)) => Ok(i64::from(*b)), + Some(_) => Err(type_error("maxsplit must be an integer")), + } +} + +/// `str.split` on runs of whitespace (the `sep is None` case), honouring +/// `maxsplit`. Leading/trailing whitespace is stripped and empty fields +/// are dropped, matching CPython. +fn str_split_whitespace(s: &str, maxsplit: i64) -> Vec { + if maxsplit < 0 { + return s.split_whitespace().map(Object::from_str).collect(); + } + let chars: Vec<(usize, char)> = s.char_indices().collect(); + let n = chars.len(); + let mut out = Vec::new(); + let mut i = 0; + let mut splits = 0; + while i < n { + while i < n && chars[i].1.is_whitespace() { + i += 1; + } + if i >= n { + break; + } + if splits >= maxsplit { + out.push(Object::from_str(s[chars[i].0..].to_string())); + return out; + } + let start = chars[i].0; + while i < n && !chars[i].1.is_whitespace() { + i += 1; + } + let end = if i < n { chars[i].0 } else { s.len() }; + out.push(Object::from_str(s[start..end].to_string())); + splits += 1; + } + out +} + +fn str_split(args: &[Object], kwargs: &[(String, Object)]) -> Result { let s = str_self(args)?; - let out: Vec = if args.len() == 1 { - s.split_whitespace().map(Object::from_str).collect() - } else { - match &args[1] { - Object::Str(sep) => s.split(&**sep).map(Object::from_str).collect(), - _ => return Err(type_error("split() argument must be str")), + let sep = arg_or_kw(args, 1, kwargs, "sep"); + let maxsplit = split_maxsplit(arg_or_kw(args, 2, kwargs, "maxsplit"))?; + let out: Vec = match sep { + None | Some(Object::None) => str_split_whitespace(s, maxsplit), + Some(Object::Str(sep)) => { + if sep.is_empty() { + return Err(value_error("empty separator")); + } + if maxsplit < 0 { + s.split(&**sep).map(Object::from_str).collect() + } else { + s.splitn((maxsplit as usize).saturating_add(1), &**sep) + .map(Object::from_str) + .collect() + } } + Some(_) => return Err(type_error("must be str or None, not other")), }; Ok(Object::new_list(out)) } @@ -3613,46 +5169,70 @@ fn str_rstrip(args: &[Object]) -> Result { Ok(Object::from_str(out)) } -fn str_rsplit(args: &[Object]) -> Result { - let s = str_self(args)?; - let maxsplit = args.get(2).and_then(|x| match x { - Object::Int(i) => Some(*i), - _ => None, - }); - let out: Vec = match args.get(1) { - None | Some(Object::None) => { - let mut parts: Vec<&str> = s.split_whitespace().collect(); - if let Some(n) = maxsplit { - if n >= 0 && (n as usize) < parts.len() - 1 { - let _keep = parts.len() - n as usize; - } - } - parts.reverse(); - parts.reverse(); - parts.into_iter().map(Object::from_str).collect() +/// `str.rsplit` on runs of whitespace, honouring `maxsplit` from the +/// right. Mirrors CPython: the *last* `maxsplit` whitespace runs split, +/// and the left remainder keeps its internal spacing. +fn str_rsplit_whitespace(s: &str, maxsplit: i64) -> Vec { + if maxsplit < 0 { + return s.split_whitespace().map(Object::from_str).collect(); + } + let chars: Vec<(usize, char)> = s.char_indices().collect(); + let n = chars.len(); + let mut out_rev: Vec = Vec::new(); + let mut i = n; + let mut splits = 0; + while i > 0 { + while i > 0 && chars[i - 1].1.is_whitespace() { + i -= 1; + } + if i == 0 { + break; + } + let end_byte = if i < n { chars[i].0 } else { s.len() }; + if splits >= maxsplit { + out_rev.push(s[..end_byte].to_string()); + break; + } + while i > 0 && !chars[i - 1].1.is_whitespace() { + i -= 1; } + let start_byte = chars[i].0; + out_rev.push(s[start_byte..end_byte].to_string()); + splits += 1; + } + out_rev.reverse(); + out_rev.into_iter().map(Object::from_str).collect() +} + +fn str_rsplit(args: &[Object], kwargs: &[(String, Object)]) -> Result { + let s = str_self(args)?; + let sep = arg_or_kw(args, 1, kwargs, "sep"); + let maxsplit = split_maxsplit(arg_or_kw(args, 2, kwargs, "maxsplit"))?; + let out: Vec = match sep { + None | Some(Object::None) => str_rsplit_whitespace(s, maxsplit), Some(Object::Str(sep)) => { - let pieces: Vec<&str> = if let Some(n) = maxsplit { - if n >= 0 { - s.rsplitn(n as usize + 1, &**sep).collect::>() - } else { - s.split(&**sep).collect() - } - } else { + if sep.is_empty() { + return Err(value_error("empty separator")); + } + let mut pieces: Vec<&str> = if maxsplit < 0 { s.split(&**sep).collect() + } else { + let mut v: Vec<&str> = s.rsplitn((maxsplit as usize).saturating_add(1), &**sep).collect(); + v.reverse(); + v }; - let mut v = pieces; - v.reverse(); - v.into_iter().map(Object::from_str).collect() + pieces.drain(..).map(Object::from_str).collect() } - _ => return Err(type_error("rsplit() argument must be str")), + Some(_) => return Err(type_error("must be str or None, not other")), }; Ok(Object::new_list(out)) } -fn str_splitlines(args: &[Object]) -> Result { +fn str_splitlines(args: &[Object], kwargs: &[(String, Object)]) -> Result { let s = str_self(args)?; - let keepends = matches!(args.get(1), Some(Object::Bool(true))); + let keepends = arg_or_kw(args, 1, kwargs, "keepends") + .map(Object::is_truthy) + .unwrap_or(false); let mut out: Vec = Vec::new(); let bytes = s.as_bytes(); let mut start = 0; @@ -3859,7 +5439,10 @@ fn str_isprintable(args: &[Object]) -> Result { fn str_zfill(args: &[Object]) -> Result { let s = str_self(args)?; let width = match args.get(1) { - Some(Object::Int(i)) => *i as usize, + // A negative width is a no-op in CPython (`'x'.zfill(-3) == 'x'`); + // clamp to 0 so `*i as usize` can't wrap to a gigantic pad count. + Some(Object::Int(i)) => (*i).max(0) as usize, + Some(Object::Bool(b)) => usize::from(*b), _ => return Err(type_error("zfill() expected int")), }; let len = s.chars().count(); @@ -3886,7 +5469,10 @@ fn str_rjust(args: &[Object]) -> Result { fn pad_str(args: &[Object], right_align: bool) -> Result { let s = str_self(args)?; let width = match args.get(1) { - Some(Object::Int(i)) => *i as usize, + // Negative widths are no-ops in CPython (`'x'.ljust(-3) == 'x'`); + // clamp so the `as usize` cast can't underflow to a huge pad count. + Some(Object::Int(i)) => (*i).max(0) as usize, + Some(Object::Bool(b)) => usize::from(*b), _ => return Err(type_error("expected int width")), }; let fill = match args.get(2) { @@ -3909,7 +5495,10 @@ fn pad_str(args: &[Object], right_align: bool) -> Result { fn str_center(args: &[Object]) -> Result { let s = str_self(args)?; let width = match args.get(1) { - Some(Object::Int(i)) => *i as usize, + // Negative widths are no-ops in CPython; clamp to avoid an `as usize` + // underflow that would request a gigantic allocation. + Some(Object::Int(i)) => (*i).max(0) as usize, + Some(Object::Bool(b)) => usize::from(*b), _ => return Err(type_error("center() expected int")), }; let fill = match args.get(2) { @@ -3922,7 +5511,10 @@ fn str_center(args: &[Object]) -> Result { return Ok(Object::from_str(s.to_owned())); } let total = width - len; - let left = total / 2; + // CPython biases the extra pad to the *left* when both the margin and the + // width are odd (`marg / 2 + (marg & width & 1)`), so `'Monday'.center(9)` + // is `' Monday '`, not `' Monday '`. + let left = total / 2 + (total & width & 1); let right = total - left; let lpad: String = std::iter::repeat_n(fill, left).collect(); let rpad: String = std::iter::repeat_n(fill, right).collect(); @@ -4094,6 +5686,13 @@ fn str_maketrans(args: &[Object]) -> Result { fn list_self(args: &[Object]) -> Result>>, RuntimeError> { match args.first() { Some(Object::List(l)) => Ok(l.clone()), + // A subclass of `list` (`class C(list)`) carries its items in the + // wrapped native payload. Unbound calls — `list.append(c, x)`, + // `super().append(x)` — pass the instance, so unwrap it here. + Some(Object::Instance(inst)) => match &inst.native { + Some(Object::List(l)) => Ok(l.clone()), + _ => Err(type_error("expected list method receiver")), + }, _ => Err(type_error("expected list method receiver")), } } @@ -4106,6 +5705,64 @@ fn list_append(args: &[Object]) -> Result { Ok(Object::None) } +// List dunders exposed on the type so `list.__setitem__` / +// `super().__getitem__` resolve for `list` subclasses (`class C(list)`). +// Integer indices only — slice subscription routes through the VM's +// dedicated subscript opcodes, not this unbound-method path. +fn list_index_arg(l_len: usize, idx: &Object, what: &str) -> Result { + match idx { + Object::Int(i) => { + let len = l_len as i64; + let n = if *i < 0 { i + len } else { *i }; + if n < 0 || n >= len { + Err(index_error("list index out of range")) + } else { + Ok(n as usize) + } + } + Object::Bool(b) => list_index_arg(l_len, &Object::Int(i64::from(*b)), what), + _ => Err(type_error(format!( + "list indices must be integers or slices, not {}", + idx.type_name() + ))), + } +} + +fn list_getitem(args: &[Object]) -> Result { + let l = list_self(args)?; + let key = args + .get(1) + .ok_or_else(|| type_error("__getitem__ expected 1 argument"))?; + let l = l.borrow(); + let n = list_index_arg(l.len(), key, "__getitem__")?; + Ok(l[n].clone()) +} + +fn list_setitem(args: &[Object]) -> Result { + let l = list_self(args)?; + let key = args + .get(1) + .ok_or_else(|| type_error("__setitem__ expected 2 arguments"))?; + let val = args + .get(2) + .ok_or_else(|| type_error("__setitem__ expected 2 arguments"))?; + let mut l = l.borrow_mut(); + let n = list_index_arg(l.len(), key, "__setitem__")?; + l[n] = val.clone(); + Ok(Object::None) +} + +fn list_delitem(args: &[Object]) -> Result { + let l = list_self(args)?; + let key = args + .get(1) + .ok_or_else(|| type_error("__delitem__ expected 1 argument"))?; + let mut l = l.borrow_mut(); + let n = list_index_arg(l.len(), key, "__delitem__")?; + l.remove(n); + Ok(Object::None) +} + fn list_pop(args: &[Object]) -> Result { let l = list_self(args)?; let mut l = l.borrow_mut(); @@ -4200,7 +5857,12 @@ fn list_count(args: &[Object]) -> Result { } let l = list_self(args)?; let l = l.borrow(); - let n = l.iter().filter(|x| x.eq_value(&args[1])).count(); + // CPython compares with `PyObject_RichCompareBool`, which is identity-first, + // so `[nan].count(nan)` (the same nan) is 1. + let n = l + .iter() + .filter(|x| x.is_same(&args[1]) || x.eq_value(&args[1])) + .count(); Ok(Object::Int(n as i64)) } @@ -4242,6 +5904,13 @@ fn list_copy(args: &[Object]) -> Result { fn dict_self(args: &[Object]) -> Result>, RuntimeError> { match args.first() { Some(Object::Dict(d)) => Ok(d.clone()), + // A subclass of `dict` (`class C(dict)`) carries its entries in the + // wrapped native payload. Unbound calls — `dict.__setitem__(c, k, v)`, + // `super().__setitem__(k, v)` — pass the instance, so unwrap it here. + Some(Object::Instance(inst)) => match &inst.native { + Some(Object::Dict(d)) => Ok(d.clone()), + _ => Err(type_error("expected dict method receiver")), + }, _ => Err(type_error("expected dict method receiver")), } } @@ -4260,6 +5929,45 @@ fn dict_get(args: &[Object]) -> Result { Ok(value) } +// Container dunders exposed on the type so `dict.__setitem__`, +// `super().__getitem__`, … resolve for `dict` subclasses. They mirror the +// VM's subscript opcodes but operate on the (possibly unwrapped) native +// payload. `__init__` reuses `dict_update` (clear-then-fill is unnecessary: +// a freshly constructed subclass starts with an empty native dict). +fn dict_setitem(args: &[Object]) -> Result { + let d = dict_self(args)?; + let key = args + .get(1) + .ok_or_else(|| type_error("__setitem__ expected 2 arguments"))?; + let val = args + .get(2) + .ok_or_else(|| type_error("__setitem__ expected 2 arguments"))?; + d.borrow_mut() + .insert(DictKey(key.clone()), val.clone()); + Ok(Object::None) +} + +fn dict_getitem(args: &[Object]) -> Result { + let d = dict_self(args)?; + let key = args + .get(1) + .ok_or_else(|| type_error("__getitem__ expected 1 argument"))?; + let found = d.borrow().get(&DictKey(key.clone())).cloned(); + found.ok_or_else(|| key_error(key.repr())) +} + +fn dict_delitem(args: &[Object]) -> Result { + let d = dict_self(args)?; + let key = args + .get(1) + .ok_or_else(|| type_error("__delitem__ expected 1 argument"))?; + if d.borrow_mut().shift_remove(&DictKey(key.clone())).is_some() { + Ok(Object::None) + } else { + Err(key_error(key.repr())) + } +} + fn dict_keys(args: &[Object]) -> Result { let d = dict_self(args)?; Ok(Object::DictView(Rc::new(crate::object::PyDictView { @@ -4352,7 +6060,10 @@ fn tuple_count(args: &[Object]) -> Result { Some(Object::Tuple(t)) => t.clone(), _ => return Err(type_error("expected tuple")), }; - let n = t.iter().filter(|x| x.eq_value(&args[1])).count(); + let n = t + .iter() + .filter(|x| x.is_same(&args[1]) || x.eq_value(&args[1])) + .count(); Ok(Object::Int(n as i64)) } diff --git a/crates/weavepy-vm/src/error.rs b/crates/weavepy-vm/src/error.rs index 465b132..c0270ed 100644 --- a/crates/weavepy-vm/src/error.rs +++ b/crates/weavepy-vm/src/error.rs @@ -69,6 +69,29 @@ impl PyException { self.traceback.push(entry); } + /// PEP 678: append a string note to the wrapped instance's + /// `__notes__` list (created on first use). Mirrors + /// `BaseException.add_note`, but callable from Rust-side machinery + /// that needs to annotate an exception before re-raising it — e.g. + /// CPython's `type.__new__` decorates a `__set_name__` failure with + /// "Error calling __set_name__ on '…' instance '…' in '…'". + pub fn add_note(&self, note: impl Into) { + use crate::object::DictKey; + if let Object::Instance(inst) = &self.instance { + let key = DictKey(Object::from_static("__notes__")); + let mut dict = inst.dict.borrow_mut(); + let mut notes = match dict.get(&key) { + Some(Object::List(l)) => l.borrow().clone(), + _ => Vec::new(), + }; + notes.push(Object::from_str(note.into())); + dict.insert( + key, + Object::List(crate::sync::Rc::new(crate::sync::GilCell::new(notes))), + ); + } + } + /// When this exception is a `SystemExit` (or a subclass), return /// its exit `code`: the explicit `.code` attribute, falling back to /// the single `args` element (`()` → `None`, `(x,)` → `x`, @@ -173,6 +196,18 @@ pub fn zero_division_error(message: impl Into) -> RuntimeError { RuntimeError::PyException(PyException::from_builtin("ZeroDivisionError", message)) } +pub fn overflow_error(message: impl Into) -> RuntimeError { + RuntimeError::PyException(PyException::from_builtin("OverflowError", message)) +} + +/// `RecursionError` — raised when the per-thread Python call depth / +/// native-recursion guard (RFC 0037 WS1) is exceeded. CPython raises +/// this from `Py_EnterRecursiveCall`, including on the C-level recursion +/// inside `do_richcompare`/`repr` of reflexive containers. +pub fn recursion_error(message: impl Into) -> RuntimeError { + RuntimeError::PyException(PyException::from_builtin("RecursionError", message)) +} + pub fn stop_iteration() -> RuntimeError { RuntimeError::PyException(PyException::from_builtin("StopIteration", "")) } @@ -220,6 +255,57 @@ pub fn import_error(message: impl Into) -> RuntimeError { RuntimeError::PyException(PyException::from_builtin("ImportError", message)) } +/// A bare `SyntaxError` carrying only a message — no source location. +/// Used for compiler-phase failures (e.g. `'return' outside function`) +/// that don't track a byte offset, so `str(e)` is just the message. +pub fn syntax_error(message: impl Into) -> RuntimeError { + RuntimeError::PyException(PyException::from_builtin("SyntaxError", message)) +} + +/// A `SyntaxError` with CPython's full location payload. Sets `.msg`, +/// `.filename`, `.lineno`, `.offset`, and `.text` on the instance and +/// shapes `args` as `(msg, (filename, lineno, offset, text))`, exactly as +/// CPython's parser does — so `str(e)` renders +/// `" (, line )"` and the attributes are +/// inspectable (`e.lineno`, `e.offset`, …). +pub fn syntax_error_located( + message: impl Into, + filename: Option<&str>, + lineno: Option, + offset: Option, + text: Option<&str>, +) -> RuntimeError { + use crate::object::DictKey; + let message = message.into(); + let pe = PyException::from_builtin("SyntaxError", message.clone()); + if let Object::Instance(inst) = &pe.instance { + let msg_obj = Object::from_str(message); + let file_obj = filename.map_or(Object::None, |s| Object::from_str(s)); + let line_obj = lineno.map_or(Object::None, |n| Object::Int(i64::from(n))); + let off_obj = offset.map_or(Object::None, |n| Object::Int(i64::from(n))); + let text_obj = text.map_or(Object::None, |s| Object::from_str(s)); + let detail = Object::new_tuple(vec![ + file_obj.clone(), + line_obj.clone(), + off_obj.clone(), + text_obj.clone(), + ]); + let mut dict = inst.dict.borrow_mut(); + dict.insert(DictKey(Object::from_static("msg")), msg_obj.clone()); + dict.insert(DictKey(Object::from_static("filename")), file_obj); + dict.insert(DictKey(Object::from_static("lineno")), line_obj); + dict.insert(DictKey(Object::from_static("offset")), off_obj); + dict.insert(DictKey(Object::from_static("text")), text_obj); + dict.insert(DictKey(Object::from_static("end_lineno")), Object::None); + dict.insert(DictKey(Object::from_static("end_offset")), Object::None); + dict.insert( + DictKey(Object::from_static("args")), + Object::new_tuple(vec![msg_obj, detail]), + ); + } + RuntimeError::PyException(pe) +} + pub fn module_not_found_error(message: impl Into) -> RuntimeError { RuntimeError::PyException(PyException::from_builtin("ModuleNotFoundError", message)) } diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 4c5f2ac..9cf872c 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -409,7 +409,17 @@ impl GcState { // `Rc::strong_count - 1` (the candidate set holds one // reference itself, in `TrackedHandle::object`). for handle in &candidate_set { - let outer = strong_count_for(&handle.object).saturating_sub(1) as i64; + // A weak reference must not keep its referent reachable, but + // each live slot holds a strong `Object` clone of the target + // (the registry's drop-driven clear model). Discount those + // clones here so an object reachable *only* through weakrefs + // collapses to `gc_refs == 0` and is collected — which fires + // `notify_clear` and flips `weakref.ref(obj)()` to `None`. + let weak_clones = + crate::weakref_registry::strong_clone_count(handle.id) as i64; + let outer = strong_count_for(&handle.object) + .saturating_sub(1) + .saturating_sub(weak_clones as usize) as i64; handle.gc_refs.store(outer, Ordering::Release); handle.color.store(color::White, Ordering::Release); } diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 35a144a..5b8f381 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -35,6 +35,7 @@ pub mod gil; pub mod import; pub mod object; pub mod pycache; +pub mod recursion; pub mod specialize; pub mod stdlib; pub mod sync; @@ -51,8 +52,8 @@ pub mod weakref_registry; use crate::builtin_types::{builtin_types, instance_is_subclass, make_exception_with_class}; use crate::error::{ attribute_error, import_error, index_error, key_error, module_not_found_error, name_error, - runtime_error, stop_async_iteration, stop_iteration, stop_iteration_with, type_error, - value_error, zero_division_error, TracebackEntry, + overflow_error, recursion_error, runtime_error, stop_async_iteration, stop_iteration, + stop_iteration_with, type_error, value_error, zero_division_error, TracebackEntry, }; pub use crate::error::{PyException, RuntimeError}; pub use crate::import::ModuleCache; @@ -200,6 +201,16 @@ impl Default for Interpreter { fn default() -> Self { let stdout: Stdout = Rc::new(RefCell::new(std::io::stdout())); let mut builtins_dict = builtins::default_builtins(); + // The `builtins` module exposes the core types/exceptions as the + // real `type` objects (CPython's `builtins.int is int`), not the + // bare-function constructors `default_builtins` seeds. Module globals + // shadow these with `as_globals()`, but `exec`/`eval`/`runpy` build + // arbitrary namespaces that fall back to `__builtins__`; without the + // real types there, `class C(object)` in exec'd code (e.g. + // `python -m calendar`) sees a `builtin_function_or_method` base. + for (name, value) in builtin_types().as_globals() { + builtins_dict.insert(DictKey(Object::from_str(&name)), value); + } // Wire `print` directly into the shared builtins dict so that // user-driven `exec` / `eval` (which builds an arbitrary // globals dict) can still find it via the normal fallback in @@ -439,6 +450,7 @@ impl Interpreter { /// Wire `print` (and friends) to this interpreter's stdout. /// `print` is installed as a special builtin — the VM intercepts /// the call so it can dispatch `__str__` on user types. + #[allow(dead_code)] fn install_print_into(&self, dict: &mut DictData) { let f = BuiltinFn { name: "print", @@ -661,10 +673,15 @@ impl Interpreter { ) -> Rc> { let globals = Rc::new(RefCell::new(DictData::new())); let mut g = globals.borrow_mut(); - self.install_print_into(&mut g); - for (n, value) in builtin_types().as_globals() { - g.insert(DictKey(Object::from_str(n)), value); - } + // Builtins (`print`, `int`, `object`, exceptions, …) are deliberately + // *not* copied into module globals: CPython resolves them via the + // module's `__builtins__` fallback, and seeding them here would + // pollute `dir(module)` / `vars(module)` / `from module import *` + // with names that don't belong to the module (test_operator's + // `test_dunder_is_original`). `lookup_global_or_builtin` already + // falls back to the shared `__builtins__` dict, which holds the real + // type objects, so name resolution and `class C(object)` are + // unaffected. g.insert( DictKey(Object::from_static("__name__")), Object::from_str(name), @@ -763,6 +780,22 @@ impl Interpreter { frame: &mut Frame, sent: Option, ) -> Result { + // RFC 0037 (WS1) — recursion guard. Every activation maps onto a + // native Rust stack frame, so we bound Python call depth before + // it can overflow the native stack. The guard is held for the + // whole activation and restores the per-thread depth on every + // exit path (return / yield / exception) via `Drop`. + let _recursion_guard = match crate::recursion::enter() { + crate::recursion::Enter::Ok(g) => g, + crate::recursion::Enter::Overflow => { + return Err(RuntimeError::PyException(crate::error::PyException::new( + crate::builtin_types::make_exception( + "RecursionError", + "maximum recursion depth exceeded", + ), + ))); + } + }; // Captured before `sent` is consumed below; only the tier-2 // entry guard reads it, so it's gated to the `jit` feature to // stay warning-free in default builds. @@ -894,6 +927,12 @@ impl Interpreter { let globals = frame.globals.clone(); let class_ns = frame.class_namespace.clone(); let snapshot_for_provider = locals_snapshot.clone(); + // At module / exec scope CPython makes `locals() is globals()`. + // We detect the module body by its conventional code name so a + // top-level `locals()` / `dir()` reflects the module namespace + // instead of an (empty) function-style snapshot. + let is_module_scope = frame.code.name == ""; + let globals_for_provider = globals.clone(); let provider: Rc Object + Send + Sync> = Rc::new(move || { let snapshot = snapshot_for_provider.borrow(); // For module / class bodies the user-visible locals are @@ -902,6 +941,9 @@ impl Interpreter { if let Some(ns) = class_ns.as_ref() { return Object::Dict(ns.clone()); } + if is_module_scope { + return Object::Dict(globals_for_provider.clone()); + } // Function frames: copy the locals array into a dict so // user code can read by name. We honour cell variables // (their value lives in the cell, not the local slot). @@ -1329,6 +1371,18 @@ impl Interpreter { .ok_or_else(|| RuntimeError::Internal("bad cell index".to_owned()))?; *cell.borrow_mut() = v; } + OpCode::DeleteDeref => { + // `del NAME` for a cell/free var clears the cell WITHOUT + // popping the stack (unlike StoreDeref). The VM marks an + // emptied binding with `Object::None`, matching the + // leniency of `DeleteFast` for locals. + let cell = frame + .cells + .get(ins.arg as usize) + .cloned() + .ok_or_else(|| RuntimeError::Internal("bad cell index".to_owned()))?; + *cell.borrow_mut() = Object::None; + } OpCode::MakeCell => { let slot = ins.arg as usize; if slot >= frame.cells.len() { @@ -1457,21 +1511,48 @@ impl Interpreter { } } OpCode::BinaryOp => { + // The low byte encodes the operator; bit 0x100 (stripped by + // `as u8`) marks an augmented assignment (`a += b`). let kind: BinOpKind = unsafe { std::mem::transmute(ins.arg as u8) }; + let inplace = (ins.arg & weavepy_compiler::BINARY_OP_INPLACE_FLAG) != 0; if !self.specialized_binary_op(frame, cache_pc, kind)? { let b = frame.pop()?; let a = frame.pop()?; - let r = self.dispatch_binary_op(&a, &b, kind, &frame.globals)?; + let r = if inplace { + self.dispatch_inplace_op(&a, &b, kind, &frame.globals)? + } else { + self.dispatch_binary_op(&a, &b, kind, &frame.globals)? + }; frame.push(r); } } OpCode::UnaryOp => { let v = frame.pop()?; let kind: UnaryKind = unsafe { std::mem::transmute(ins.arg as u8) }; - let r = if matches!(kind, UnaryKind::Not) && matches!(v, Object::Instance(_)) { - // `not obj` must honour __bool__/__len__. + let r = if matches!(v, Object::Instance(_)) { let g = frame.globals.clone(); - Object::Bool(!self.obj_truthy(&v, &g)?) + match kind { + // `not obj` must honour __bool__/__len__. + UnaryKind::Not => Object::Bool(!self.obj_truthy(&v, &g)?), + // -obj / +obj / ~obj dispatch the numeric dunders so + // pure-Python numeric types (fractions.Fraction, + // decimal.Decimal, user classes) participate. + UnaryKind::Neg | UnaryKind::Pos | UnaryKind::Invert => { + let dunder = match kind { + UnaryKind::Neg => "__neg__", + UnaryKind::Pos => "__pos__", + _ => "__invert__", + }; + match instance_method(&v, dunder) { + Some(method) => self.call(&method, &[], &[], &g)?, + // No user override: a built-in numeric subclass + // (`class C(complex)`, `class C(int)`, …) applies + // the base type's unary op to its native payload, + // matching CPython's inherited slot. + None => unary_op(&v.native_value().unwrap_or_else(|| v.clone()), kind)?, + } + } + } } else { unary_op(&v, kind)? }; @@ -1504,6 +1585,23 @@ impl Interpreter { &frame.globals.clone(), )?; r.is_truthy() + } else if let Object::Instance(inst) = &container { + if let Some(native) = inst.native.clone() { + // Subclass of a built-in container (`class C(dict)`, + // `class C(list)`, …) without a Python `__contains__`: + // CPython inherits the native C membership test. Use the + // wrapped payload directly — crucially this avoids the + // legacy `__getitem__` iteration path, which would loop + // forever for a mapping subclass whose `__getitem__` + // never raises `IndexError`. + native.contains(&item)? + } else { + // Pure-Python class: CPython falls back to iteration + // (`_PySequence_IterSearch`), dispatching `__iter__` / + // `__getitem__` and comparing each element with `==`. + // Exceptions raised while iterating propagate. + self.contains_via_iter(&container, &item, &frame.globals.clone())? + } } else { container.contains(&item)? }; @@ -1827,9 +1925,18 @@ impl Interpreter { ))) } }; - if items.len() != n { + // CPython distinguishes the two arity errors: it stops + // pulling once it has one too many (so the "too many" + // message omits the actual count), but reports the + // shortfall exactly when there are too few. + if items.len() > n { + return Err(value_error(format!( + "too many values to unpack (expected {n})" + ))); + } + if items.len() < n { return Err(value_error(format!( - "expected {} values to unpack, got {}", + "not enough values to unpack (expected {}, got {})", n, items.len() ))); @@ -2002,9 +2109,20 @@ impl Interpreter { .borrow_mut() .insert(DictKey(Object::from_static("__module__")), name_obj); } + // Pin __name__ and __qualname__ as stable objects so + // repeated `func.__name__` reads (and delegated reads + // through classmethod/staticmethod wrappers) return the + // *same* object — CPython exposes these as slots with + // stable identity, which `assertIs(wrapper.__name__, + // func.__name__)` in test_decorators relies on. + let name_obj = Object::from_str(name.clone()); + attrs.borrow_mut().insert( + DictKey(Object::from_static("__name__")), + name_obj.clone(), + ); attrs.borrow_mut().insert( DictKey(Object::from_static("__qualname__")), - Object::from_str(name.clone()), + name_obj, ); if let Some(ann) = annotations_obj { attrs @@ -2706,6 +2824,21 @@ impl Interpreter { .ok_or_else(|| RuntimeError::Internal("bad name index".to_owned())) } + /// Confirm that the `DictKey` sitting at a cached slot index still + /// equals the name `name_idx` refers to. Slot-index inline caches + /// (LOAD_GLOBAL / LOAD_ATTR) guard only on the owning dict's `Rc` + /// identity, which does **not** change when `del` shift-removes an + /// earlier entry and renumbers every later slot. Without this check + /// a stale index would silently alias a different binding's value. + #[inline] + fn cached_slot_name_matches(&self, code: &CodeObject, name_idx: u32, key: &DictKey) -> bool { + matches!( + &key.0, + Object::Str(s) + if code.names.get(name_idx as usize).is_some_and(|n| n.as_str() == &**s) + ) + } + fn lookup_global_or_builtin( &self, globals: &Rc>, @@ -2722,6 +2855,16 @@ impl Interpreter { } fn load_attr(&mut self, obj: &Object, name: &str) -> Result { + // `__class__` is readable on *every* object and returns its + // type. Instances and classes keep their dedicated handling + // below (which honours `__class__` reassignment and the + // metaclass-owner rule); everything else — ints, str, tuples, + // functions, … — resolves uniformly here. Pure-Python code such + // as `_py_abc.ABCMeta.__instancecheck__` relies on + // `instance.__class__` working for primitive instances. + if name == "__class__" && !matches!(obj, Object::Instance(_) | Object::Type(_)) { + return Ok(Object::Type(crate::builtins::class_of(obj))); + } match obj { Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { let allowed: &[&str] = match obj { @@ -2752,6 +2895,25 @@ impl Interpreter { "fset" => Ok(p.fset.clone()), "fdel" => Ok(p.fdel.clone()), "__doc__" => Ok(p.doc.clone()), + // CPython computes `property.__isabstractmethod__` as the + // OR of the wrapped accessors' flags, so the modern + // `@property` / `@abstractmethod` stacking marks the + // whole property abstract. + "__isabstractmethod__" => { + for accessor in [&p.fget, &p.fset, &p.fdel] { + if matches!(accessor, Object::None) { + continue; + } + if self + .load_attr(accessor, "__isabstractmethod__") + .unwrap_or(Object::Bool(false)) + .is_truthy() + { + return Ok(Object::Bool(true)); + } + } + Ok(Object::Bool(false)) + } _ => { if let Some(method) = self.lookup_method(obj, name) { return Ok(Object::BoundMethod(Rc::new(BoundMethod { @@ -2766,7 +2928,8 @@ impl Interpreter { } }, Object::StaticMethod(inner) => match name { - "__func__" => Ok((**inner).clone()), + // `__func__`/`__wrapped__` expose the wrapped callable. + "__func__" | "__wrapped__" => Ok((**inner).clone()), "__isabstractmethod__" => { // Honour an `@abstractmethod` decorator applied // *under* `@staticmethod` (`@staticmethod @@ -2775,16 +2938,24 @@ impl Interpreter { .load_attr(inner.as_ref(), "__isabstractmethod__") .unwrap_or(Object::Bool(false))) } + // Metadata transparently mirrors the wrapped function so + // `getattr(staticmethod(f), attr) is getattr(f, attr)`. + "__module__" | "__qualname__" | "__name__" | "__doc__" + | "__annotations__" | "__dict__" => self.load_attr(inner.as_ref(), name), _ => Err(attribute_error(format!( "'staticmethod' object has no attribute '{}'", name ))), }, Object::ClassMethod(inner) => match name { - "__func__" => Ok((**inner).clone()), + // `__func__` and `__wrapped__` both expose the underlying + // callable; `functools.wraps`/inspect walk `__wrapped__`. + "__func__" | "__wrapped__" => Ok((**inner).clone()), "__isabstractmethod__" => Ok(self .load_attr(inner.as_ref(), "__isabstractmethod__") .unwrap_or(Object::Bool(false))), + "__module__" | "__qualname__" | "__name__" | "__doc__" + | "__annotations__" | "__dict__" => self.load_attr(inner.as_ref(), name), _ => Err(attribute_error(format!( "'classmethod' object has no attribute '{}'", name @@ -2802,6 +2973,21 @@ impl Interpreter { "__dict__" => return Ok(Object::Dict(m.dict.clone())), _ => {} } + // PEP 562: a module-level `__getattr__(name)` is consulted + // for any attribute missing from the module namespace. + // Used by e.g. `calendar.January` (deprecation shim) and + // many lazy-import stdlib modules. + if !matches!(name, "__getattr__" | "__path__" | "__loader__" | "__spec__") { + let getattr = m + .dict + .borrow() + .get(&DictKey(Object::from_str("__getattr__"))) + .cloned(); + if let Some(getattr) = getattr { + let globals = m.dict.clone(); + return self.call(&getattr, &[Object::from_str(name)], &[], &globals); + } + } Err(attribute_error(format!( "module '{}' has no attribute '{}'", m.name, name @@ -2967,7 +3153,9 @@ impl Interpreter { ))), }, Object::Builtin(b) => match name { - "__name__" | "__qualname__" => Ok(Object::from_static(b.name)), + "__name__" | "__qualname__" => { + Ok(Object::from_static(builtin_display_name(b.name))) + } "__module__" => Ok(Object::from_static("builtins")), "__doc__" => Ok(Object::None), "__self__" => Ok(Object::None), @@ -2981,7 +3169,7 @@ impl Interpreter { "__self__" => Ok(bm.receiver.clone()), "__name__" => match &bm.function { Object::Function(f) => Ok(Object::from_str(f.name.clone())), - Object::Builtin(b) => Ok(Object::from_static(b.name)), + Object::Builtin(b) => Ok(Object::from_static(builtin_display_name(b.name))), _ => Ok(Object::from_static("?")), }, "__doc__" => Ok(Object::None), @@ -3010,6 +3198,37 @@ impl Interpreter { } }, _ => { + // File-object data attributes. `.buffer` is the binary + // underlayer of a text stream — CPython's + // `sys.stdout.buffer.write(bytes)`. Our `PyFile` already + // accepts bytes through `write`, so the same object can + // serve as its own buffer. + if let Object::File(f) = obj { + match name { + "buffer" | "raw" => { + return Ok(obj.clone()); + } + "name" => return Ok(Object::from_str(&f.name)), + "mode" => return Ok(Object::from_str(&f.mode)), + "closed" => return Ok(Object::Bool(f.is_closed())), + "encoding" => { + return Ok(if f.binary { + Object::None + } else { + Object::from_static("utf-8") + }) + } + "errors" => { + return Ok(if f.binary { + Object::None + } else { + Object::from_static("strict") + }) + } + "newlines" => return Ok(Object::None), + _ => {} + } + } // Numeric data attributes — exposed by the // ``numbers`` protocol (``real``, ``imag``, // ``numerator``, ``denominator``). Returned as @@ -3017,6 +3236,18 @@ impl Interpreter { if let Some(v) = numeric_data_attr(obj, name) { return Ok(v); } + // `slice.start` / `.stop` / `.step` read-only data + // attributes (CPython's `slice` members). The values are + // stored verbatim (including `None`) so `slice(2).stop` + // is `2` while `.start`/`.step` are `None`. + if let Object::Slice(s) = obj { + match name { + "start" => return Ok(s.start.clone()), + "stop" => return Ok(s.stop.clone()), + "step" => return Ok(s.step.clone()), + _ => {} + } + } if let Some(method) = self.lookup_method(obj, name) { return Ok(Object::BoundMethod(Rc::new(BoundMethod { receiver: obj.clone(), @@ -3045,11 +3276,97 @@ impl Interpreter { /// functions become bound methods. /// 5. Otherwise, dispatch the class's `__getattr__` if any. /// 6. Otherwise, raise `AttributeError`. + /// Attribute access on an instance, honouring a user-defined + /// `__getattribute__` override (CPython's `tp_getattro`). Explicit + /// access — `x.attr`, `getattr(x, name)` — routes here and through any + /// override; *implicit* special-method lookups (`len(x)`, `x[i]`, …) use + /// type-level `lookup`/`lookup_method` instead and correctly bypass it. + /// + /// On an `AttributeError` from either the override or the default lookup, + /// `__getattr__` is consulted (CPython's slot-wrapper hook order). fn load_attr_instance( &mut self, inst: &Rc, instance_obj: &Object, name: &str, + ) -> Result { + let result = if let Some(getattribute) = self.user_getattribute(&inst.class) { + let bound = Object::BoundMethod(Rc::new(BoundMethod { + receiver: instance_obj.clone(), + function: getattribute, + })); + self.call( + &bound, + &[Object::from_str(name)], + &[], + &self.builtins.clone(), + ) + } else { + self.load_attr_instance_default(inst, instance_obj, name) + }; + match result { + Err(e) if self.is_attribute_error(&e) => { + if let Some(getattr) = inst.class.lookup("__getattr__") { + let bound = Object::BoundMethod(Rc::new(BoundMethod { + receiver: instance_obj.clone(), + function: getattr, + })); + self.call( + &bound, + &[Object::from_str(name)], + &[], + &self.builtins.clone(), + ) + } else { + Err(e) + } + } + other => other, + } + } + + /// A user-defined `__getattribute__` override, if any. Returns `None` + /// when the class uses the default `object.__getattribute__` (resolved as + /// the `.object_getattribute` sentinel installed on `object`). + fn user_getattribute(&self, class: &Rc) -> Option { + // Fast path: a type cached as "default `__getattribute__`" (the common + // case) skips the MRO walk entirely. Override / not-yet-computed types + // fall through to the lookup and refresh the cache. + if class.getattribute_kind.get() == 1 { + return None; + } + let result = match class.lookup("__getattribute__") { + Some(Object::Builtin(b)) if b.name == ".object_getattribute" => None, + other => other, + }; + class + .getattribute_kind + .set(if result.is_some() { 2 } else { 1 }); + result + } + + /// `object.__getattribute__(recv, name)` — the default lookup, used by the + /// sentinel dispatch so a user override can delegate up without recursing. + fn object_default_getattribute( + &mut self, + recv: &Object, + name: &str, + ) -> Result { + match recv { + Object::Instance(inst) => self.load_attr_instance_default(inst, recv, name), + _ => self.load_attr(recv, name), + } + } + + /// The default `object.__getattribute__` body: data descriptor → instance + /// dict → class attr → built-in-subclass payload, ending in + /// `AttributeError`. Does **not** consult `__getattr__` (the caller does) + /// nor any `__getattribute__` override (that would recurse). + fn load_attr_instance_default( + &mut self, + inst: &Rc, + instance_obj: &Object, + name: &str, ) -> Result { // Super proxies stash the real receiver under `__self__`. // Re-bind methods looked up via the proxy so they run @@ -3073,6 +3390,20 @@ impl Interpreter { }; return self.descriptor_get(&v, &receiver, &owner); } + // The MRO beyond the starting class reaches a built-in base + // (`dict`, `list`, …) whose methods aren't stored on the type + // dict and so don't surface above. Resolve `name` against the + // receiver's native payload so `super().__setitem__`, + // `super().append`, … dispatch to the wrapped built-in and + // operate on the shared payload. (Only names absent from the + // MRO reach here, so this never shadows a user override.) + if let Object::Instance(recv) = &receiver { + if let Some(native) = &recv.native { + if let Ok(v) = self.load_attr(&native.clone(), name) { + return Ok(v); + } + } + } return Err(attribute_error(format!( "'super' object has no attribute '{}'", name @@ -3110,20 +3441,63 @@ impl Interpreter { _ => {} } - // (4) __getattr__ fall-back. - if let Some(getattr) = inst.class.lookup("__getattr__") { - let bound = Object::BoundMethod(Rc::new(BoundMethod { - receiver: instance_obj.clone(), - function: getattr, - })); - return self.call( - &bound, - &[Object::from_str(name)], - &[], - &self.builtins.clone(), - ); + // (3c) Subclasses of a built-in (`class C(list)`, `class C(int)`, + // …) inherit that built-in's methods. WeavePy dispatches those + // methods by matching the `Object` variant rather than storing + // them on the type dict, so they don't surface via the MRO walk + // above — resolve them against the wrapped native payload, which + // binds them to that payload (so `c.append(x)` mutates the + // shared list, `c.bit_length()` reads the wrapped int, etc.). + // Dunders are excluded: those route through dedicated protocol + // paths and must not be hijacked here. + if !(name.starts_with("__") && name.ends_with("__")) { + if let Some(native) = &inst.native { + if let Ok(v) = self.load_attr(&native.clone(), name) { + return Ok(v); + } + } + } + + // (3d) `__getnewargs__` for subclasses of immutable built-ins + // (`class C(tuple)`, `class C(int)`, …). CPython defines it on the + // base type so `copy`/`pickle` reconstruct `cls.__new__(cls, value)` + // — without it the rebuilt instance is empty. Excluded from (3c) + // because it's a dunder; resolved here against the native payload. + if name == "__getnewargs__" { + if let Some(native) = &inst.native { + if let Some(m) = crate::builtins::immutable_subclass_getnewargs(native) { + return Ok(Object::BoundMethod(Rc::new(BoundMethod { + receiver: instance_obj.clone(), + function: m, + }))); + } + } + } + + // (3e) Numeric/value dunders a built-in base synthesizes rather than + // storing on its type dict (`complex.__complex__`, `int.__add__`, + // `float.__eq__`, …). They don't surface in the MRO walk, so for a + // built-in subclass instance *without* a user override resolve them + // against the native payload and bind them *there* — matching + // CPython, e.g. `ComplexSubclass(3,4).__complex__()` returns a plain + // `complex` and `sub.__add__(x)` operates on the wrapped value. Only + // names `lookup_method` actually provides for the payload bind here, + // so ordinary dunders (`__repr__`, `__init__`, …) still fall through. + if name.starts_with("__") && name.ends_with("__") { + if let Some(native) = &inst.native { + let native = native.clone(); + if let Some(m) = crate::builtins::lookup_method(&native, name) { + return Ok(Object::BoundMethod(Rc::new(BoundMethod { + receiver: native, + function: m, + }))); + } + } } + // `__getattr__` fall-back is applied by `load_attr_instance` (the + // override-aware wrapper) so it fires after *both* this default path + // and any `__getattribute__` override raise `AttributeError`. Err(attribute_error(format!( "'{}' object has no attribute '{}'", inst.class.name, name @@ -3181,6 +3555,40 @@ impl Interpreter { } "__class__" => return Ok(Object::Type(meta)), "__dict__" => return Ok(Object::Dict(ty.dict.clone())), + "__subclasses__" => { + // `type.__subclasses__` is a bound method; the actual + // work is done in `Interpreter::call` via the sentinel + // builtin name (it needs the live `Rc`, + // which isn't `Send + Sync` and so can't be captured by + // a plain `BuiltinFn` closure). + let builtin = Object::Builtin(Rc::new(BuiltinFn { + name: ".type_subclasses", + call: Box::new(|_args| { + Err(RuntimeError::Internal( + "type.__subclasses__ must be dispatched via Interpreter::call" + .to_owned(), + )) + }), + call_kw: None, + })); + return Ok(Object::BoundMethod(Rc::new(BoundMethod { + receiver: Object::Type(ty.clone()), + function: builtin, + }))); + } + "__module__" => { + // User classes record `__module__` in their dict; built-in + // types (object, int, …) live in `builtins`. CPython exposes + // `.__module__` for both — test_operator/test_descr read it. + if let Some(v) = ty + .dict + .borrow() + .get(&DictKey(Object::from_static("__module__"))) + { + return Ok(v.clone()); + } + return Ok(Object::from_static("builtins")); + } _ => {} } @@ -3194,6 +3602,15 @@ impl Interpreter { return Ok(b); } + // (6) Unbound instance methods reached via the type: `str.upper`, + // `float.hex`, `list.append`, … CPython exposes every instance method + // on its type as a function taking `self` explicitly. The builtins + // already treat `args[0]` as the receiver, so we hand back the raw + // function object (unbound) rather than binding it to the class. + if let Some(b) = crate::builtins::unbound_method(&ty.name, name) { + return Ok(b); + } + Err(attribute_error(format!( "type object '{}' has no attribute '{}'", ty.name, name @@ -3267,6 +3684,27 @@ impl Interpreter { } } Object::Instance(inner_inst) => { + // Subclasses of `property` / `classmethod` / `staticmethod` + // carry the wrapped descriptor in `native`; route the + // descriptor protocol through it so an instance of e.g. + // abc's `abstractproperty` still computes its getter on + // access. On *class* access (`instance` is `None`) a + // property returns the wrapper itself, matching + // `property.__get__(None, owner)` returning `self`. + if let Some(native) = &inner_inst.native { + match native { + Object::Property(_) => { + if matches!(instance, Object::None) { + return Ok(attr.clone()); + } + return self.descriptor_get(native, instance, owner); + } + Object::ClassMethod(_) | Object::StaticMethod(_) => { + return self.descriptor_get(native, instance, owner); + } + _ => {} + } + } // User-defined descriptor: invoke its `__get__` if // present, otherwise pass the descriptor through. if let Some(get_method) = inner_inst.class.lookup("__get__") { @@ -3461,9 +3899,8 @@ impl Interpreter { Ok(Object::from_str(self.stringify(v, globals)?)) } - /// VM-aware variant of [`str_format_impl`] that dispatches - /// `__str__` / `__repr__` for conversions on instances, so - /// `"{!r}".format(obj)` mirrors `repr(obj)` for user types. + /// `str.format(*args, **kwargs)` entry point. See + /// [`Self::format_template_str`] for the formatting engine. fn do_str_format( &mut self, template: &str, @@ -3471,144 +3908,57 @@ impl Interpreter { keyword: &[(String, Object)], globals: &Rc>, ) -> Result { - // Pre-stringify any Instance arg by converting through the - // VM so user dunders run. We don't know yet which conversion - // each field will pick, so we materialise both `!s` and `!r` - // upfront when the arg is an Instance. - let mut positional_resolved: Vec = Vec::with_capacity(positional.len()); - let mut keyword_resolved: Vec<(String, Object)> = Vec::with_capacity(keyword.len()); - // We let `str_format_impl` do the normal field resolution - // and conversion; the only thing we need to fix is that when - // the value is an Instance and there is an `!s` / `!r` - // conversion, the plain `to_str()` / `repr()` in - // `render_format_field` won't dispatch dunders. We do a - // pre-pass and replace bare Instances with proxy strings - // when no conversion is requested. - // - // Conversion dispatch: we recognise `{x!s}` / `{x!r}` by - // post-processing — we leave Instances alone for the - // straight `{x}` path (the default conversion falls back to - // `value.to_str()` which CPython's `format` actually also - // does — it calls `__format__`, but lacking that here we - // emit a `` placeholder). - for arg in positional { - positional_resolved.push(arg.clone()); - } - for (k, v) in keyword { - keyword_resolved.push((k.clone(), v.clone())); - } - // The straightforward fix is to override conversion: parse - // each field's `!s` / `!r` and substitute the user-method - // result back into the field as a Str literal before - // delegating to `str_format_impl`. We do that with a - // pre-pass below. - let preprocessed = - self.preprocess_str_format(template, &positional_resolved, &keyword_resolved, globals)?; - str_format_impl(&preprocessed, &positional_resolved, &keyword_resolved) - } - - /// Walk every `{...}` field; when the conversion is `!s` or `!r` - /// and the referenced value is an Instance, replace the field - /// with a pre-rendered literal so the downstream formatter sees - /// a string instead of the unconverted object. - fn preprocess_str_format( + let mut state = FmtState::default(); + self.format_template_str(template, positional, keyword, None, &mut state, globals) + } + + fn do_str_format_map( + &mut self, + template: &str, + mapping: &Rc>, + globals: &Rc>, + ) -> Result { + let mut state = FmtState::default(); + self.format_template_str(template, &[], &[], Some(mapping), &mut state, globals) + } + + /// Single-pass, interpreter-aware implementation of CPython's + /// `str.format` / `str.format_map` and the recursive expansion of a + /// nested `{...}` inside a format spec. The auto/manual field + /// numbering carried in `state` is shared across the whole call + /// (including nested specs), so `'{:^{}}'.format(s, w)` consumes the + /// width argument *after* `s` — mirroring + /// `Objects/stringlib/unicode_format.h`. Field values are formatted + /// through `__format__` (and converted through `__str__`/`__repr__`) + /// so user dunders run. + fn format_template_str( &mut self, template: &str, positional: &[Object], keyword: &[(String, Object)], + mapping: Option<&Rc>>, + state: &mut FmtState, globals: &Rc>, ) -> Result { let bytes = template.as_bytes(); let mut out = String::with_capacity(template.len()); let mut i = 0; - let mut auto_idx = 0usize; while i < bytes.len() { let b = bytes[i]; if b == b'{' { if i + 1 < bytes.len() && bytes[i + 1] == b'{' { - out.push_str("{{"); + out.push('{'); i += 2; continue; } let (field, end) = scan_format_field(bytes, i + 1)?; i = end; - let (name_part, conv, spec_part) = split_format_field(&field); - let conv_char = conv; - let mut tmp_idx = auto_idx; - let resolved = - resolve_field_name(name_part, positional, keyword, &mut tmp_idx, None); - let consumed_auto = name_part.is_empty(); - if matches!(conv_char, Some('s') | Some('r')) { - if let Ok(value) = resolved.as_ref() { - if matches!(value, Object::Instance(_)) { - let rendered = match conv_char { - Some('s') => self.stringify(value, globals)?, - Some('r') => self.repr_of(value, globals)?, - _ => unreachable!(), - }; - let final_text = match spec_part { - Some(spec) => format_via_spec(&Object::from_str(rendered), spec)?, - None => rendered, - }; - for ch in final_text.chars() { - if ch == '{' || ch == '}' { - out.push(ch); - out.push(ch); - } else { - out.push(ch); - } - } - auto_idx = tmp_idx; - continue; - } - } - } else if conv_char.is_none() { - // `{x}` with no explicit conversion: CPython calls - // `__format__(x, spec)`, which on instances falls - // back to `__str__`. We don't yet hop through - // `__format__`, but invoking `__str__` is the - // common-case match users expect. - if let Ok(value) = resolved.as_ref() { - if matches!(value, Object::Instance(_)) { - let s = self.stringify(value, globals)?; - let final_text = match spec_part { - Some(spec) => format_via_spec(&Object::from_str(s), spec)?, - None => s, - }; - for ch in final_text.chars() { - if ch == '{' || ch == '}' { - out.push(ch); - out.push(ch); - } else { - out.push(ch); - } - } - auto_idx = tmp_idx; - continue; - } - } - } - // Field unchanged. If we consumed an auto-index slot - // and the field doesn't carry a name, rewrite it as a - // positional `{N}` so the downstream formatter's - // separate auto-index counter doesn't desync with ours. - if consumed_auto { - let idx = auto_idx; - auto_idx = tmp_idx; - out.push('{'); - out.push_str(&idx.to_string()); - // Preserve trailers (everything after the empty name_part). - let after_base = &field[name_part.len()..]; - out.push_str(after_base); - out.push('}'); - } else { - out.push('{'); - out.push_str(&field); - out.push('}'); - } + let rendered = + self.render_field_str(&field, positional, keyword, mapping, state, globals)?; + out.push_str(&rendered); } else if b == b'}' { if i + 1 < bytes.len() && bytes[i + 1] == b'}' { - out.push_str("}}"); + out.push('}'); i += 2; continue; } @@ -3623,6 +3973,174 @@ impl Interpreter { Ok(out) } + /// Render one `{name!conv:spec}` replacement field. + fn render_field_str( + &mut self, + field: &str, + positional: &[Object], + keyword: &[(String, Object)], + mapping: Option<&Rc>>, + state: &mut FmtState, + globals: &Rc>, + ) -> Result { + let (name_part, conv, spec_part) = split_format_field(field); + let value = + self.resolve_field_str(name_part, positional, keyword, mapping, state, globals)?; + // Expand a nested spec first (it may consume further auto args), + // threading the shared numbering state. + let spec_owned: Option = match spec_part { + Some(s) if s.as_bytes().contains(&b'{') => { + Some(self.format_template_str(s, positional, keyword, mapping, state, globals)?) + } + Some(s) => Some(s.to_owned()), + None => None, + }; + let spec = spec_owned.as_deref().unwrap_or(""); + match conv { + None => self.format_obj_str(&value, spec, globals), + Some(c) => { + // A conversion always yields a `str`; the spec then + // applies to that string. + let converted = match c { + 's' => self.stringify(&value, globals)?, + 'r' => self.repr_of(&value, globals)?, + 'a' => self.ascii_of(&value, globals)?, + other => { + return Err(value_error(format!( + "Unknown conversion specifier {other}" + ))) + } + }; + self.format_obj_str(&Object::from_str(converted), spec, globals) + } + } + } + + /// Resolve a field name (`""`, `"0"`, `"name"`, with optional + /// `.attr` / `[key]` trailers), enforcing CPython's rule that a + /// single format string cannot mix automatic (`{}`) and manual + /// (`{0}`) field numbering. + fn resolve_field_str( + &mut self, + name: &str, + positional: &[Object], + keyword: &[(String, Object)], + mapping: Option<&Rc>>, + state: &mut FmtState, + globals: &Rc>, + ) -> Result { + let (base, trailers) = split_name_trailers(name); + let mut value = if base.is_empty() { + if state.manual_used { + return Err(value_error( + "cannot switch from manual field specification to automatic field numbering", + )); + } + state.auto_used = true; + let idx = state.auto_next; + state.auto_next += 1; + positional.get(idx).cloned().ok_or_else(|| { + index_error("Replacement index out of range for positional args tuple".to_string()) + })? + } else if let Ok(idx) = base.parse::() { + if state.auto_used { + return Err(value_error( + "cannot switch from automatic field numbering to manual field specification", + )); + } + state.manual_used = true; + positional.get(idx).cloned().ok_or_else(|| { + index_error(format!( + "Replacement index {idx} out of range for positional args tuple" + )) + })? + } else if let Some(map) = mapping { + let key = DictKey(Object::from_str(base)); + map.borrow() + .get(&key) + .cloned() + .ok_or_else(|| key_error(format!("'{base}'")))? + } else { + keyword + .iter() + .find_map(|(k, v)| (k == base).then(|| v.clone())) + .ok_or_else(|| key_error(format!("'{base}'")))? + }; + for trailer in trailers { + value = self.apply_trailer_interp(value, trailer, globals)?; + } + Ok(value) + } + + /// Apply a single `.attr` / `[key]` trailer through the interpreter + /// so `__getattr__` / `__getitem__` run. + fn apply_trailer_interp( + &mut self, + value: Object, + trailer: &str, + _globals: &Rc>, + ) -> Result { + if let Some(attr) = trailer.strip_prefix('.') { + self.load_attr(&value, attr) + } else if trailer.starts_with('[') && trailer.ends_with(']') { + let inner = &trailer[1..trailer.len() - 1]; + let key = if let Ok(i) = inner.parse::() { + Object::Int(i) + } else { + Object::from_str(inner) + }; + self.binary_subscr(&value, &key) + } else { + Err(value_error("invalid field name")) + } + } + + /// Format `value` with `spec`, dispatching to a user `__format__` + /// when present (CPython's `PyObject_Format`). Built-in objects use + /// the format mini-language. + fn format_obj_str( + &mut self, + value: &Object, + spec: &str, + globals: &Rc>, + ) -> Result { + if let Object::Instance(inst) = value { + if let Some(method) = instance_method(value, "__format__") { + let r = self.call(&method, &[Object::from_str(spec)], &[], globals)?; + return match r { + Object::Str(s) => Ok(s.to_string()), + other => Ok(other.to_str()), + }; + } + // No user `__format__`: a built-in subclass (and `IntEnum`/`StrEnum` + // members, whose payload is an `int`/`str`) inherits the base type's + // `__format__`, so honour the spec against the native value. + if let Some(native) = &inst.native { + return format_via_spec(native, spec); + } + // Otherwise `object.__format__` returns str(self) for an empty spec + // and rejects any non-empty spec with a TypeError (it does *not* + // silently string-format the repr). + let s = self.stringify(value, globals)?; + return if spec.is_empty() { + Ok(s) + } else { + Err(unsupported_format_string(value)) + }; + } + format_via_spec(value, spec) + } + + /// `ascii(value)` — interpreter `repr`, then escape non-ASCII. + fn ascii_of( + &mut self, + value: &Object, + globals: &Rc>, + ) -> Result { + let r = self.repr_of(value, globals)?; + Ok(ascii_escape(&r)) + } + fn do_repr_call( &mut self, v: &Object, @@ -3649,36 +4167,320 @@ impl Interpreter { Ok(Object::Int(v.len()? as i64)) } - /// VM-aware Python truthiness. For instances this dispatches - /// `__bool__` (then `__len__`) so user classes that define either - /// dunder are honoured in boolean contexts; everything else falls - /// back to the pure [`Object::is_truthy`]. Mirrors CPython's - /// `PyObject_IsTrue`. - fn obj_truthy( + /// `abs(x)` — dispatch `__abs__` for class instances (CPython calls + /// `type(x).__abs__(x)`), falling back to the primitive numeric + /// implementation for ints/floats/complex. Without this, `abs()` on a + /// pure-Python numeric type (e.g. `fractions.Fraction`) raised a + /// spurious "bad operand type" error. + fn do_abs_call( &mut self, v: &Object, globals: &Rc>, - ) -> Result { - if let Object::Instance(_) = v { - if let Some(method) = instance_method(v, "__bool__") { - let r = self.call(&method, &[], &[], globals)?; - return match r { - Object::Bool(b) => Ok(b), - other => match other.as_i64() { - Some(i) => Ok(i != 0), - None => Err(type_error(format!( - "__bool__ should return bool, returned {}", - other.type_name() - ))), - }, - }; - } - if let Some(method) = instance_method(v, "__len__") { - let r = self.call(&method, &[], &[], globals)?; - return Ok(r.is_truthy()); + ) -> Result { + if let Some(method) = instance_method(v, "__abs__") { + return self.call(&method, &[], &[], globals); + } + // A built-in numeric subclass with no `__abs__` override (e.g. + // `class CS(complex)`) unwraps to its native payload so `abs()` + // applies the base type's magnitude rather than tripping + // `b_abs`'s "bad operand type" guard. + let unwrapped = v.native_value().unwrap_or_else(|| v.clone()); + crate::builtins::b_abs(std::slice::from_ref(&unwrapped)) + } + + /// `round(x[, n])` — dispatch `__round__` for class instances (CPython + /// calls `type(x).__round__(x[, n])`). `round(x)` invokes it with no + /// argument; `round(x, n)` forwards `n`. Falls back to the primitive + /// numeric rounding for ints/floats. + fn do_round_call( + &mut self, + args: &[Object], + globals: &Rc>, + ) -> Result { + if let Some(value) = args.first() { + if let Some(method) = instance_method(value, "__round__") { + let extra: &[Object] = if args.len() >= 2 { &args[1..2] } else { &[] }; + return self.call(&method, extra, &[], globals); } } - Ok(v.is_truthy()) + // Unwrap a built-in numeric subclass with no `__round__` override + // to its native payload (`class MyFloat(float)` → the float) so + // `round()` rounds the underlying value. + let unwrapped: Vec = args + .iter() + .map(|a| a.native_value().unwrap_or_else(|| a.clone())) + .collect(); + crate::builtins::b_round(&unwrapped) + } + + /// `divmod(a, b)` — dispatch `__divmod__`/`__rdivmod__` for instances, + /// matching CPython. Falls back to the primitive implementation for + /// built-in numeric types. + fn do_divmod_call( + &mut self, + args: &[Object], + globals: &Rc>, + ) -> Result { + let (a, b) = (&args[0], &args[1]); + // Mirror CPython's `binary_op1`: try `a.__divmod__(b)` then the + // reflected `b.__rdivmod__(a)`, treating a `NotImplemented` result as + // a decline (keep looking) rather than a value to return. + let not_impl = crate::vm_singletons::not_implemented(); + if let Some(method) = instance_method(a, "__divmod__") { + let r = self.call(&method, std::slice::from_ref(b), &[], globals)?; + if !r.is_same(¬_impl) { + return Ok(r); + } + } + if let Some(method) = instance_method(b, "__rdivmod__") { + let r = self.call(&method, std::slice::from_ref(a), &[], globals)?; + if !r.is_same(¬_impl) { + return Ok(r); + } + } + // The dunder protocol is exhausted. For a user instance with no + // native numeric payload, raise the canonical `divmod()` TypeError + // (falling through to the primitive path would misreport it as `//`). + let a_native = a.native_value(); + let b_native = b.native_value(); + if (matches!(a, Object::Instance(_)) || matches!(b, Object::Instance(_))) + && a_native.is_none() + && b_native.is_none() + { + return Err(type_error(format!( + "unsupported operand type(s) for divmod(): '{}' and '{}'", + a.type_name_owned(), + b.type_name_owned() + ))); + } + // Built-in numeric subclasses with no `__divmod__` override unwrap + // to their native payloads so `divmod()` operates on the values. + let unwrapped: Vec = args + .iter() + .map(|a| a.native_value().unwrap_or_else(|| a.clone())) + .collect(); + crate::builtins::b_divmod(&unwrapped) + } + + /// `complex(x)` — dispatch `__complex__` for instances, then fall back + /// to `__float__` (CPython's `complex()` accepts any object exposing + /// either). Without this, `complex(fraction)` (reached when a + /// `Fraction` is combined with a `complex`) raised a spurious + /// "argument must be a number" error. + fn do_complex_call( + &mut self, + args: &[Object], + globals: &Rc>, + ) -> Result { + // Coerce each argument to a value `b_complex` understands. The real + // (first) argument honours `__complex__` then `__float__`; the + // imaginary (second) argument must be a *real* number, so it only + // honours `__float__` (CPython rejects an imag arg whose only numeric + // hook is `__complex__`, e.g. `complex(0, WithComplex(..))`). + let mut coerced = Vec::with_capacity(args.len()); + for (idx, a) in args.iter().enumerate() { + coerced.push(self.coerce_complex_arg(a, idx == 0, globals)?); + } + crate::builtins::b_complex(&coerced) + } + + /// Reduce one `complex()` argument to a `complex`/`float`/`int`/`str` + /// that `b_complex` accepts: dispatch `__complex__` (real arg only) then + /// `__float__` on a user instance, unwrap a built-in numeric subclass to + /// its payload. + fn coerce_complex_arg( + &mut self, + a: &Object, + allow_complex: bool, + globals: &Rc>, + ) -> Result { + if let Object::Instance(_) = a { + // Real arg only: `__complex__` is consulted first and must return + // a `complex` (a strict subclass is accepted with a + // DeprecationWarning); CPython never calls it for the imag arg. + if allow_complex { + if let Some(method) = instance_method(a, "__complex__") { + let r = self.call(&method, &[], &[], globals)?; + return self.check_complex_result(r); + } + } + // Otherwise fall back to `__float__` then `__index__` — CPython's + // `PyNumber_Float`, each hook carrying its own return-type contract. + if let Some(method) = instance_method(a, "__float__") { + let r = self.call(&method, &[], &[], globals)?; + return match r { + Object::Float(_) => Ok(r), + other => Err(type_error(format!( + "{}.__float__ returned non-float (type {})", + a.type_name_owned(), + other.type_name_owned() + ))), + }; + } + if let Some(method) = instance_method(a, "__index__") { + let r = self.call(&method, &[], &[], globals)?; + return match r { + Object::Int(_) | Object::Long(_) | Object::Bool(_) => { + if long_overflows_f64(&r) { + return Err(overflow_error("int too large to convert to float")); + } + Ok(Object::Float(r.as_f64().expect("int-like"))) + } + other => Err(type_error(format!( + "{}.__index__ returned non-int (type {})", + a.type_name_owned(), + other.type_name_owned() + ))), + }; + } + // A built-in numeric subclass instance unwraps to its payload; + // anything else (e.g. only `__int__`, like a bare `MyInt`) is not + // a number for `complex()`. + if let Some(native) = a.native_value() { + return Ok(native); + } + return Err(type_error(if allow_complex { + format!( + "complex() first argument must be a string or a number, not '{}'", + a.type_name_owned() + ) + } else { + format!( + "complex() second argument must be a number, not '{}'", + a.type_name_owned() + ) + })); + } + Ok(a.native_value().unwrap_or_else(|| a.clone())) + } + + /// Validate the value returned by a user `__complex__`. CPython requires + /// a `complex`: an exact instance is used as-is, a strict subclass is + /// unwrapped to its payload after a `DeprecationWarning`, and anything + /// else is a `TypeError`. + fn check_complex_result(&mut self, r: Object) -> Result { + match &r { + Object::Complex(_) => Ok(r), + Object::Instance(inst) if matches!(inst.native, Some(Object::Complex(_))) => { + self.emit_deprecation_warning(format!( + "__complex__ returned non-complex (type {}). The ability to return \ + an instance of a strict subclass of complex is deprecated, and may \ + be removed in a future version of Python.", + r.type_name_owned() + ))?; + Ok(inst.native.clone().expect("complex payload")) + } + other => Err(type_error(format!( + "__complex__ returned non-complex (type {})", + other.type_name_owned() + ))), + } + } + + /// Validate the value returned by a user `__float__`. CPython requires a + /// `float`: an exact instance is used as-is, a strict subclass is unwrapped + /// to its payload after a `DeprecationWarning`, and anything else (an + /// `int`, a `str`, …) is a `TypeError`. + fn check_float_result(&mut self, obj: &Object, r: Object) -> Result { + match &r { + Object::Float(_) => Ok(r), + Object::Instance(inst) if matches!(inst.native, Some(Object::Float(_))) => { + self.emit_deprecation_warning(format!( + "{}.__float__ returned non-float (type {}). The ability to return \ + an instance of a strict subclass of float is deprecated, and may \ + be removed in a future version of Python.", + obj.type_name_owned(), + r.type_name_owned() + ))?; + Ok(inst.native.clone().expect("float payload")) + } + other => Err(type_error(format!( + "{}.__float__ returned non-float (type {})", + obj.type_name_owned(), + other.type_name_owned() + ))), + } + } + + /// Emit a `DeprecationWarning` via the `warnings` machinery (so + /// `assertWarns`/filters observe it). Degrades to a no-op when the + /// module is unavailable. + fn emit_deprecation_warning(&mut self, message: String) -> Result<(), RuntimeError> { + let Some(warn) = self.module_attr("warnings", "warn") else { + return Ok(()); + }; + let category = + Object::Type(crate::builtin_types::builtin_types().deprecation_warning.clone()); + let globals = self.builtins.clone(); + self.call(&warn, &[Object::from_str(message), category], &[], &globals) + .map(|_| ()) + } + + /// `pow(base, exp[, mod])` — dispatch the numeric dunders for class + /// instances. The two-argument form routes through the normal + /// `__pow__`/`__rpow__` binary-op machinery; the three-argument form + /// forwards `(exp, mod)` to a ternary `__pow__`. Built-in numerics + /// fall back to the primitive implementation. + fn do_pow_call( + &mut self, + args: &[Object], + globals: &Rc>, + ) -> Result { + if args.len() == 3 && !matches!(args[2], Object::None) { + if let Some(method) = instance_method(&args[0], "__pow__") { + return self.call(&method, &args[1..3], &[], globals); + } + // `complex.__pow__` rejects a modulus outright (CPython raises + // `ValueError("complex modulo")` rather than the integer-only + // `TypeError` that `b_pow`'s modular path would give). + if args.iter().any(|a| matches!(a, Object::Complex(_))) { + return Err(value_error("complex modulo")); + } + return crate::builtins::b_pow(args); + } + let (a, b) = (&args[0], &args[1]); + if matches!(a, Object::Instance(_)) || matches!(b, Object::Instance(_)) { + return self.dispatch_binary_op(a, b, BinOpKind::Pow, globals); + } + // `pow(complex, complex)` (and complex mixed with int/float) routes + // through the binary-op path, which carries the complex `**` + // implementation; `b_pow`'s primitive table doesn't cover complex. + if matches!(a, Object::Complex(_)) || matches!(b, Object::Complex(_)) { + return self.dispatch_binary_op(a, b, BinOpKind::Pow, globals); + } + crate::builtins::b_pow(args) + } + + /// VM-aware Python truthiness. For instances this dispatches + /// `__bool__` (then `__len__`) so user classes that define either + /// dunder are honoured in boolean contexts; everything else falls + /// back to the pure [`Object::is_truthy`]. Mirrors CPython's + /// `PyObject_IsTrue`. + fn obj_truthy( + &mut self, + v: &Object, + globals: &Rc>, + ) -> Result { + if let Object::Instance(_) = v { + if let Some(method) = instance_method(v, "__bool__") { + let r = self.call(&method, &[], &[], globals)?; + return match r { + Object::Bool(b) => Ok(b), + other => match other.as_i64() { + Some(i) => Ok(i != 0), + None => Err(type_error(format!( + "__bool__ should return bool, returned {}", + other.type_name() + ))), + }, + }; + } + if let Some(method) = instance_method(v, "__len__") { + let r = self.call(&method, &[], &[], globals)?; + return Ok(r.is_truthy()); + } + } + Ok(v.is_truthy()) } /// `bool(x)` constructor — routes through [`Self::obj_truthy`] so a @@ -3706,7 +4508,34 @@ impl Interpreter { if args.is_empty() { return Ok(Object::Int(0)); } + if args.len() > 2 { + return Err(type_error(format!( + "int() takes at most 2 arguments ({} given)", + args.len() + ))); + } + let has_base = args.len() == 2; + // CPython accepts any object implementing `__index__` as the base + // argument (`int('101', MyIndexable(2))`). Normalise it to a plain + // int before dispatching so the parser only sees integer bases. + let normalized; + let args: &[Object] = if has_base && matches!(&args[1], Object::Instance(_)) { + let base = builtins::coerce_index_i64(&args[1])?; + normalized = [args[0].clone(), Object::Int(base)]; + &normalized + } else { + args + }; match &args[0] { + // A real number with an explicit base is a TypeError — the base + // only applies to string/bytes parsing. + Object::Int(_) | Object::Long(_) | Object::Bool(_) | Object::Float(_) + if has_base => + { + Err(type_error( + "int() can't convert non-string with explicit base", + )) + } Object::Int(_) | Object::Long(_) | Object::Bool(_) @@ -3714,31 +4543,158 @@ impl Interpreter { | Object::Str(_) | Object::Bytes(_) | Object::ByteArray(_) => builtins::b_int_compat(args), + // `int(buffer)` parses the buffer's bytes as an int literal + // (CPython accepts any bytes-like object), but an explicit base + // with a non-string buffer is a TypeError. + Object::MemoryView(mv) => { + if has_base { + return Err(type_error( + "int() can't convert non-string with explicit base", + )); + } + let a: Vec = vec![Object::Bytes(Rc::from(mv.to_bytes()))]; + builtins::b_int_compat(&a) + } other => { - if let Some(method) = instance_method(other, "__int__") { - let r = self.call(&method, &[], &[], globals)?; - return match r { - Object::Int(i) => Ok(Object::Int(i)), - Object::Bool(b) => Ok(Object::Int(i64::from(b))), - other => Err(type_error(format!( - "'__int__' should return int, not '{}'", - other.type_name() - ))), - }; + // CPython's `PyNumber_Long`: with no explicit base, try + // `__int__` then `__index__` (each must return an int; a strict + // subclass is accepted with a DeprecationWarning). An int + // subclass inherits the base type's value-returning `__int__`, + // so this also covers `int(IntSubclass())`. + if !has_base { + if let Some(method) = instance_method(other, "__int__") { + let r = self.call(&method, &[], &[], globals)?; + return self.check_int_result(other, "__int__", r); + } + if let Some(method) = instance_method(other, "__index__") { + let r = self.call(&method, &[], &[], globals)?; + return self.check_int_result(other, "__index__", r); + } + // `__trunc__` is a deprecated last resort (PEP: removed in + // a future version). Its result must itself be Integral. + if let Some(method) = instance_method(other, "__trunc__") { + self.emit_deprecation_warning( + "The delegation of int() to __trunc__ is deprecated.".to_owned(), + )?; + let r = self.call(&method, &[], &[], globals)?; + return self.int_from_trunc_result(r, globals); + } + } + // A str/bytes subclass parses its native text (honouring an + // explicit base, e.g. `int(CustomStr('ff'), 16)`). + if let Some(native @ (Object::Str(_) | Object::Bytes(_) | Object::ByteArray(_))) = + other.native_value() + { + let mut a: Vec = vec![native]; + a.extend_from_slice(&args[1..]); + return self.do_int_call(&a, globals); + } + if has_base { + return Err(type_error( + "int() can't convert non-string with explicit base", + )); } - // `int` subclass instance with no `__int__` override: - // `int(x)` yields a plain int of the wrapped value. if let Some(native) = other.native_value() { return self.do_int_call(&[native], globals); } + // Buffer protocol (PEP 688) — e.g. `int(array('B', b'42'))`. + if let Some(method) = instance_method(other, "__buffer__") { + let view = self.call(&method, &[Object::Int(0)], &[], globals)?; + if let Some(bytes) = view.as_bytes_view() { + return builtins::b_int_compat(&[Object::Bytes(Rc::from(bytes))]); + } + } Err(type_error(format!( - "int() argument must be a string or a real number, not '{}'", + "int() argument must be a string, a bytes-like object or a real number, not '{}'", other.type_name() ))) } } } + /// Validate the value returned by `__int__`/`__index__` for `int()`. + /// CPython requires an `int`: an exact instance is used as-is, a strict + /// subclass is unwrapped to its payload after a `DeprecationWarning`, and + /// anything else is a `TypeError`. + fn check_int_result( + &mut self, + obj: &Object, + which: &str, + r: Object, + ) -> Result { + match &r { + Object::Int(_) | Object::Long(_) => Ok(r), + // `bool` is a strict subclass of `int`, so returning one trips the + // same deprecation as any other int subclass. + Object::Bool(_) => { + self.emit_deprecation_warning(format!( + "{}.{} returned non-int (type bool). The ability to return \ + an instance of a strict subclass of int is deprecated, and may \ + be removed in a future version of Python.", + obj.type_name_owned(), + which, + ))?; + let Object::Bool(b) = &r else { unreachable!() }; + Ok(Object::Int(i64::from(*b))) + } + Object::Instance(inst) + if matches!( + inst.native, + Some(Object::Int(_) | Object::Long(_) | Object::Bool(_)) + ) => + { + self.emit_deprecation_warning(format!( + "{}.{} returned non-int (type {}). The ability to return \ + an instance of a strict subclass of int is deprecated, and may \ + be removed in a future version of Python.", + obj.type_name_owned(), + which, + r.type_name_owned() + ))?; + let native = inst.native.clone().expect("int payload"); + self.do_int_call(&[native], &self.builtins.clone()) + } + other => Err(type_error(format!( + "{} returned non-int (type {})", + which, + other.type_name_owned() + ))), + } + } + + /// Convert the value returned by a deprecated `__trunc__` into an int. + /// CPython requires the result to be Integral: an int (or subclass), or + /// an object that itself implements `__index__`/`__int__`. Anything else + /// is a `TypeError: __trunc__ returned non-Integral (type X)`. + fn int_from_trunc_result( + &mut self, + r: Object, + globals: &Rc>, + ) -> Result { + match &r { + Object::Int(_) | Object::Long(_) => Ok(r), + Object::Bool(b) => Ok(Object::Int(i64::from(*b))), + _ => { + if let Some(native @ (Object::Int(_) | Object::Long(_) | Object::Bool(_))) = + r.native_value() + { + return self.do_int_call(&[native], globals); + } + // CPython requires the `__trunc__` result to be Integral via + // `__index__` specifically (it does not fall back to + // `__int__`); anything else is non-Integral. + if let Some(method) = instance_method(&r, "__index__") { + let v = self.call(&method, &[], &[], globals)?; + return self.check_int_result(&r, "__index__", v); + } + Err(type_error(format!( + "__trunc__ returned non-Integral (type {})", + r.type_name_owned() + ))) + } + } + } + fn do_float_call( &mut self, args: &[Object], @@ -3755,21 +4711,44 @@ impl Interpreter { | Object::Str(_) | Object::Bytes(_) | Object::ByteArray(_) => builtins::b_float_compat(args), + Object::MemoryView(_) => builtins::b_float_compat(args), other => { + // CPython's `PyNumber_Float`: try `__float__` (which must + // return a float; a strict subclass is accepted with a + // DeprecationWarning), then `__index__` (an int, converted with + // an overflow check). if let Some(method) = instance_method(other, "__float__") { + let r = self.call(&method, &[], &[], globals)?; + return self.check_float_result(other, r); + } + if let Some(method) = instance_method(other, "__index__") { let r = self.call(&method, &[], &[], globals)?; return match r { - Object::Float(f) => Ok(Object::Float(f)), - Object::Int(i) => Ok(Object::Float(i as f64)), + Object::Int(_) | Object::Long(_) | Object::Bool(_) => { + if long_overflows_f64(&r) { + return Err(overflow_error("int too large to convert to float")); + } + Ok(Object::Float(r.as_f64().expect("int-like"))) + } other => Err(type_error(format!( - "'__float__' should return float, not '{}'", - other.type_name() + "{}.__index__ returned non-int (type {})", + other.type_name_owned(), + other.type_name_owned() ))), }; } if let Some(native) = other.native_value() { return self.do_float_call(&[native], globals); } + // CPython's `PyFloat_FromString` accepts any object exposing + // the buffer protocol (e.g. `array.array`); honour PEP 688's + // `__buffer__` and parse the bytes as a float literal. + if let Some(method) = instance_method(other, "__buffer__") { + let view = self.call(&method, &[Object::Int(0)], &[], globals)?; + if let Some(bytes) = view.as_bytes_view() { + return builtins::b_float_compat(&[Object::Bytes(Rc::from(bytes))]); + } + } Err(type_error(format!( "float() argument must be a string or a real number, not '{}'", other.type_name() @@ -3873,9 +4852,32 @@ impl Interpreter { v: &Object, globals: &Rc>, ) -> Result, RuntimeError> { + // Native mappings: `dict(other_dict, **kw)` / `dict(mappingproxy)` + // copy key→value directly rather than being mistaken for an + // iterable of pairs (which would walk the *keys*). + match v { + Object::Dict(inner) => { + return Ok(Some(Object::Dict(Rc::new(RefCell::new( + inner.borrow().clone(), + ))))); + } + Object::MappingProxy(inner) => { + return Ok(Some(Object::Dict(Rc::new(RefCell::new( + inner.borrow().clone(), + ))))); + } + _ => {} + } let Object::Instance(inst) = v else { return Ok(None); }; + // A subclass of `dict` (`class C(dict)`) wraps a native dict; copy + // its entries directly rather than walking keys via subscript. + if let Some(Object::Dict(inner)) = &inst.native { + return Ok(Some(Object::Dict(Rc::new(RefCell::new( + inner.borrow().clone(), + ))))); + } // Prefer the instance's own `keys` (rare), then walk the MRO. // `inst.class.lookup` already handles inheritance, which is // how `_MappingMixin` subclasses (defaultdict, Counter, …) @@ -4073,6 +5075,75 @@ impl Interpreter { Ok(Object::Bool(!want_any)) } + /// Lazy, VM-driven `zip(*iterables, strict=False)`. The static `b_zip` + /// materialises every argument up-front via `Object::make_iter`, which + /// (a) can't drive a Python generator/instance iterator and (b) would + /// deadlock on an unbounded one such as `itertools.count()`. This pulls + /// one element per iterable per round and stops at the shortest — + /// CPython's lazy contract — so `zip(words, count())` terminates. + /// Returns an eager list, mirroring WeavePy's existing `zip` result type. + fn do_zip_call( + &mut self, + args: &[Object], + kwargs: &[(String, Object)], + globals: &Rc>, + ) -> Result { + let mut strict = false; + for (k, v) in kwargs { + if k == "strict" { + strict = v.is_truthy(); + } else { + return Err(type_error(format!( + "zip() got an unexpected keyword argument '{k}'" + ))); + } + } + if args.is_empty() { + return Ok(Object::new_list(Vec::new())); + } + let iters: Vec = args + .iter() + .map(|a| self.make_iter(a, globals)) + .collect::>()?; + let n = iters.len(); + let mut out: Vec = Vec::new(); + loop { + let mut tup: Vec = Vec::with_capacity(n); + for (i, it) in iters.iter().enumerate() { + match self.iter_next(it, globals)? { + Some(v) => tup.push(v), + None => { + if strict && i > 0 { + let than = if i > 1 { + format!("arguments 1-{i}") + } else { + "argument 1".to_owned() + }; + return Err(value_error(format!( + "zip() argument {} is shorter than {than}", + i + 1 + ))); + } + if strict { + // First iterable ran out: any later one that still + // yields is "longer than argument 1". + for (j, it2) in iters.iter().enumerate().skip(1) { + if self.iter_next(it2, globals)?.is_some() { + return Err(value_error(format!( + "zip() argument {} is longer than argument 1", + j + 1 + ))); + } + } + } + return Ok(Object::new_list(out)); + } + } + } + out.push(Object::new_tuple(tup)); + } + } + /// `isinstance(obj, classinfo)` — honours `__instancecheck__` on /// the *metaclass* of any class in `classinfo`, falling back to /// the plain MRO walk otherwise. ABCMeta uses this to register @@ -4083,7 +5154,19 @@ impl Interpreter { classinfo: &Object, globals: &Rc>, ) -> Result { - // Tuple of types: short-circuit on first match. + // `isinstance` participates in the recursion limit (CPython wraps + // `object_isinstance` in `Py_EnterRecursiveCall`): a deeply nested + // tuple of classinfos, or a cyclic `__bases__`/`__class__` graph, + // must raise `RecursionError` rather than blow the native stack. + let _guard = match crate::recursion::enter() { + crate::recursion::Enter::Ok(g) => g, + crate::recursion::Enter::Overflow => { + return Err(recursion_error( + "maximum recursion depth exceeded in __instancecheck__", + )); + } + }; + // Tuple of classinfos: short-circuit on the first match. if let Object::Tuple(items) = classinfo { for it in items.iter() { if self.do_isinstance_call(obj, it, globals)?.is_truthy() { @@ -4107,9 +5190,91 @@ impl Interpreter { return Ok(Object::Bool(res.is_truthy())); } } + // Concrete `type` check (CPython `recursive_isinstance`'s type + // branch), including the post-check `__class__` consultation. + return self.recursive_isinstance_type(obj, cls); + } + // PEP 585 parameterized generic (`list[int]`): CPython rejects these + // for instance checks — you can't ask "is x a list-of-int?". + if is_generic_alias(classinfo) { + return Err(type_error( + "isinstance() argument 2 cannot be a parameterized generic", + )); + } + // Structured matchers the builtin already understands: PEP 604 unions + // (`int | str`) and bare `None` (legacy `isinstance(x, None)` ⇒ + // `type(None)` check). + if matches!(classinfo, Object::None) || crate::is_pep604_union(classinfo).is_some() { + return Ok(Object::Bool(builtins::matches_classinfo(obj, classinfo)?)); + } + // PEP 3119: a `__instancecheck__` defined on `type(classinfo)` + // overrides the default. The concrete-`type` case is handled above + // via the metaclass; this branch covers class-like *instances* such + // as `typing` aliases (`typing.List`, `int | typing.List`) and ABC + // shims implemented as ordinary instances. + if let Object::Instance(inst) = classinfo { + if let Some(hook) = inst.class.lookup("__instancecheck__") { + let bound = Object::BoundMethod(Rc::new(BoundMethod { + receiver: classinfo.clone(), + function: hook, + })); + let res = self.call(&bound, std::slice::from_ref(obj), &[], globals)?; + return Ok(Object::Bool(res.is_truthy())); + } + } + // Otherwise `classinfo` is some other object emulating a class via a + // `__bases__` attribute (the legacy "abstract class" protocol). It + // must expose `__bases__`; if not, raise the canonical TypeError — + // unless reading `__bases__` raised something other than + // AttributeError, which must propagate (test_mask_attribute_error vs + // test_dont_mask_non_attribute_error). + if self.abstract_get_bases(classinfo)?.is_none() { + return Err(type_error( + "isinstance() arg 2 must be a type, a tuple of types, or a union", + )); + } + // Consult `obj.__class__` (a `__class__` property may lie about, or + // raise while computing, the class). A missing `__class__` ⇒ False; + // any non-AttributeError propagates. + let icls = match self.load_attr(obj, "__class__") { + Ok(c) => c, + Err(e) if self.is_attribute_error(&e) => return Ok(Object::Bool(false)), + Err(e) => return Err(e), + }; + Ok(Object::Bool(self.abstract_issubclass(&icls, classinfo)?)) + } + + /// `isinstance(obj, cls)` for a concrete `type` `cls` — CPython + /// `recursive_isinstance`'s type branch. A direct type check first; + /// then, only when that fails and only for instances that *can* override + /// it, consult `obj.__class__` so a `__class__` property is honoured. + /// Errors raised by that property are propagated, not masked + /// (bpo-1574217 / `test_isinstance_dont_mask_non_attribute_error`). + fn recursive_isinstance_type( + &mut self, + obj: &Object, + cls: &Rc, + ) -> Result { + let real = builtins::class_of(obj); + if real.is_subclass_of(cls) { + return Ok(Object::Bool(true)); + } + // Only `Instance`s can carry a custom `__class__`; for every other + // object the real type *is* `__class__`, so skip the (observable) + // attribute access on the negative path. + if let Object::Instance(_) = obj { + match self.load_attr(obj, "__class__") { + Ok(Object::Type(c)) => { + if !Rc::ptr_eq(&c, &real) && c.is_subclass_of(cls) { + return Ok(Object::Bool(true)); + } + } + Ok(_) => {} + Err(e) if self.is_attribute_error(&e) => {} + Err(e) => return Err(e), + } } - // Default path: delegate to the builtin. - Ok(Object::Bool(builtins::matches_classinfo(obj, classinfo)?)) + Ok(Object::Bool(false)) } /// `issubclass(cls, classinfo)` — same protocol as @@ -4120,6 +5285,14 @@ impl Interpreter { classinfo: &Object, globals: &Rc>, ) -> Result { + let _guard = match crate::recursion::enter() { + crate::recursion::Enter::Ok(g) => g, + crate::recursion::Enter::Overflow => { + return Err(recursion_error( + "maximum recursion depth exceeded in __subclasscheck__", + )); + } + }; if let Object::Tuple(items) = classinfo { for it in items.iter() { if self.do_issubclass_call(cls, it, globals)?.is_truthy() { @@ -4141,13 +5314,114 @@ impl Interpreter { } } } - let cls_inner = match cls { - Object::Type(t) => t.clone(), - _ => return Err(type_error("issubclass() arg 1 must be a class")), + // PEP 585 parameterized generic (`list[int]`) as the classinfo: as + // with `isinstance`, CPython rejects it for subclass checks. + if is_generic_alias(classinfo) { + return Err(type_error( + "issubclass() argument 2 cannot be a parameterized generic", + )); + } + // When the first argument is a real `type`, defer to the builtin for + // the structured matchers it understands (concrete type, bare + // `None`, PEP 604 union). + if let Object::Type(cls_inner) = cls { + if matches!(classinfo, Object::Type(_) | Object::None) + || crate::is_pep604_union(classinfo).is_some() + { + return Ok(Object::Bool(builtins::class_matches_classinfo( + cls_inner, classinfo, + )?)); + } + } + // PEP 3119: a `__subclasscheck__` on `type(classinfo)` overrides the + // default (class-like instances such as `typing` aliases / unions). + if let Object::Instance(inst) = classinfo { + if let Some(hook) = inst.class.lookup("__subclasscheck__") { + let bound = Object::BoundMethod(Rc::new(BoundMethod { + receiver: classinfo.clone(), + function: hook, + })); + let res = self.call(&bound, std::slice::from_ref(cls), &[], globals)?; + return Ok(Object::Bool(res.is_truthy())); + } + } + // Duck-typed "abstract class" protocol (CPython `recursive_issubclass` + // + `check_class`): both arguments must expose a `__bases__` tuple. A + // missing/non-tuple `__bases__` ⇒ TypeError; a non-AttributeError + // raised while reading it propagates unchanged. + self.check_class(cls, "issubclass() arg 1 must be a class")?; + self.check_class(classinfo, "issubclass() arg 2 must be a class or tuple of classes")?; + Ok(Object::Bool(self.abstract_issubclass(cls, classinfo)?)) + } + + /// CPython `abstract_get_bases`: fetch `cls.__bases__` for the duck-typed + /// "abstract class" protocol. Returns `Ok(Some(bases))` when `__bases__` + /// is a tuple (it's class-like), `Ok(None)` when `__bases__` is missing + /// (AttributeError) or not a tuple (treat as "not a class"), and `Err` + /// when reading `__bases__` raised something other than AttributeError + /// (CPython does not mask those — see the `test_*dont_mask*` cases). + fn abstract_get_bases(&mut self, cls: &Object) -> Result>, RuntimeError> { + match self.load_attr(cls, "__bases__") { + Ok(Object::Tuple(items)) => Ok(Some(items.iter().cloned().collect())), + Ok(_) => Ok(None), + Err(e) if self.is_attribute_error(&e) => Ok(None), + Err(e) => Err(e), + } + } + + /// CPython `check_class`: a duck-typed class must expose a `__bases__` + /// tuple. A missing/non-tuple `__bases__` becomes `error` (TypeError); a + /// non-AttributeError raised while reading it propagates unchanged. + fn check_class(&mut self, cls: &Object, error: &str) -> Result<(), RuntimeError> { + match self.abstract_get_bases(cls)? { + Some(_) => Ok(()), + None => Err(type_error(error.to_owned())), + } + } + + /// CPython `abstract_issubclass`: walk `derived`'s `__bases__` graph + /// looking for `cls` by identity. Recursion-guarded so cyclic or + /// unbounded `__bases__` chains raise `RecursionError` instead of blowing + /// the native stack (mirrors `Py_EnterRecursiveCall`). Single-inheritance + /// links loop without recursing (CPython's tail-call optimisation). + fn abstract_issubclass( + &mut self, + derived: &Object, + cls: &Object, + ) -> Result { + let _guard = match crate::recursion::enter() { + crate::recursion::Enter::Ok(g) => g, + crate::recursion::Enter::Overflow => { + return Err(recursion_error( + "maximum recursion depth exceeded in __subclasscheck__", + )); + } }; - Ok(Object::Bool(builtins::class_matches_classinfo( - &cls_inner, classinfo, - )?)) + let mut derived = derived.clone(); + loop { + if derived.is_same(cls) { + return Ok(true); + } + let bases = match self.abstract_get_bases(&derived)? { + Some(b) => b, + None => return Ok(false), + }; + match bases.len() { + 0 => return Ok(false), + 1 => { + derived = bases[0].clone(); + continue; + } + _ => { + for base in &bases { + if self.abstract_issubclass(base, cls)? { + return Ok(true); + } + } + return Ok(false); + } + } + } } /// `hash(obj)` — dispatch through the instance's `__hash__` if @@ -4181,6 +5455,47 @@ impl Interpreter { builtins::hash_object(obj) } + /// Reentrant `__hash__` for use by `DictKey::hash` on user instances. + /// `set`/`dict` keys are wrapped in a `DictKey` whose `Hash`/`Eq` impls + /// have no interpreter handle, so they reach back through the ambient + /// interpreter pointer (the same mechanism `_imp`/`_thread`/the C-API + /// use). Returns `None` when there is no active interpreter or the + /// `__hash__` dispatch fails, so the caller can fall back to the native + /// structural hash. + pub(crate) fn reentrant_py_hash(&mut self, obj: &Object) -> Option { + // Only dispatch when the class supplies a *callable* `__hash__`. + // Without this guard, an instance that inherits the default + // (object) hash would send `do_hash_call` down its + // `builtins::hash_object` fallback, which re-enters `DictKey::hash` + // and recurses until the stack overflows. Returning `None` here + // lets `DictKey` use its constant fallback (identity semantics), + // exactly as before this hook existed. + let Object::Instance(inst) = obj else { + return None; + }; + if !matches!( + inst.class.lookup("__hash__"), + Some(Object::Function(_) | Object::BoundMethod(_)) + ) { + return None; + } + let globals = self.builtins.clone(); + match self.do_hash_call(obj, &globals) { + Ok(Object::Int(i)) => Some(i), + Ok(Object::Bool(b)) => Some(i64::from(b)), + Ok(Object::Long(b)) => Some(crate::object::py_hash_long_bigint(&b)), + _ => None, + } + } + + /// Reentrant `a == b` (via `__eq__`) for `DictKey::eq` on user instances. + /// Returns `None` when there is no active interpreter or the comparison + /// errored, so the caller falls back to native identity equality. + pub(crate) fn reentrant_py_eq(&mut self, a: &Object, b: &Object) -> Option { + let globals = self.builtins.clone(); + self.dispatch_compare_op(a, b, CompareKind::Eq, &globals).ok() + } + /// VM-routed `getattr(obj, name[, default])`. Routes through the /// full `load_attr` path so descriptors (properties, classmethods, /// user `__get__` / `__getattr__`) behave exactly as `obj.name` @@ -4255,6 +5570,58 @@ impl Interpreter { Ok(Object::new_list(items)) } + /// VM-aware `reversed(obj)` for objects only the interpreter can + /// reverse: a user `__reversed__`, or the legacy sequence protocol + /// (`__len__` + `__getitem__`) when no `__reversed__` exists. Returns + /// an iterator over the reversed items. + fn do_reversed_call( + &mut self, + args: &[Object], + globals: &Rc>, + ) -> Result { + let obj = args + .first() + .ok_or_else(|| type_error("reversed() missing required argument"))?; + if let Object::Instance(inst) = obj { + if let Some(method) = instance_method(obj, "__reversed__") { + return self.call(&method, &[], &[], globals); + } + if let (Some(len_m), Some(getitem)) = ( + instance_method(obj, "__len__"), + instance_method(obj, "__getitem__"), + ) { + let n = self.call(&len_m, &[], &[], globals)?; + let n = n.as_i64().ok_or_else(|| { + type_error("__len__() should return an integer for reversed()") + })?; + let mut out = Vec::with_capacity(n.max(0) as usize); + let mut i = n - 1; + while i >= 0 { + out.push(self.call(&getitem, &[Object::Int(i)], &[], globals)?); + i -= 1; + } + return self.make_iter(&Object::new_list(out), globals); + } + // A built-in container subclass with no overrides reverses + // the native payload it wraps. + if let Some(native) = &inst.native { + let native = native.clone(); + let items = self.collect_iterable(&native, globals)?; + let reversed: Vec = items.into_iter().rev().collect(); + return self.make_iter(&Object::new_list(reversed), globals); + } + return Err(type_error(format!( + "'{}' object is not reversible", + obj.type_name() + ))); + } + // Generators / coroutines aren't reversible; everything else + // (lists, tuples, ranges, …) has a native reverse. + let items = self.collect_iterable(obj, globals)?; + let reversed: Vec = items.into_iter().rev().collect(); + self.make_iter(&Object::new_list(reversed), globals) + } + fn do_list_sort_call( &mut self, args: &[Object], @@ -4316,13 +5683,24 @@ impl Interpreter { v: &Object, globals: &Rc>, ) -> Result { - if let Object::Instance(_) = v { + if let Object::Instance(inst) = v { if let Some(method) = instance_method(v, "__str__") { let r = self.call(&method, &[], &[], globals)?; return Ok(r.to_str()); } + // A subclass of a built-in (`class S(str)`, `class F(float)`, …) + // with no custom `__str__` inherits the base type's `__str__`, + // i.e. it stringifies its native payload rather than falling back + // to `object.__str__` (the `` repr). + if let Some(native) = &inst.native { + let native = native.clone(); + return self.stringify(&native, globals); + } return self.repr_of(v, globals); } + if let Object::Long(b) = v { + crate::builtins::long_str_limit_check(b)?; + } Ok(v.to_str()) } @@ -4331,17 +5709,60 @@ impl Interpreter { v: &Object, globals: &Rc>, ) -> Result { - if let Object::Instance(_) = v { + if let Object::Instance(inst) = v { if let Some(method) = instance_method(v, "__repr__") { let r = self.call(&method, &[], &[], globals)?; return Ok(r.to_str()); } + // Built-in subclass with no custom `__repr__` uses the base + // type's `__repr__` on its native payload (e.g. `repr(F(2.5))` + // is `'2.5'`, not ``). + if let Some(native) = &inst.native { + let native = native.clone(); + return self.repr_of(&native, globals); + } + } + if let Object::Long(b) = v { + crate::builtins::long_str_limit_check(b)?; } Ok(v.repr()) } /// Either build a native iterator (for built-ins) or call /// `__iter__` and return whatever the user method produced. + /// CPython `_PySequence_IterSearch` fallback for `item in container` + /// when `container` has no `__contains__`: iterate (dispatching + /// `__iter__`, then the legacy `__getitem__` protocol) and compare + /// each element to `item` using identity-first rich equality (so a + /// container holding `nan` still "contains" that same `nan`). + fn contains_via_iter( + &mut self, + container: &Object, + item: &Object, + globals: &Rc>, + ) -> Result { + let it = match self.make_iter(container, globals) { + Ok(it) => it, + // CPython reports the non-iterable case as "argument of type + // 'X' is not iterable" for the `in` operator specifically. + Err(e) if is_type_error(&e) => { + return Err(type_error(format!( + "argument of type '{}' is not iterable", + container.type_name_owned() + ))); + } + Err(e) => return Err(e), + }; + while let Some(x) = self.iter_next(&it, globals)? { + if item.is_same(&x) + || self.dispatch_compare_op(item, &x, CompareKind::Eq, globals)? + { + return Ok(true); + } + } + Ok(false) + } + fn make_iter( &mut self, v: &Object, @@ -4374,6 +5795,15 @@ impl Interpreter { let list = Object::new_list(out); return self.make_iter(&list, globals); } + // A subclass of a built-in container (`class C(list)`, + // `class C(dict)`, …) that doesn't override `__iter__` + // iterates the native payload it wraps. + if let Object::Instance(inst) = v { + if let Some(native) = &inst.native { + let native = native.clone(); + return self.make_iter(&native, globals); + } + } Err(type_error(format!( "'{}' object is not iterable", v.type_name_owned() @@ -4535,26 +5965,40 @@ impl Interpreter { spec: Option<&Object>, globals: &Rc>, ) -> Result { - let s = match conversion { - 0 => self.stringify(value, globals)?, - 1 => self.stringify(value, globals)?, // !s - 2 => self.repr_of(value, globals)?, // !r - 3 => ascii_repr(value), + // CPython's FORMAT_VALUE applies the `!s`/`!r`/`!a` conversion + // *first*, then calls `format(converted, spec)` (i.e. + // `type(x).__format__(x, spec)`). A converted value is therefore a + // plain string that the spec formats as a string (so `{1.25!s:10.10}` + // left-aligns), while an unconverted value goes through its *own* + // `__format__` — which is how custom objects (and the numeric/str + // mini-language) get a crack at the spec. + let converted = match conversion { + 0 => None, + 1 => Some(self.stringify(value, globals)?), // !s + 2 => Some(self.repr_of(value, globals)?), // !r + 3 => Some(ascii_repr(value)), // !a _ => { return Err(RuntimeError::Internal(format!( "unknown f-string conversion {conversion}" ))) } }; - match spec { - None => Ok(s), - Some(Object::Str(spec_str)) => { - if spec_str.is_empty() { - return Ok(s); + match (spec, converted) { + (None, Some(s)) => Ok(s), + (None, None) => self.stringify(value, globals), + (Some(Object::Str(spec_str)), conv) => { + let empty = spec_str.is_empty(); + match conv { + // Converted: the target is the resulting string. + Some(s) if empty => Ok(s), + Some(s) => self.format_obj_str(&Object::from_str(s), spec_str, globals), + // Unconverted: keep the original value (single stringify + // for the empty-spec case; `__format__` otherwise). + None if empty => self.stringify(value, globals), + None => self.format_obj_str(value, spec_str, globals), } - apply_format_spec(value, spec_str, &s) } - Some(_) => Err(type_error("format spec must be a string")), + (Some(_), _) => Err(type_error("format spec must be a string")), } } @@ -5023,6 +6467,69 @@ impl Interpreter { } } + /// Augmented assignment (`a += b`). CPython's `binary_iop`: if + /// `type(a)` defines the in-place dunder (`__iadd__`, …) and it does + /// not decline via `NotImplemented`, use its result; otherwise fall + /// back to the regular binary operator (`a + b`), which itself tries + /// `__add__`/`__radd__`. Built-in mutable containers gain in-place + /// semantics here too (`list += iterable` extends in place). + fn dispatch_inplace_op( + &mut self, + a: &Object, + b: &Object, + op: BinOpKind, + globals: &Rc>, + ) -> Result { + // User instances: dispatch the in-place dunder first. + if matches!(a, Object::Instance(_)) { + if let Some(method) = instance_method(a, op.inplace_dunder()) { + let not_impl = crate::vm_singletons::not_implemented(); + let r = self.call(&method, std::slice::from_ref(b), &[], globals)?; + if !r.is_same(¬_impl) { + return Ok(r); + } + } + } + // Built-in mutable containers mutate in place and return `self`. + match (a, op) { + // `list += iterable` extends in place and accepts *any* iterable + // (not just another list, unlike `list + list`). + (Object::List(items), BinOpKind::Add) => { + let extra = self.collect_iterable(b, globals)?; + items.borrow_mut().extend(extra); + return Ok(a.clone()); + } + // `set`/`frozenset` in-place set algebra. `frozenset` is + // immutable, so it falls through to the binary path which + // returns a fresh object; only mutable `set` mutates here. + (Object::Set(_), BinOpKind::BitOr | BinOpKind::BitAnd | BinOpKind::Sub | BinOpKind::BitXor) => + { + let r = self.dispatch_binary_op(a, b, op, globals)?; + if let (Object::Set(dst), Object::Set(src)) = (a, &r) { + let new_items = src.borrow().clone(); + *dst.borrow_mut() = new_items; + return Ok(a.clone()); + } + return Ok(r); + } + // `bytearray += bytes-like` extends in place. + (Object::ByteArray(buf), BinOpKind::Add) => match b { + Object::Bytes(extra) => { + buf.borrow_mut().extend_from_slice(extra); + return Ok(a.clone()); + } + Object::ByteArray(extra) => { + let extra = extra.borrow().clone(); + buf.borrow_mut().extend_from_slice(&extra); + return Ok(a.clone()); + } + _ => {} + }, + _ => {} + } + self.dispatch_binary_op(a, b, op, globals) + } + fn dispatch_binary_op( &mut self, a: &Object, @@ -5031,20 +6538,50 @@ impl Interpreter { globals: &Rc>, ) -> Result { let (dunder, rdunder) = binop_dunders(op); - // `a.__op__(b)` first, then `b.__rop__(a)` if it returns - // NotImplemented. Our slice treats "no method" as - // NotImplemented and the missing-symmetric falls through to - // [`binary_op`] for built-in types. + // CPython's `binary_op1`: try `a.__op__(b)`, then `b.__rop__(a)`. + // Either may *decline* by returning `NotImplemented`, in which case + // we must keep looking rather than propagate the sentinel — a + // missing method is treated as an implicit decline. Only when both + // operands decline do we fall through to the native [`binary_op`] + // (built-in numerics/sequences) which raises the canonical + // "unsupported operand type(s)" TypeError for two instances. + let not_impl = crate::vm_singletons::not_implemented(); + let mut a_declined = false; + let mut b_declined = false; if let Some(method) = instance_method(a, dunder) { - return self.call(&method, std::slice::from_ref(b), &[], globals); + let r = self.call(&method, std::slice::from_ref(b), &[], globals)?; + if !r.is_same(¬_impl) { + return Ok(r); + } + a_declined = true; } if let Some(method) = instance_method(b, rdunder) { - return self.call(&method, std::slice::from_ref(a), &[], globals); + let r = self.call(&method, std::slice::from_ref(a), &[], globals)?; + if !r.is_same(¬_impl) { + return Ok(r); + } + b_declined = true; + } + // Both operands defined the operator and *declined* via + // `NotImplemented`. CPython raises `TypeError` here; we must NOT fall + // through to the native `binary_op`, which would otherwise apply the + // base type's operator to a wrapped value when the operands are + // `int`/`str`/… subclass instances that override `__op__` to decline + // (e.g. `test_numeric_tower`'s `DummyIntegral`). When a dunder was + // merely *absent* (not declined) we still defer to `binary_op` so a + // plain subclass without an override keeps the inherited behaviour. + if a_declined && b_declined { + return Err(type_error(format!( + "unsupported operand type(s) for {}: '{}' and '{}'", + op.as_str(), + a.type_name_owned(), + b.type_name_owned() + ))); } - // `str % args`: route through a VM-aware formatter so `%s` / `%r` - // of user instances dispatch `__str__` / `__repr__` (e.g. - // `"err: %s" % some_exception`). Other `%` operand types fall - // through to the pure `binary_op` path. + // `str % args` / `bytes % args`: route through a VM-aware formatter + // so `%s`/`%r` of user instances dispatch `__str__`/`__repr__` and + // (in bytes mode) `%b`/`%s` dispatch `__bytes__`. Other `%` operand + // types fall through to the pure `binary_op` path. if matches!(op, BinOpKind::Mod) { if let Object::Str(template) = a { let template = template.clone(); @@ -5064,13 +6601,61 @@ impl Interpreter { return Ok(Object::from_str(percent_format_with( &template, b, + PercentMode::Str, &mut resolve, )?)); } + if matches!(a, Object::Bytes(_) | Object::ByteArray(_)) { + return self.bytes_percent_format(a, b, globals); + } } binary_op(a, b, op) } + /// PEP 461 `bytes % args` / `bytearray % args`. The template is decoded + /// latin-1 so it can share the text `%`-engine, then re-encoded; the + /// result type follows the left operand. `%s`/`%b` dispatch `__bytes__` + /// (and `%a`/`%r` `__repr__`) on user instances via the VM. + fn bytes_percent_format( + &mut self, + a: &Object, + b: &Object, + globals: &Rc>, + ) -> Result { + let template: String = match a { + Object::Bytes(t) => t.iter().map(|x| *x as char).collect(), + Object::ByteArray(t) => t.borrow().iter().map(|x| *x as char).collect(), + _ => unreachable!("bytes_percent_format on non-bytes"), + }; + let mut resolve = |obj: &Object, kind: char| -> Result, RuntimeError> { + if let Object::Instance(_) = obj { + match kind { + 's' => Ok(Some(self.stringify(obj, globals)?)), + 'r' => Ok(Some(self.repr_of(obj, globals)?)), + 'b' => match instance_method(obj, "__bytes__") { + Some(m) => { + let r = self.call(&m, &[], &[], globals)?; + let raw = r.as_bytes_view().ok_or_else(|| { + type_error("__bytes__ returned non-bytes") + })?; + Ok(Some(raw.iter().map(|x| *x as char).collect())) + } + None => Ok(None), + }, + _ => Ok(None), + } + } else { + Ok(None) + } + }; + let rendered = percent_format_with(&template, b, PercentMode::Bytes, &mut resolve)?; + let out: Vec = rendered.chars().map(|c| c as u8).collect(); + Ok(match a { + Object::ByteArray(_) => Object::new_bytearray(out), + _ => Object::new_bytes(out), + }) + } + fn dispatch_compare_op( &mut self, a: &Object, @@ -5079,17 +6664,51 @@ impl Interpreter { globals: &Rc>, ) -> Result { let (dunder, swapped) = cmp_dunder(op); + // A reflected/forward dunder may decline by returning + // `NotImplemented`; treating that sentinel as a truthy result is + // wrong (e.g. `Fraction < complex` must raise, not return True). + // Mirror CPython's `do_richcompare`: try forward, then reflected, + // and only if *both* decline fall through to the native default + // (identity for ==/!=, `TypeError` for an ordering). + let not_impl = crate::vm_singletons::not_implemented(); if let Some(method) = instance_method(a, dunder) { let r = self.call(&method, std::slice::from_ref(b), &[], globals)?; - return Ok(r.is_truthy()); + if !r.is_same(¬_impl) { + return Ok(r.is_truthy()); + } } if let Some(method) = instance_method(b, swapped) { let r = self.call(&method, std::slice::from_ref(a), &[], globals)?; - return Ok(r.is_truthy()); + if !r.is_same(¬_impl) { + return Ok(r.is_truthy()); + } + } + // CPython's default `object.__ne__` inverts `__eq__`: a class that + // defines only `__eq__` still compares with `!=`. When neither + // operand supplied a usable `__ne__` above, derive the result from + // `__eq__` (forward then reflected) before falling back to identity. + if matches!(op, CompareKind::NotEq) { + if let Some(method) = instance_method(a, "__eq__") { + let r = self.call(&method, std::slice::from_ref(b), &[], globals)?; + if !r.is_same(¬_impl) { + return Ok(!r.is_truthy()); + } + } + if let Some(method) = instance_method(b, "__eq__") { + let r = self.call(&method, std::slice::from_ref(a), &[], globals)?; + if !r.is_same(¬_impl) { + return Ok(!r.is_truthy()); + } + } } - // Container equality must defer to per-element `__eq__` so - // that wrapper objects with custom equality (e.g. mock.ANY) - // compare as expected when embedded in a tuple/list. + // Container comparison must recurse *through the interpreter* + // (not the native `Object::eq_value`/`cmp`) so that (a) per-element + // `__eq__` is honoured for wrapper objects (e.g. mock.ANY) embedded + // in a tuple/list, and (b) a reflexive container raises + // `RecursionError` via the WS1 guard rather than overflowing the + // native Rust stack and `abort()`ing. The helpers hold a + // `recursion::enter()` guard on each descent, mirroring CPython's + // `Py_EnterRecursiveCall(" in comparison")` in `do_richcompare`. if matches!(op, CompareKind::Eq | CompareKind::NotEq) { if let Some(rv) = self.deep_equal_collection(a, b, globals)? { let truth = match op { @@ -5098,6 +6717,8 @@ impl Interpreter { }; return Ok(truth); } + } else if let Some(rv) = self.deep_order_collection(a, b, op, globals)? { + return Ok(rv); } compare_op(a, b, op) } @@ -5345,9 +6966,23 @@ impl Interpreter { return self.deopt_load_global_slow(frame, cache_pc, name_idx); } let g = frame.globals.borrow(); - if let Some((_, v)) = g.get_index(key_idx as usize) { - specialize::record_hit(op_idx); - return Ok(v.clone()); + if let Some((k, v)) = g.get_index(key_idx as usize) { + // Verify the key at the cached slot still matches the + // expected name. `del` of an earlier global shift-removes + // an IndexMap entry, renumbering every later slot without + // changing the dict's Rc identity — so the cached index + // would otherwise alias a *different* global's value. + if let Object::Str(s) = &k.0 { + if frame + .code + .names + .get(name_idx as usize) + .is_some_and(|n| n.as_str() == &**s) + { + specialize::record_hit(op_idx); + return Ok(v.clone()); + } + } } drop(g); self.deopt_load_global_slow(frame, cache_pc, name_idx) @@ -5362,19 +6997,30 @@ impl Interpreter { // Guard that the name *isn't* shadowed in globals // since we last specialized — otherwise we'd // bypass user code that subsequently bound the name - // at module scope. - let name = self.name_at(&frame.code, name_idx)?; + // at module scope. Read the name as a borrowed `&str` + // (no `String` clone) on this hot builtin-load path. + let name: &str = frame + .code + .names + .get(name_idx as usize) + .map(String::as_str) + .ok_or_else(|| RuntimeError::Internal("bad name index".to_owned()))?; if frame .globals .borrow() - .contains_key(&DictKey(Object::from_str(&name))) + .contains_key(&DictKey(Object::from_str(name))) { return self.deopt_load_global_slow(frame, cache_pc, name_idx); } let b = self.builtins.borrow(); - if let Some((_, v)) = b.get_index(key_idx as usize) { - specialize::record_hit(op_idx); - return Ok(v.clone()); + if let Some((k, v)) = b.get_index(key_idx as usize) { + // Same staleness guard as LoadGlobalModule: a removal from + // the builtins dict renumbers slots without changing its + // Rc identity, so confirm the key still matches `name`. + if matches!(&k.0, Object::Str(s) if &**s == name) { + specialize::record_hit(op_idx); + return Ok(v.clone()); + } } drop(b); self.deopt_load_global_slow(frame, cache_pc, name_idx) @@ -5449,12 +7095,14 @@ impl Interpreter { if let Object::Instance(inst) = &receiver { if specialize::rc_id(&inst.class) == type_id { let dict = inst.dict.borrow(); - if let Some((_, v)) = dict.get_index(key_idx as usize) { - let v = v.clone(); - drop(dict); - frame.pop()?; - specialize::record_hit(op_idx); - return Ok(v); + if let Some((k, v)) = dict.get_index(key_idx as usize) { + if self.cached_slot_name_matches(&frame.code, name_idx, k) { + let v = v.clone(); + drop(dict); + frame.pop()?; + specialize::record_hit(op_idx); + return Ok(v); + } } } } @@ -5465,12 +7113,14 @@ impl Interpreter { if let Object::Module(m) = &receiver { if specialize::rc_id(&m.dict) == module_id { let dict = m.dict.borrow(); - if let Some((_, v)) = dict.get_index(key_idx as usize) { - let v = v.clone(); - drop(dict); - frame.pop()?; - specialize::record_hit(op_idx); - return Ok(v); + if let Some((k, v)) = dict.get_index(key_idx as usize) { + if self.cached_slot_name_matches(&frame.code, name_idx, k) { + let v = v.clone(); + drop(dict); + frame.pop()?; + specialize::record_hit(op_idx); + return Ok(v); + } } } } @@ -5481,7 +7131,8 @@ impl Interpreter { if let Object::Instance(inst) = &receiver { if specialize::rc_id(&inst.class) == type_id { let dict = inst.class.dict.borrow(); - if let Some((_, v)) = dict.get_index(key_idx as usize) { + if let Some((k, v)) = dict.get_index(key_idx as usize) { + if self.cached_slot_name_matches(&frame.code, name_idx, k) { let v = v.clone(); drop(dict); frame.pop()?; @@ -5507,6 +7158,7 @@ impl Interpreter { return self.deopt_load_attr_slow(frame, cache_pc, name_idx); } return Ok(v); + } } } } @@ -5896,13 +7548,39 @@ impl Interpreter { b: &Object, globals: &Rc>, ) -> Result, RuntimeError> { + // Only homogeneous container pairs recurse element-wise; everything + // else falls through to the native scalar comparison. Entering the + // recursion guard *only* on the container path keeps scalar `==` + // free of the depth bookkeeping while still bounding the reflexive + // case (CPython does the same via `Py_EnterRecursiveCall`). + let is_container = matches!( + (a, b), + (Object::Tuple(_), Object::Tuple(_)) + | (Object::List(_), Object::List(_)) + | (Object::Dict(_), Object::Dict(_)) + ); + if !is_container { + return Ok(None); + } + let _rg = match crate::recursion::enter() { + crate::recursion::Enter::Ok(g) => g, + crate::recursion::Enter::Overflow => { + return Err(recursion_error( + "maximum recursion depth exceeded in comparison", + )); + } + }; match (a, b) { (Object::Tuple(xs), Object::Tuple(ys)) => { if xs.len() != ys.len() { return Ok(Some(false)); } + let xs = xs.clone(); + let ys = ys.clone(); for (x, y) in xs.iter().zip(ys.iter()) { - if !self.dispatch_compare_op(x, y, CompareKind::Eq, globals)? { + // `PyObject_RichCompareBool` is identity-first, so a + // sequence containing `nan` is equal to itself. + if !(x.is_same(y) || self.dispatch_compare_op(x, y, CompareKind::Eq, globals)?) { return Ok(Some(false)); } } @@ -5915,16 +7593,95 @@ impl Interpreter { return Ok(Some(false)); } for (x, y) in xs.iter().zip(ys.iter()) { - if !self.dispatch_compare_op(x, y, CompareKind::Eq, globals)? { + if !(x.is_same(y) || self.dispatch_compare_op(x, y, CompareKind::Eq, globals)?) { return Ok(Some(false)); } } Ok(Some(true)) } + (Object::Dict(xs), Object::Dict(ys)) => { + // Snapshot both mappings before recursing so a user + // `__eq__` that mutates a dict can't invalidate a live + // borrow. CPython compares two dicts as equal iff they + // have the same keys and each maps to an equal value. + let xs: Vec<(DictKey, Object)> = xs + .borrow() + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + let ys = ys.borrow().clone(); + if xs.len() != ys.len() { + return Ok(Some(false)); + } + for (k, v) in xs { + match ys.get(&k) { + Some(v2) => { + let v2 = v2.clone(); + if !(v.is_same(&v2) + || self.dispatch_compare_op(&v, &v2, CompareKind::Eq, globals)?) + { + return Ok(Some(false)); + } + } + None => return Ok(Some(false)), + } + } + Ok(Some(true)) + } _ => Ok(None), } } + /// Element-wise ordering (`<`, `<=`, `>`, `>=`) for `list`/`tuple`, + /// recursing *through the interpreter* so reflexive sequences raise + /// `RecursionError` (WS1) rather than overflowing the native stack via + /// `Object::cmp`. Returns `Ok(None)` for any non-sequence pair so the + /// caller falls through to the native total-order comparison (which + /// raises `TypeError` for unorderable types like two dicts). + fn deep_order_collection( + &mut self, + a: &Object, + b: &Object, + op: CompareKind, + globals: &Rc>, + ) -> Result, RuntimeError> { + let (xs, ys): (Vec, Vec) = match (a, b) { + (Object::Tuple(xs), Object::Tuple(ys)) => { + (xs.iter().cloned().collect(), ys.iter().cloned().collect()) + } + (Object::List(xs), Object::List(ys)) => { + (xs.borrow().clone(), ys.borrow().clone()) + } + _ => return Ok(None), + }; + let _rg = match crate::recursion::enter() { + crate::recursion::Enter::Ok(g) => g, + crate::recursion::Enter::Overflow => { + return Err(recursion_error( + "maximum recursion depth exceeded in comparison", + )); + } + }; + // Find the first index where the sequences differ (by `==`), then + // decide the ordering on that element. If one is a prefix of the + // other, fall back to comparing lengths — matching CPython's + // `list`/`tuple` rich comparison. + let common = xs.len().min(ys.len()); + for i in 0..common { + if !self.dispatch_compare_op(&xs[i], &ys[i], CompareKind::Eq, globals)? { + return Ok(Some(self.dispatch_compare_op(&xs[i], &ys[i], op, globals)?)); + } + } + let truth = match op { + CompareKind::Lt => xs.len() < ys.len(), + CompareKind::LtE => xs.len() <= ys.len(), + CompareKind::Gt => xs.len() > ys.len(), + CompareKind::GtE => xs.len() >= ys.len(), + _ => unreachable!("deep_order_collection only handles ordering ops"), + }; + Ok(Some(truth)) + } + fn store_attr(&mut self, obj: &Object, name: &str, value: Object) -> Result<(), RuntimeError> { match obj { Object::Instance(inst) => self.store_attr_instance(inst, obj, name, value), @@ -5938,6 +7695,11 @@ impl Interpreter { ty.dict .borrow_mut() .insert(DictKey(Object::from_str(name)), value); + // Reassigning `__getattribute__` changes the resolved slot for + // this type and its subclasses; drop the cached classification. + if name == "__getattribute__" { + ty.invalidate_getattribute_cache(); + } Ok(()) } Object::Module(m) => { @@ -6153,6 +7915,28 @@ impl Interpreter { } Ok(()) } + Object::Type(ty) => { + // `del Cls.attr` (CPython `type.__delattr__`) removes the + // name from the class's *own* dict only — inherited + // attributes can't be deleted via a subclass. Mirrors + // `assertFalse(hasattr(A, 'foo'))` after `del A.foo` in + // `abc.update_abstractmethods` round-trips. + let removed = ty + .dict + .borrow_mut() + .shift_remove(&DictKey(Object::from_str(name))) + .is_some(); + if !removed { + return Err(attribute_error(format!( + "type object '{}' has no attribute '{}'", + ty.name, name + ))); + } + if name == "__getattribute__" { + ty.invalidate_getattribute_cache(); + } + Ok(()) + } _ => Err(type_error(format!( "'{}' object has no attribute '{}'", obj.type_name(), @@ -6161,6 +7945,37 @@ impl Interpreter { } } + /// Coerce a path-like argument to `str`/`bytes` via `__fspath__`, for the + /// Rust path builtins (`open`, `os.fspath`/`fsdecode`/`fsencode`) which + /// can't call back into the interpreter themselves. `str`/`bytes` pass + /// through unchanged; an object whose type defines `__fspath__` (e.g. a + /// `pathlib` path, or `tempfile`'s `FakePath`) is reduced to that method's + /// result (which CPython requires to be str or bytes); anything else is + /// returned untouched so the builtin raises its own `TypeError`. + fn fspath_coerce( + &mut self, + obj: &Object, + globals: &Rc>, + ) -> Result { + match obj { + Object::Str(_) | Object::Bytes(_) => Ok(obj.clone()), + _ => match instance_method(obj, "__fspath__") { + Some(method) => { + let res = self.call(&method, &[], &[], globals)?; + match res { + Object::Str(_) | Object::Bytes(_) => Ok(res), + other => Err(type_error(format!( + "expected {}.__fspath__() to return str or bytes, not {}", + obj.type_name(), + other.type_name() + ))), + } + } + None => Ok(obj.clone()), + }, + } + } + fn binary_subscr(&self, container: &Object, index: &Object) -> Result { // `Type[...]` dispatches to `__class_getitem__` when defined. // We can't reach into Vm::call from a `&self` method, so we @@ -6179,15 +7994,36 @@ impl Interpreter { container: &Object, index: &Object, ) -> Result { - // An `int` subclass instance used as an index (`xs[op]` where - // `op` is e.g. a `_NamedIntConstant`) acts as its int value. + // `bool` is an `int` subclass, so `seq[True]`/`seq[False]` index as + // `seq[1]`/`seq[0]` (CPython relies on this, e.g. `mimetypes` stores + // its strict/non-strict maps in a 2-tuple indexed by a bool). An + // `int` subclass *instance* (e.g. a `_NamedIntConstant`) used as an + // index likewise acts as its int value. Normalise both to a plain + // `Int` before dispatching so the sequence arms below match. let unwrapped = match index { + Object::Bool(b) => Some(Object::Int(i64::from(*b))), Object::Instance(_) => index .native_value() - .filter(|n| matches!(n, Object::Int(_) | Object::Long(_) | Object::Bool(_))), + .filter(|n| matches!(n, Object::Int(_) | Object::Long(_) | Object::Bool(_))) + .map(normalize_bool_index), _ => None, }; let index = unwrapped.as_ref().unwrap_or(index); + // A subclass of a built-in container (`class C(list)`, + // `class C(dict)`, …) subscripts the native payload it wraps + // when it doesn't override `__getitem__` (the dispatch site + // tries `__getitem__` first, so reaching here means it didn't). + let native_container; + let container = match container { + Object::Instance(inst) => match &inst.native { + Some(native) => { + native_container = native.clone(); + &native_container + } + None => container, + }, + _ => container, + }; match (container, index) { (Object::List(items), Object::Int(i)) => { let items = items.borrow(); @@ -6204,6 +8040,7 @@ impl Interpreter { Ok(Object::from_str(chars[idx].to_string())) } (Object::Dict(d), key) => { + crate::builtins::ensure_hashable(key)?; let d = d.borrow(); d.get(&DictKey(key.clone())) .cloned() @@ -6321,6 +8158,32 @@ impl Interpreter { value: Object, globals: &Rc>, ) -> Result<(), RuntimeError> { + // `bool` indexes as `int` (`seq[True] = …` ≡ `seq[1] = …`), and an + // `int` subclass instance acts as its int value — same normalisation + // as the read path in `binary_subscr_basic`. + let unwrapped = match index { + Object::Bool(b) => Some(Object::Int(i64::from(*b))), + Object::Instance(_) => index + .native_value() + .filter(|n| matches!(n, Object::Int(_) | Object::Long(_) | Object::Bool(_))) + .map(normalize_bool_index), + _ => None, + }; + let index = unwrapped.as_ref().unwrap_or(index); + // A subclass of a mutable built-in container (`class C(list)`, + // `class C(dict)`, …) that doesn't override `__setitem__` + // assigns into the native payload it wraps. + let native_container; + let container = match container { + Object::Instance(inst) => match &inst.native { + Some(native) => { + native_container = native.clone(); + &native_container + } + None => container, + }, + _ => container, + }; match (container, index) { (Object::List(items), Object::Int(i)) => { let mut items = items.borrow_mut(); @@ -6349,6 +8212,7 @@ impl Interpreter { Ok(()) } (Object::Dict(d), key) => { + crate::builtins::ensure_hashable(key)?; d.borrow_mut().insert(DictKey(key.clone()), value); Ok(()) } @@ -6370,6 +8234,18 @@ impl Interpreter { } fn delete_subscr(&self, container: &Object, index: &Object) -> Result<(), RuntimeError> { + // `bool` indexes as `int` (`del seq[True]` ≡ `del seq[1]`); an int + // subclass instance acts as its int value (mirrors the read/store + // paths above). + let unwrapped = match index { + Object::Bool(b) => Some(Object::Int(i64::from(*b))), + Object::Instance(_) => index + .native_value() + .filter(|n| matches!(n, Object::Int(_) | Object::Long(_) | Object::Bool(_))) + .map(normalize_bool_index), + _ => None, + }; + let index = unwrapped.as_ref().unwrap_or(index); match (container, index) { (Object::List(items), Object::Int(i)) => { let mut items = items.borrow_mut(); @@ -6381,6 +8257,7 @@ impl Interpreter { apply_slice_deletion(&mut items.borrow_mut(), s) } (Object::Dict(d), key) => { + crate::builtins::ensure_hashable(key)?; if d.borrow_mut().shift_remove(&DictKey(key.clone())).is_none() { return Err(key_error(key.repr())); } @@ -6431,10 +8308,15 @@ impl Interpreter { if b.name == "__new__" && args.len() == 4 { if let Object::Type(mcs) = &args[0] { if mcs.is_subclass_of(&builtin_types().type_) { + // Bare `type.__new__`: build the class but do + // not run the metaclass `__init__` (the + // caller's `type.__call__` / `build_class` + // does that). return self.dynamic_type_call_with_meta( mcs.clone(), &args[1..], kwargs, + false, ); } } @@ -6451,6 +8333,22 @@ impl Interpreter { if b.name == "len" && args.len() == 1 { return self.do_len_call(&args[0], outer_globals); } + if b.name == "abs" && args.len() == 1 { + return self.do_abs_call(&args[0], outer_globals); + } + if b.name == "round" && (args.len() == 1 || args.len() == 2) { + return self.do_round_call(args, outer_globals); + } + if b.name == "divmod" && args.len() == 2 { + return self.do_divmod_call(args, outer_globals); + } + if b.name == "complex" && (args.len() == 1 || args.len() == 2) && kwargs.is_empty() + { + return self.do_complex_call(args, outer_globals); + } + if b.name == "pow" && (args.len() == 2 || args.len() == 3) { + return self.do_pow_call(args, outer_globals); + } if b.name == "bool" && args.len() <= 1 { return self.do_bool_call(args, outer_globals); } @@ -6512,6 +8410,23 @@ impl Interpreter { if b.name == "hasattr" && args.len() == 2 { return self.do_hasattr_call(args, outer_globals); } + // Unbound `object.__getattribute__(self, name)` — the default + // attribute lookup, reached when a user override delegates up + // (`return object.__getattribute__(self, name)`). Runs the + // default path directly so it never re-enters the override. + if b.name == ".object_getattribute" && args.len() == 2 { + let recv = args[0].clone(); + let name = match &args[1] { + Object::Str(s) => s.to_string(), + other => { + return Err(type_error(format!( + "attribute name must be string, not '{}'", + other.type_name() + ))) + } + }; + return self.object_default_getattribute(&recv, &name); + } if b.name == "globals" && args.is_empty() && kwargs.is_empty() { // CPython returns the calling function's module // globals. With our frame-by-argument model, the @@ -6531,6 +8446,28 @@ impl Interpreter { } return Ok(Object::Dict(outer_globals.clone())); } + if b.name == "dir" && args.is_empty() && kwargs.is_empty() { + // `dir()` with no argument returns the sorted names + // bound in the *current* local scope — CPython's + // `sorted(locals())`. (With an argument it falls + // through to the generic `b_dir` introspection.) + let locals = match self.frame_stack.borrow().last() { + Some(top) => top.locals(), + None => Object::Dict(outer_globals.clone()), + }; + let mut names: Vec = Vec::new(); + if let Object::Dict(d) = &locals { + for (k, _) in d.borrow().iter() { + if let Object::Str(s) = &k.0 { + names.push(s.to_string()); + } + } + } + names.sort(); + return Ok(Object::new_list( + names.into_iter().map(Object::from_str).collect(), + )); + } if b.name == "breakpoint" { return self.do_breakpoint_call(args, kwargs, outer_globals); } @@ -6589,6 +8526,40 @@ impl Interpreter { return (b.call)(&new_args); } } + // Same hazard for the free-function builtins that iterate + // their argument via `Object::make_iter` (which can't drive a + // Python generator/coroutine frame, nor a user `__iter__`). + // `list`/`tuple`/`set`/`dict`/`sorted`/`min`/`max` have their + // own VM-aware paths; route the remaining consumers through + // `collect_iterable` when handed something only the + // interpreter can iterate. Single-iterable builtins take it + // in `args[0]`; `zip` takes one per argument. + fn needs_vm_iter(o: &Object) -> bool { + matches!( + o, + Object::Generator(_) + | Object::Coroutine(_) + | Object::AsyncGenerator(_) + | Object::Instance(_) + ) + } + if matches!(b.name, "enumerate" | "sum" | "all" | "any") + && args.first().is_some_and(needs_vm_iter) + { + // These consume their iterable in full (or short-circuit + // on a finite prefix), so eager materialisation matches + // CPython's observable result. + let collected = self.collect_iterable(&args[0], outer_globals)?; + let mut new_args = args.to_vec(); + new_args[0] = Object::new_list(collected); + return (b.call)(&new_args); + } + // `zip` must NOT pre-materialise — it stops at the shortest + // iterable, so a paired unbounded iterator (`itertools.count`) + // would hang. Drive it lazily through the interpreter instead. + if b.name == "zip" && args.iter().any(needs_vm_iter) { + return self.do_zip_call(args, kwargs, outer_globals); + } if b.name == "sorted" && !args.is_empty() { return self.do_sorted_call(args, kwargs, outer_globals); } @@ -6598,30 +8569,96 @@ impl Interpreter { if (b.name == "min" || b.name == "max") && !args.is_empty() { return self.do_min_max_call(b.name, args, kwargs, outer_globals); } - // `format`'s dispatching: when args[0] is a string we - // assume this is `"...".format(...)` (str_format - // builtin) and pass kwargs through. Otherwise fall - // back to the global builtin `format(value, spec)`. + if b.name == "reversed" && args.first().is_some_and(needs_vm_iter) { + return self.do_reversed_call(args, outer_globals); + } + // `setattr`/`delattr` must honour the descriptor protocol + // (data descriptors / `property` setters), `__slots__` + // enforcement, and a user `__setattr__`/`__delattr__` — the + // same machinery the `STORE_ATTR`/`DELETE_ATTR` opcodes use. + // The bare builtins write straight to the instance dict and + // would silently bypass all of that. + if b.name == "setattr" && args.len() == 3 { + let name = match &args[1] { + Object::Str(s) => s.to_string(), + _ => return Err(type_error("attribute name must be string")), + }; + self.store_attr(&args[0], &name, args[2].clone())?; + return Ok(Object::None); + } + if b.name == "delattr" && args.len() == 2 { + let name = match &args[1] { + Object::Str(s) => s.to_string(), + _ => return Err(type_error("attribute name must be string")), + }; + self.delete_attr(&args[0], &name)?; + return Ok(Object::None); + } + // The `str.format` / `str.format_map` *methods* are + // registered under the sentinel names `.format` / + // `.format_map` so they're distinguishable from the + // global `format(value, spec)` builtin (which shares the + // user-visible name `format`). Route the methods through + // the interpreter-aware engine so nested specs, + // conversions and user `__format__` all work. + if b.name == ".format" { + let template = match args.first() { + Some(Object::Str(s)) => s.to_string(), + _ => return Err(type_error("str.format requires a 'str' receiver")), + }; + let rest = &args[1..]; + return self + .do_str_format(&template, rest, kwargs, outer_globals) + .map(Object::from_str); + } + if b.name == ".format_map" { + let template = match args.first() { + Some(Object::Str(s)) => s.to_string(), + _ => return Err(type_error("str.format_map requires a 'str' receiver")), + }; + let mapping = match args.get(1) { + Some(Object::Dict(d)) => d.clone(), + Some(_) | None => { + return Err(type_error("format_map() argument must be a mapping")) + } + }; + return self + .do_str_format_map(&template, &mapping, outer_globals) + .map(Object::from_str); + } + // Global `format(value[, spec])` builtin — dispatches to + // `value.__format__(spec)` (interpreter-aware). if b.name == "format" { - if matches!(args.first(), Some(Object::Str(_))) && !args.is_empty() { - let template = match &args[0] { - Object::Str(s) => s.to_string(), - _ => unreachable!(), - }; - let rest = &args[1..]; - return self - .do_str_format(&template, rest, kwargs, outer_globals) - .map(Object::from_str); - } - if args.len() == 1 || args.len() == 2 { - let spec = match args.get(1) { - Some(Object::Str(s)) => s.to_string(), - None => String::new(), - Some(_) => return Err(type_error("format() spec must be a string")), - }; - return Ok(Object::from_str(format_via_spec(&args[0], &spec)?)); + if args.is_empty() || args.len() > 2 { + return Err(type_error("format() takes 1 or 2 arguments")); } + let spec = match args.get(1) { + Some(Object::Str(s)) => s.to_string(), + None => String::new(), + Some(_) => return Err(type_error("format() spec must be a string")), + }; + return Ok(Object::from_str( + self.format_obj_str(&args[0], &spec, outer_globals)?, + )); } + // PathLike (`__fspath__`) coercion for the path-accepting + // builtins. Our Rust `open`/`os.fspath`/`os.fsdecode`/ + // `os.fsencode` only understand str/bytes, so reduce an + // instance argument here (where we can call back into the + // interpreter) — this is what lets `open(pathlib_path)` and + // friends accept any `os.PathLike`. + let coerced_args; + let args = if matches!(b.name, "open" | "fspath" | "fsdecode" | "fsencode") + && matches!(args.first(), Some(Object::Instance(_))) + { + let head = self.fspath_coerce(&args[0], outer_globals)?; + let mut v = args.to_vec(); + v[0] = head; + coerced_args = v; + &coerced_args[..] + } else { + args + }; if let Some(call_kw) = b.call_kw.as_ref() { return call_kw(args, kwargs); } @@ -6634,6 +8671,10 @@ impl Interpreter { (b.call)(args) } Object::Function(f) => self.call_python(f, args, kwargs), + // Since Python 3.10 (bpo-43682) `staticmethod` objects are + // themselves callable and simply forward to the wrapped + // function with the arguments unchanged. + Object::StaticMethod(inner) => self.call(inner, args, kwargs, outer_globals), Object::BoundMethod(bm) => { // Generator / coroutine / async-generator methods are // wired through internal builtin names so the @@ -6642,6 +8683,19 @@ impl Interpreter { // `fn(&[Object])` and can't.) if let Object::Builtin(b) = &bm.function { match b.name { + ".type_subclasses" => { + if let Object::Type(ty) = &bm.receiver { + let subs = ty + .subclasses() + .into_iter() + .map(Object::Type) + .collect(); + return Ok(Object::new_list(subs)); + } + return Err(type_error( + "__subclasses__() requires a type receiver", + )); + } ".gen_send" => { let value = args.first().cloned().unwrap_or(Object::None); return self.gen_method_send(&bm.receiver, value); @@ -6685,6 +8739,50 @@ impl Interpreter { ".agen_close" => { return self.gen_method_close(&bm.receiver); } + // `object.__reduce_ex__(self, protocol)` — the + // default pickling/copy reduction. Needs VM access + // to import `copyreg` and call the receiver's + // `__getstate__`/`__getnewargs__` hooks, so it is + // wired through a sentinel name here. + ".object_reduce_ex" => { + let proto = args + .first() + .and_then(|o| o.as_i64()) + .unwrap_or(0); + return self.object_reduce_ex( + &bm.receiver, + proto, + outer_globals, + ); + } + ".object_reduce" => { + return self.object_default_reduce( + &bm.receiver, + 2, + outer_globals, + ); + } + // Bound `x.__getattribute__(name)` resolving to + // `object.__getattribute__` (i.e. no user override): + // run the default lookup against the bound receiver. + ".object_getattribute" => { + let name = match args.first() { + Some(Object::Str(s)) => s.to_string(), + Some(other) => { + return Err(type_error(format!( + "attribute name must be string, not '{}'", + other.type_name() + ))) + } + None => { + return Err(type_error( + "__getattribute__() takes exactly one argument (0 given)" + .to_owned(), + )) + } + }; + return self.object_default_getattribute(&bm.receiver, &name); + } _ => {} } } @@ -6708,13 +8806,13 @@ impl Interpreter { } // `type(name, bases, ns)` builds a new class dynamically. if Rc::ptr_eq(ty, &builtin_types().type_) && args.len() == 3 { - return self.dynamic_type_call_with_meta(ty.clone(), args, kwargs); + return self.dynamic_type_call_with_meta(ty.clone(), args, kwargs, true); } // `Meta(name, bases, ns)` for a user metaclass — // route through the metaclass-aware class builder. let bt = builtin_types(); if ty.is_subclass_of(&bt.type_) && !Rc::ptr_eq(ty, &bt.type_) && args.len() == 3 { - return self.dynamic_type_call_with_meta(ty.clone(), args, kwargs); + return self.dynamic_type_call_with_meta(ty.clone(), args, kwargs, true); } // If the class's *metaclass* overrides `__call__`, // dispatch through it so EnumMeta etc. can hook @@ -6775,8 +8873,48 @@ impl Interpreter { Object::Str(s) => s.to_string(), _ => return Err(type_error("__build_class__ arg 2 must be a str")), }; + // PEP 560: resolve `__mro_entries__` for any base that isn't a + // class. `class P(NamedTuple)` / `class G(SomeAlias[int])` pass a + // *non-type* base (a function or generic alias) that knows how to + // substitute the real MRO entries. We hand each such base the + // original bases tuple and splice in whatever it returns, then + // stamp `__orig_bases__` so introspection (typing, dataclasses) + // can recover the unresolved bases — exactly like CPython's + // `__build_class__` / `types.resolve_bases`. + let orig_bases: Vec = args[2..].to_vec(); + let mut resolved_bases: Vec = Vec::with_capacity(orig_bases.len()); + let mut bases_replaced = false; + for b in &orig_bases { + if matches!(b, Object::Type(_)) { + resolved_bases.push(b.clone()); + continue; + } + let entries_method = match self.load_attr(b, "__mro_entries__") { + Ok(m) => Some(m), + Err(e) if self.is_attribute_error(&e) => None, + Err(e) => return Err(e), + }; + match entries_method { + Some(method) => { + let orig_tuple = Object::new_tuple(orig_bases.clone()); + let result = + self.call(&method, std::slice::from_ref(&orig_tuple), &[], &body_fn.globals)?; + match result { + Object::Tuple(t) => { + for e in t.iter() { + resolved_bases.push(e.clone()); + } + bases_replaced = true; + } + _ => return Err(type_error("__mro_entries__ must return a tuple")), + } + } + None => resolved_bases.push(b.clone()), + } + } + let mut bases: Vec> = Vec::new(); - for b in &args[2..] { + for b in &resolved_bases { match b { Object::Type(t) => bases.push(t.clone()), other => { @@ -6837,6 +8975,15 @@ impl Interpreter { { ns.insert(DictKey(Object::from_static("__module__")), module_name); } + // PEP 560: preserve the pre-resolution bases so `typing` / + // `dataclasses` introspection (and `NamedTuple`/`Generic`) + // can read the original `(NamedTuple,)` / `(Generic[T],)`. + if bases_replaced { + ns.insert( + DictKey(Object::from_static("__orig_bases__")), + Object::new_tuple(orig_bases.clone()), + ); + } } // Build a frame for the class body. Locals are unused; names // store and load through `class_ns`. The body's `__class__` @@ -6923,15 +9070,26 @@ impl Interpreter { metaclass.clone(), &call_args, &subclass_kwargs, + true, )?, }; let ty = match class_obj { Object::Type(t) => t, + // CPython's `type.__call__`: when `metaclass.__new__` + // returns something that isn't an instance of the + // metaclass, the result is returned verbatim and + // `__init__` is skipped. (`__set_name__` / + // `__init_subclass__` already ran inside the real + // `type.__new__` via `dynamic_type_call_with_meta`.) other => { - return Err(type_error(format!( - "metaclass.__new__ must return a type, got '{}'", - other.type_name() - ))) + for (i, cell_name) in body_fn.code.cellvars.iter().enumerate() { + if cell_name == "__class__" { + if let Some(cell) = frame.cells.get(i) { + *cell.borrow_mut() = other.clone(); + } + } + } + return Ok(other); } }; // Run `__init__` if a user `__new__` was used (the @@ -6941,14 +9099,24 @@ impl Interpreter { if !is_type_new_sentinel { let _ = new_fn; if let Some(init) = metaclass.lookup("__init__") { + // Only a Python metaclass `__init__` consumes the + // class-creation keywords; the default builtin + // `type.__init__` ignores them (see CPython + // `type_init`), so don't hand it kwargs it rejects. + let init_consumes_kwargs = matches!(init, Object::Function(_)); let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: Object::Type(ty.clone()), function: init, })); + let init_kwargs: &[(String, Object)] = if init_consumes_kwargs { + &subclass_kwargs + } else { + &[] + }; let _ = self.call( &bound, &call_args, - &subclass_kwargs, + init_kwargs, &Rc::new(RefCell::new(DictData::new())), )?; } @@ -6978,6 +9146,14 @@ impl Interpreter { metaclass: Rc, args: &[Object], kwargs: &[(String, Object)], + // CPython splits class creation into `type.__new__` (build the + // object, run `__set_name__`/`__init_subclass__`) and + // `type.__init__`. This helper models the whole `type.__call__` + // when `call_init` is true; when invoked as bare `type.__new__` + // (a metaclass chaining through `super().__new__(...)`) it must + // NOT run `__init__` — `build_class` invokes the metaclass + // `__init__` afterwards with the real class-creation kwargs. + call_init: bool, ) -> Result { let name = match &args[0] { Object::Str(s) => s.to_string(), @@ -7023,9 +9199,17 @@ impl Interpreter { // If we're under a user metaclass, run its `__init__` so it // can mutate the class (member registration in EnumMeta, - // abstract-method tracking in ABCMeta). - if !is_plain_type { + // abstract-method tracking in ABCMeta). Skipped on the bare + // `type.__new__` path (see `call_init`). + if !is_plain_type && call_init { if let Some(init) = metaclass.lookup("__init__") { + // CPython's `type.__init__` accepts and ignores the + // class-creation keywords in its 3-argument form; only a + // user-defined metaclass `__init__` actually consumes + // them. Forward kwargs solely to a Python `__init__` so + // the default builtin doesn't reject them with + // "builtin '__init__' does not accept keyword arguments". + let init_consumes_kwargs = matches!(init, Object::Function(_)); let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: Object::Type(ty.clone()), function: init, @@ -7036,10 +9220,15 @@ impl Interpreter { .map(|b| Object::Type(b.clone())) .collect(), ); + let init_kwargs: &[(String, Object)] = if init_consumes_kwargs { + &subclass_kwargs + } else { + &[] + }; let _ = self.call( &bound, &[Object::from_str(&name), bases_tuple, ns_dict_obj], - &subclass_kwargs, + init_kwargs, &Rc::new(RefCell::new(DictData::new())), )?; } @@ -7070,6 +9259,25 @@ impl Interpreter { } } + // CPython `type.__new__` implicitly wraps a `__new__` written as + // a plain `def` in the class body as a *staticmethod*. Without + // this, looking it up through a `super()` proxy (whose receiver + // is non-`None`) binds the receiver and passes one positional + // argument too many — e.g. `super().__new__(cls, name, bases, ns)` + // inside a metaclass `__new__` raises "takes 4 positional + // arguments but 5 were given". The `instantiate` / `build_class` + // paths already prepend `cls` explicitly, so unwrapping the + // staticmethod there keeps them working unchanged. + { + let key = DictKey(Object::from_static("__new__")); + let current = ty.dict.borrow().get(&key).cloned(); + if let Some(Object::Function(f)) = current { + ty.dict + .borrow_mut() + .insert(key, Object::StaticMethod(Rc::new(Object::Function(f)))); + } + } + // Pull __slots__ out if present. let slots_obj = ty .dict @@ -7150,12 +9358,23 @@ impl Interpreter { receiver: value.clone(), function: hook, })); - let _ = self.call( + let res = self.call( &bound, &[Object::Type(ty.clone()), Object::from_str(&attr_name)], &[], &self.builtins.clone(), - )?; + ); + // PEP 678 / CPython `type.__new__`: a failing + // `__set_name__` is re-raised with a note naming the + // descriptor, attribute, and owning class so the + // traceback points at the offending assignment. + if let Err(RuntimeError::PyException(pe)) = &res { + pe.add_note(format!( + "Error calling __set_name__ on '{}' instance '{}' in '{}'", + inst.class.name, attr_name, ty.name + )); + } + res?; } } } @@ -7207,6 +9426,96 @@ impl Interpreter { Ok(()) } + /// For a *non-builtin* subclass of a value/container built-in + /// (`int`, `float`, `complex`, `str`, `bytes`, `bytearray`, + /// `tuple`, `list`, `set`, `frozenset`, `dict`), build the + /// underlying native payload the instance wraps so the inherited + /// numeric / sequence / mapping protocols keep firing through the + /// subclass — the moral equivalent of CPython storing the C-level + /// value in the object struct. + /// + /// Immutable payloads (and mutable ones for a plain `class C(list): + /// pass` that inherits the default `__init__`) are built eagerly + /// here from the constructor `args`, exactly as `int.__new__` / + /// `tuple.__new__` / `list(...)` would. A mutable subclass that + /// defines its *own* `__init__` (e.g. `collections.Counter`) gets a + /// fresh empty container instead, leaving that `__init__` to fill it + /// via the (now native-aware) item-assignment protocol. + /// + /// Returns `None` for an ordinary `object` subclass. + fn native_for_value_subclass( + &mut self, + cls: &Rc, + args: &[Object], + kwargs: &[(String, Object)], + ) -> Result, RuntimeError> { + if cls.flags.is_builtin { + return Ok(None); + } + let bt = builtin_types(); + let is_strict = |base: &Rc| cls.is_subclass_of(base) && !Rc::ptr_eq(cls, base); + // The only subclass relationship among these built-ins is + // `bool <: int`; `bool` itself is `final` in CPython, so the + // order of the remaining (disjoint) checks is irrelevant. + let (base, mutable): (Rc, bool) = if is_strict(&bt.int_) { + (bt.int_.clone(), false) + } else if is_strict(&bt.float_) { + (bt.float_.clone(), false) + } else if is_strict(&bt.complex_) { + (bt.complex_.clone(), false) + } else if is_strict(&bt.bytearray_) { + (bt.bytearray_.clone(), true) + } else if is_strict(&bt.bytes_) { + (bt.bytes_.clone(), false) + } else if is_strict(&bt.str_) { + (bt.str_.clone(), false) + } else if is_strict(&bt.tuple_) { + (bt.tuple_.clone(), false) + } else if is_strict(&bt.list_) { + (bt.list_.clone(), true) + } else if is_strict(&bt.frozenset_) { + (bt.frozenset_.clone(), false) + } else if is_strict(&bt.set_) { + (bt.set_.clone(), true) + } else if is_strict(&bt.dict_) { + (bt.dict_.clone(), true) + } else { + return Ok(None); + }; + + // A mutable container subclass that supplies its own `__init__` + // owns the filling; hand it a fresh empty container. + if mutable && !init_is_from_object(cls) { + let empty = match base.name.as_str() { + "list" => Object::new_list(Vec::new()), + "set" => Object::new_set_from(Vec::::new()), + "bytearray" => Object::ByteArray(Rc::new(RefCell::new(Vec::new()))), + _ => Object::Dict(Rc::new(RefCell::new(DictData::new()))), + }; + return Ok(Some(empty)); + } + + // Otherwise build the payload eagerly from the constructor args by + // routing through the built-in base's own constructor. When the + // subclass defines its own `__init__`, the surplus constructor + // arguments belong to it: the inherited `__new__` only consumes the + // seed value(s) it understands and ignores the rest (CPython's + // `float`/`int`/`str` `__new__` do exactly this, so + // `class C(float): def __init__(self, a, b=None)` then `C(2.5, b=3)` + // works). + if !init_is_from_object(cls) { + let arity = match base.name.as_str() { + "complex" | "int" => 2, + _ => 1, + }; + let seed = &args[..args.len().min(arity)]; + let native = self.instantiate(base, seed, &[])?; + return Ok(Some(native)); + } + let native = self.instantiate(base, args, kwargs)?; + Ok(Some(native)) + } + /// Allocate an instance of `cls`, then run the `__new__` / /// `__init__` two-phase initialisation. The descriptor protocol /// gives us classmethod binding for `__new__` for free. @@ -7238,6 +9547,20 @@ impl Interpreter { "classmethod" => { return builtins::construct_classmethod(args); } + // `types.MethodType(func, obj)` — bind `func` to `obj`, + // producing a callable bound method (CPython `method`). + "method" => { + if args.len() != 2 { + return Err(type_error(format!( + "method expected 2 arguments, got {}", + args.len() + ))); + } + return Ok(Object::BoundMethod(Rc::new(BoundMethod { + receiver: args[1].clone(), + function: args[0].clone(), + }))); + } _ => {} } // Special-case `list(it)` / `tuple(it)` so generators flow @@ -7259,8 +9582,10 @@ impl Interpreter { // route lazy iterables (generators, `zip`/`map`/`filter` // views, genexprs) through the VM-aware collector — the plain // builtins below can only drive eager containers (RFC 0033). - if matches!(&args.first(), Some(Object::Generator(_) | Object::Iter(_))) - && args.len() == 1 + if matches!( + &args.first(), + Some(Object::Generator(_) | Object::Iter(_) | Object::Instance(_)) + ) && args.len() == 1 && kwargs.is_empty() { if cls.name == "set" || cls.name == "frozenset" { @@ -7289,16 +9614,64 @@ impl Interpreter { return Ok(Object::Dict(Rc::new(RefCell::new(d)))); } } + // `dict(**kw)` / `dict(mapping, **kw)` / `dict(pairs, **kw)`: + // CPython seeds the dict from at most one positional (a mapping + // with `keys()`, else an iterable of key/value pairs) and then + // overlays the keyword arguments. The kwargs-empty cases are + // already handled above / by the plain builtin constructor, so + // this branch only needs to cover the keyword-argument form. + if cls.name == "dict" && !kwargs.is_empty() { + if args.len() > 1 { + return Err(type_error(format!( + "dict expected at most 1 argument, got {}", + args.len() + ))); + } + let global_dummy = Rc::new(RefCell::new(DictData::new())); + let mut d = DictData::new(); + if let Some(arg0) = args.first() { + if let Some(Object::Dict(inner)) = + self.try_dict_from_mapping(arg0, &global_dummy)? + { + d = inner.borrow().clone(); + } else { + let items = self.collect_iterable(arg0, &global_dummy)?; + for (i, pair) in items.into_iter().enumerate() { + let kv = self.collect_iterable(&pair, &global_dummy)?; + if kv.len() != 2 { + return Err(type_error(format!( + "dictionary update sequence element #{i} has length {}; 2 is required", + kv.len() + ))); + } + d.insert(DictKey(kv[0].clone()), kv[1].clone()); + } + } + } + for (k, v) in kwargs { + d.insert(DictKey(Object::Str(Rc::from(k.as_str()))), v.clone()); + } + return Ok(Object::Dict(Rc::new(RefCell::new(d)))); + } // `int(x)` / `float(x)` honour the user's `__int__` / // `__float__` when `x` is a non-primitive — matches CPython. - if cls.name == "int" && args.len() <= 2 && kwargs.is_empty() { + if cls.name == "int" { + let bound = bind_int_args(args, kwargs)?; let global_dummy = Rc::new(RefCell::new(DictData::new())); - return self.do_int_call(args, &global_dummy); + return self.do_int_call(&bound, &global_dummy); } if cls.name == "float" && args.len() <= 1 && kwargs.is_empty() { let global_dummy = Rc::new(RefCell::new(DictData::new())); return self.do_float_call(args, &global_dummy); } + // `complex(real=0, imag=0)` honours __complex__ / __float__ on + // instances and unwraps built-in numeric subclasses. `real`/`imag` + // are positional-or-keyword, so bind keyword forms too. + if cls.name == "complex" && (!args.is_empty() || !kwargs.is_empty()) { + let bound = bind_complex_args(args, kwargs)?; + let global_dummy = Rc::new(RefCell::new(DictData::new())); + return self.do_complex_call(&bound, &global_dummy); + } // `bool(x)` must consult __bool__/__len__ for instances. if cls.name == "bool" && args.len() <= 1 && kwargs.is_empty() { let global_dummy = Rc::new(RefCell::new(DictData::new())); @@ -7343,6 +9716,48 @@ impl Interpreter { } } + // PEP 3119: refuse to instantiate a class that still carries + // unimplemented abstract methods. CPython enforces this inside + // `object.__new__` (gated on `Py_TPFLAGS_IS_ABSTRACT`); we check + // the observable `__abstractmethods__` set instead so the rule + // holds for `abc.ABCMeta` / `_py_abc.ABCMeta` and any metaclass + // that populates it. + if let Some(abstracts) = cls.lookup("__abstractmethods__") { + let mut names: Vec = Vec::new(); + match &abstracts { + Object::FrozenSet(s) => { + for k in s.iter() { + if let Object::Str(name) = &k.0 { + names.push(name.to_string()); + } + } + } + Object::Set(s) => { + for k in s.borrow().iter() { + if let Object::Str(name) = &k.0 { + names.push(name.to_string()); + } + } + } + _ => {} + } + if !names.is_empty() { + names.sort(); + let joined = names + .iter() + .map(|n| format!("'{n}'")) + .collect::>() + .join(", "); + return Err(type_error(format!( + "Can't instantiate abstract class {} without an implementation \ + for abstract method{} {}", + cls.name, + if names.len() == 1 { "" } else { "s" }, + joined, + ))); + } + } + // `__new__` chain: walk the MRO; the first base that defines a // user `__new__` (other than the implicit `object.__new__`) // owns instance allocation. If none is found, fall back to the @@ -7375,7 +9790,42 @@ impl Interpreter { )? } _ => { - let inst = Object::Instance(Rc::new(PyInstance::new(cls.clone()))); + // Subclasses of the descriptor built-ins + // (`property` / `classmethod` / `staticmethod`) stash the + // wrapped descriptor in `native`, exactly as `int` + // subclasses stash their value, so the descriptor + // protocol keeps firing through the subclass. This is + // what makes abc's deprecated `abstractproperty` / + // `abstractclassmethod` / `abstractstaticmethod` (each a + // subclass of the matching built-in) behave like real + // descriptors. + let bt = builtin_types(); + let native_desc: Option = if cls.flags.is_builtin { + None + } else if cls.is_subclass_of(&bt.property_) + && !Rc::ptr_eq(&cls, &bt.property_) + { + Some(builtins::construct_property(args)?) + } else if cls.is_subclass_of(&bt.classmethod_) + && !Rc::ptr_eq(&cls, &bt.classmethod_) + { + Some(builtins::construct_classmethod(args)?) + } else if cls.is_subclass_of(&bt.staticmethod_) + && !Rc::ptr_eq(&cls, &bt.staticmethod_) + { + Some(builtins::construct_staticmethod(args)?) + } else { + // Subclasses of the value/container built-ins + // (`class C(list)`, `class C(int)`, …) wrap a native + // payload so the inherited protocols keep working. + self.native_for_value_subclass(&cls, args, kwargs)? + }; + let inst = match native_desc { + Some(desc) => { + Object::Instance(Rc::new(PyInstance::with_native(cls.clone(), desc))) + } + None => Object::Instance(Rc::new(PyInstance::new(cls.clone()))), + }; // RFC 0024: auto-track every fresh user instance with // the cycle collector. CPython does the same for any // type whose `tp_traverse` is non-NULL — for us that's @@ -7450,6 +9900,18 @@ impl Interpreter { if let Some(first) = args.first() { dict.insert(DictKey(Object::from_static("message")), first.clone()); } + // CPython's `BaseException` always exposes these slots (default + // None/None/False/None), so attribute access and exception-context + // chain walks (e.g. `contextlib._fix_exception_context`, which reads + // `exc.__context__` of every link) never raise `AttributeError`, + // even on an exception that was constructed but never raised/chained. + dict.insert(DictKey(Object::from_static("__context__")), Object::None); + dict.insert(DictKey(Object::from_static("__cause__")), Object::None); + dict.insert( + DictKey(Object::from_static("__suppress_context__")), + Object::Bool(false), + ); + dict.insert(DictKey(Object::from_static("__traceback__")), Object::None); drop(dict); Object::Instance(Rc::new(inst)) } @@ -7457,6 +9919,12 @@ impl Interpreter { /// Look up the existing built-in callable that mirrors `cls`'s /// constructor — `int`, `range`, `list`, etc. fn builtin_constructor_for(&self, cls: &TypeObject) -> Option> { + // Core types resolve to their native constructor independently of the + // `__builtins__` dict, which now exposes the real `type` objects + // rather than the bare-function constructors. + if let Some(ctor) = crate::builtins::builtin_type_constructor(&cls.name) { + return Some(ctor); + } let key = DictKey(Object::from_str(&cls.name)); match self.builtins.borrow().get(&key).cloned() { Some(Object::Builtin(b)) => Some(b), @@ -7504,9 +9972,26 @@ impl Interpreter { positional[star_idx] = Object::new_tuple(rest); filled[star_idx] = true; } else if provided > total_args { + // Mirror CPython's `too_many_positional`: when the callable + // has positional defaults the count is a range ("from MIN to + // MAX"); pluralise "argument" and pick "was"/"were" exactly as + // CPython does so error-message assertions match. + let defcount = f.defaults.len(); + let min = total_args.saturating_sub(defcount); + let sig = if defcount > 0 { + format!("from {min} to {total_args}") + } else { + format!("{total_args}") + }; + let plural = if defcount > 0 || total_args != 1 { + "s" + } else { + "" + }; + let given_verb = if provided == 1 { "was" } else { "were" }; return Err(type_error(format!( - "{}() takes {} positional arguments but {} were given", - f.name, total_args, provided + "{}() takes {} positional argument{} but {} {} given", + f.name, sig, plural, provided, given_verb ))); } // Keyword args: match by name. Unmatched ones go into the @@ -7518,15 +10003,19 @@ impl Interpreter { // addressed by keyword. Locals beyond it MUST NOT pull the // kwarg out of the **kwargs catchall. let mut extra_kwargs = crate::object::DictData::new(); + // Positional-only parameters occupy `[0, posonly)` and CANNOT be + // addressed by keyword (PEP 570): a keyword matching such a name + // flows into `**kwargs` instead, or — absent `**kwargs` — raises + // the dedicated "positional-only ... passed as keyword" error. + let posonly = code.posonly_count as usize; for (name, value) in kwargs { let mut slot = None; if let Some(p) = code .varnames - .iter() - .take(total_args) - .position(|n| n == name) + .get(posonly..total_args) + .and_then(|range| range.iter().position(|n| n == name)) { - slot = Some(p); + slot = Some(posonly + p); } else if let Some(p) = code .varnames .get(kwonly_start..kwonly_end) @@ -7551,6 +10040,12 @@ impl Interpreter { crate::object::DictKey(Object::from_str(name.clone())), value.clone(), ); + } else if code.varnames.iter().take(posonly).any(|n| n == name) { + return Err(type_error(format!( + "{}() got some positional-only arguments passed as \ + keyword arguments: '{}'", + f.name, name + ))); } else { return Err(type_error(format!( "{}() got an unexpected keyword argument '{}'", @@ -7578,17 +10073,48 @@ impl Interpreter { } } } - // Then plug kwonly defaults by name. - for (name, default) in &f.kw_defaults { - if let Some(p) = code - .varnames - .get(kwonly_start..kwonly_end) - .and_then(|range| range.iter().position(|n| n == name)) - { - let slot = kwonly_start + p; - if !filled[slot] { - positional[slot] = default.clone(); - filled[slot] = true; + // Then plug kwonly defaults by name. Guarded on `kwonly_count` + // so the overwhelmingly common no-keyword-only call skips this + // entirely (no per-call attrs probe). A user-assigned + // `__kwdefaults__` (stored on the function's attrs dict) replaces + // the compiled set wholesale — CPython's `func.__kwdefaults__ = + // {...}` makes any keyword-only name absent from the new mapping + // required again. Only the override path allocates; otherwise we + // borrow the compiled `kw_defaults` directly. + if kwonly_count > 0 { + let kwd_override = f + .attrs + .borrow() + .get(&DictKey(Object::from_static("__kwdefaults__"))) + .cloned(); + let overridden: Option> = match kwd_override { + Some(Object::Dict(d)) => Some( + d.borrow() + .iter() + .filter_map(|(k, v)| match &k.0 { + Object::Str(s) => Some((s.to_string(), v.clone())), + _ => None, + }) + .collect(), + ), + Some(Object::None) => Some(Vec::new()), + _ => None, + }; + let kw_defaults_iter: &[(String, Object)] = match &overridden { + Some(v) => v, + None => &f.kw_defaults, + }; + for (name, default) in kw_defaults_iter { + if let Some(p) = code + .varnames + .get(kwonly_start..kwonly_end) + .and_then(|range| range.iter().position(|n| n == name)) + { + let slot = kwonly_start + p; + if !filled[slot] { + positional[slot] = default.clone(); + filled[slot] = true; + } } } } @@ -7911,6 +10437,144 @@ impl Interpreter { } } + /// Parse `source`, replaying any tokenizer-collected invalid-escape + /// `SyntaxWarning`s through the `warnings` machinery, then map a parse + /// failure to a located `SyntaxError`. The single funnel for the + /// `compile`/`eval`/`exec` and module-import front ends so escape + /// diagnostics behave identically everywhere (CPython parity). + fn parse_source_emitting_warnings( + &mut self, + source: &str, + filename: &str, + ) -> Result { + let (parsed, warnings) = weavepy_parser::parse_module_with_warnings(source); + // Emit warnings first: under `simplefilter('always')` they are + // recorded and a later parse error still propagates; under + // `simplefilter('error')` the first escape escalates to a + // SyntaxError that preempts the parse error — both match CPython. + self.emit_escape_warnings(source, filename, &warnings)?; + parsed.map_err(|e| parse_error_to_syntax_error(&e, source, filename)) + } + + /// Replay tokenizer-collected invalid-escape diagnostics (CPython's + /// `SyntaxWarning`s for unrecognised and oversized-octal escapes) + /// through the runtime `warnings` machinery, attributing each to its + /// source `(filename, lineno)`. Under an active `error` filter the + /// warning is raised; we convert it into a located `SyntaxError` + /// (carrying the backslash's `offset`), exactly as CPython's compiler + /// does. A no-op when there are no diagnostics or no `warnings` module. + fn emit_escape_warnings( + &mut self, + source: &str, + filename: &str, + warnings: &[weavepy_parser::EscapeWarning], + ) -> Result<(), RuntimeError> { + if warnings.is_empty() { + return Ok(()); + } + let Some(warn_explicit) = self.warnings_warn_explicit() else { + return Ok(()); + }; + let syntax_warning_ty = crate::builtin_types::builtin_types().syntax_warning.clone(); + let syntax_warning = Object::Type(syntax_warning_ty.clone()); + let globals = self.builtins.clone(); + for w in warnings { + let (lineno, offset, text) = line_col_text(source, w.offset); + let args = [ + Object::from_str(w.message.clone()), + syntax_warning.clone(), + Object::from_str(filename.to_owned()), + Object::Int(i64::from(lineno)), + ]; + if let Err(e) = self.call(&warn_explicit, &args, &[], &globals) { + if let RuntimeError::PyException(pe) = &e { + if let Object::Instance(inst) = &pe.instance { + if inst.class.is_subclass_of(&syntax_warning_ty) { + return Err(crate::error::syntax_error_located( + w.message.clone(), + Some(filename), + Some(lineno), + Some(offset), + Some(&text), + )); + } + } + } + return Err(e); + } + } + Ok(()) + } + + /// Resolve `warnings.warn_explicit`, importing the module on demand. + /// Returns `None` if the module is unavailable (so callers degrade to + /// silently dropping the diagnostics rather than failing the compile). + fn warnings_warn_explicit(&mut self) -> Option { + let module = self.import_path("warnings").ok()?; + if let Object::Module(m) = module { + return m + .dict + .borrow() + .get(&DictKey(Object::from_static("warn_explicit"))) + .cloned(); + } + None + } + + /// Import `module` and fetch one of its top-level attributes by name. + /// Returns `None` if the module can't be imported or lacks the name. + fn module_attr(&mut self, module: &str, attr: &str) -> Option { + let m = self.import_path(module).ok()?; + if let Object::Module(m) = m { + return m + .dict + .borrow() + .get(&DictKey(Object::from_str(attr))) + .cloned(); + } + None + } + + /// `object.__reduce_ex__(self, protocol)`. Honours a user-defined + /// `__reduce__` override (CPython semantics) and otherwise produces the + /// default `copyreg`-based reduction so `copy`/`pickle` can rebuild the + /// instance. + fn object_reduce_ex( + &mut self, + recv: &Object, + proto: i64, + globals: &Rc>, + ) -> Result { + let cls = crate::builtins::class_of(recv); + if let Some(reduce) = cls.lookup("__reduce__") { + let is_default = + matches!(&reduce, Object::Builtin(b) if b.name == ".object_reduce"); + if !is_default { + let bound = Object::BoundMethod(Rc::new(BoundMethod { + receiver: recv.clone(), + function: reduce, + })); + return self.call(&bound, &[], &[], globals); + } + } + self.object_default_reduce(recv, proto, globals) + } + + /// The default object reduction, delegated to the verbatim-ported + /// `copyreg._reduce_newobj` so the (subtle) protocol-2+ rules live in + /// one place. + fn object_default_reduce( + &mut self, + recv: &Object, + proto: i64, + globals: &Rc>, + ) -> Result { + let helper = self + .module_attr("copyreg", "_reduce_newobj") + .ok_or_else(|| runtime_error("copyreg._reduce_newobj unavailable"))?; + self.call(&helper, &[recv.clone(), Object::Int(proto)], &[], globals) + } + fn do_compile_call( &mut self, args: &[Object], @@ -7944,30 +10608,26 @@ impl Interpreter { ); match mode.as_str() { "exec" => { - let module = weavepy_parser::parse_module(&source) - .map_err(|e| crate::error::value_error(format!("compile error: {e}")))?; + let module = self.parse_source_emitting_warnings(&source, &filename)?; let code = weavepy_compiler::compile_module_with_source(&module, &source, &filename) - .map_err(|e| crate::error::value_error(format!("compile error: {e}")))?; + .map_err(|e| crate::error::syntax_error(e.to_string()))?; Ok(Object::Code(Rc::new(code))) } "eval" => { - let module = weavepy_parser::parse_module(&source) - .map_err(|e| crate::error::value_error(format!("compile error: {e}")))?; - let code = - weavepy_compiler::compile_module_with_source(&module, &source, &filename) - .map_err(|e| crate::error::value_error(format!("compile error: {e}")))?; + let module = self.parse_source_emitting_warnings(&source, &filename)?; + let code = weavepy_compiler::compile_eval_with_source(&module, &source, &filename) + .map_err(|e| crate::error::syntax_error(e.to_string()))?; Ok(Object::Code(Rc::new(code))) } // Interactive mode: top-level expression statements echo // through `sys.displayhook` (`PrintExpr`). Powers the REPL, // `code`/`codeop`, and `doctest`'s example execution. "single" => { - let module = weavepy_parser::parse_module(&source) - .map_err(|e| crate::error::value_error(format!("compile error: {e}")))?; + let module = self.parse_source_emitting_warnings(&source, &filename)?; let code = weavepy_compiler::compile_interactive_with_source(&module, &source, &filename) - .map_err(|e| crate::error::value_error(format!("compile error: {e}")))?; + .map_err(|e| crate::error::syntax_error(e.to_string()))?; Ok(Object::Code(Rc::new(code))) } other => Err(crate::error::value_error(format!( @@ -7999,11 +10659,14 @@ impl Interpreter { let code_rc = match source { Object::Code(c) => c, Object::Str(src) => { - let module = weavepy_parser::parse_module(&src) - .map_err(|e| crate::error::value_error(format!("exec error: {e}")))?; + // A malformed source must surface as `SyntaxError` (with a + // location), exactly like `compile()` — CPython's `exec` + // never raises `ValueError` for bad syntax. Invalid-escape + // `SyntaxWarning`s replay here too. + let module = self.parse_source_emitting_warnings(&src, "")?; let compiled = weavepy_compiler::compile_module_with_source(&module, &src, "") - .map_err(|e| crate::error::value_error(format!("exec error: {e}")))?; + .map_err(|e| crate::error::syntax_error(e.to_string()))?; Rc::new(compiled) } other => { @@ -8048,12 +10711,60 @@ impl Interpreter { Some(Object::None) | None => outer_globals.clone(), _ => return Err(type_error("eval() globals must be a dict")), }; + // Resolve the locals namespace. CPython: an explicit mapping is + // used directly; when omitted/None, name resolution falls back to + // the *calling frame's* live locals (so `eval("args[1]")` inside a + // function sees that function's `args`). + let locals_dict: Option>> = match args.get(2) { + Some(Object::Dict(d)) => Some(d.clone()), + Some(Object::None) | None => { + let caller = self.frame_stack.borrow().last().cloned(); + caller.and_then(|f| { + f.invalidate_locals(); + match f.locals() { + Object::Dict(d) => Some(d), + _ => None, + } + }) + } + _ => return Err(type_error("eval() locals must be a mapping")), + }; + // Build the execution namespace. When distinct locals are present, + // run in a *combined* snapshot (globals overlaid with locals) so + // bare-name lookups resolve locals first without mutating either + // caller dict. Otherwise run directly in globals (current path). + let use_combined = match &locals_dict { + Some(l) => !Rc::ptr_eq(l, &globals_dict), + None => false, + }; + let ns = if use_combined { + let combined = Rc::new(RefCell::new(globals_dict.borrow().clone())); + if let Some(l) = &locals_dict { + let mut c = combined.borrow_mut(); + for (k, v) in l.borrow().iter() { + c.insert(k.clone(), v.clone()); + } + } + combined + } else { + globals_dict.clone() + }; + if !ns + .borrow() + .contains_key(&DictKey(Object::from_static("__builtins__"))) + { + ns.borrow_mut().insert( + DictKey(Object::from_static("__builtins__")), + Object::Dict(self.builtins.clone()), + ); + } let src = match source { Object::Code(c) => { - let mut frame = - self.make_frame(c, Vec::new(), Vec::new(), globals_dict.clone(), true); - self.run_frame(&mut frame)?; - return Ok(Object::None); + // `eval`-mode code returns its expression value (see + // `compile_eval_with_source`); run it in the combined + // namespace and hand that value straight back. + let mut frame = self.make_frame(c, Vec::new(), Vec::new(), ns.clone(), true); + return self.run_frame(&mut frame); } Object::Str(s) => s.to_string(), other => { @@ -8063,40 +10774,19 @@ impl Interpreter { ))) } }; - // Wrap as a single-expression module: parse the source, then - // synthesize a `_result_ = ` statement so the value is - // captured in the globals dict. - let wrapped = format!("__weavepy_eval_result = ({})\n", src); - let module = weavepy_parser::parse_module(&wrapped) - .map_err(|e| crate::error::value_error(format!("eval error: {e}")))?; - let code = weavepy_compiler::compile_module_with_source(&module, &wrapped, "") - .map_err(|e| crate::error::value_error(format!("eval error: {e}")))?; - { - let mut g = globals_dict.borrow_mut(); - if !g.contains_key(&DictKey(Object::from_static("__builtins__"))) { - g.insert( - DictKey(Object::from_static("__builtins__")), - Object::Dict(self.builtins.clone()), - ); - } - } - let mut frame = self.make_frame( - Rc::new(code), - Vec::new(), - Vec::new(), - globals_dict.clone(), - true, - ); - self.run_frame(&mut frame)?; - let result = globals_dict - .borrow() - .get(&DictKey(Object::from_static("__weavepy_eval_result"))) - .cloned() - .unwrap_or(Object::None); - globals_dict - .borrow_mut() - .shift_remove(&DictKey(Object::from_static("__weavepy_eval_result"))); - Ok(result) + // `eval` evaluates a single expression. CPython tolerates leading + // whitespace/newlines in the source, so trim them, then compile in + // *eval mode* — the resulting code object yields the expression's + // value, which `run_frame` hands straight back. A malformed source + // must raise a located `SyntaxError` (matching `compile`), never a + // `ValueError`; this is what `test_fstring`'s negative cases (which + // call `eval("f'...'")` and assert `SyntaxError`) rely on. + let trimmed = src.trim_start_matches([' ', '\t', '\n', '\r', '\x0c']); + let module = self.parse_source_emitting_warnings(trimmed, "")?; + let code = weavepy_compiler::compile_eval_with_source(&module, trimmed, "") + .map_err(|e| crate::error::syntax_error(e.to_string()))?; + let mut frame = self.make_frame(Rc::new(code), Vec::new(), Vec::new(), ns.clone(), true); + self.run_frame(&mut frame) } fn do_import( @@ -8146,7 +10836,13 @@ impl Interpreter { } } } - Ok(leaf) + // Re-read sys.modules: a module may replace itself during + // execution (e.g. `decimal` rebinds to `_pydecimal` via + // `sys.modules[__name__] = _pydecimal`). CPython resolves the + // fromlist source from sys.modules, so `from decimal import + // Decimal` must see the replacement, not the husk module object + // that `import_path` first created. + Ok(self.cache.get(&absolute).unwrap_or(leaf)) } /// Walk a dotted name (`a.b.c`), loading each part lazily and @@ -8263,10 +10959,16 @@ impl Interpreter { is_package: bool, filename: &str, ) -> Result { - let module = weavepy_parser::parse_module(source) - .map_err(|e| import_error(format!("parse error in '{full}': {e}")))?; + // A syntax error inside an imported module is a `SyntaxError`, + // not an `ImportError` (CPython parity). Frozen modules have no + // on-disk path, so the diagnostic filename is the leaf name + `.py` + // — matching the basename CPython would show for the same file. + let module = weavepy_parser::parse_module(source).map_err(|e| { + let diag = format!("{}.py", full.rsplit('.').next().unwrap_or(full)); + parse_error_to_syntax_error(&e, source, &diag) + })?; let code = weavepy_compiler::compile_module_with_source(&module, source, filename) - .map_err(|e| import_error(format!("compile error in '{full}': {e}")))?; + .map_err(|e| crate::error::syntax_error(e.to_string()))?; // RFC 0021 — populate the process-global frozen cache so the // *next* interpreter in this process skips parse + compile. // We cache only the compiled code, never the running module @@ -8349,9 +11051,9 @@ impl Interpreter { let source = std::fs::read_to_string(path) .map_err(|e| import_error(format!("failed to read '{}': {e}", path.display())))?; let module = weavepy_parser::parse_module(&source) - .map_err(|e| import_error(format!("parse error in '{}': {e}", path.display())))?; + .map_err(|e| parse_error_to_syntax_error(&e, &source, &filename))?; let code = weavepy_compiler::compile_module_with_source(&module, &source, &filename) - .map_err(|e| import_error(format!("compile error in '{}': {e}", path.display())))?; + .map_err(|e| crate::error::syntax_error(e.to_string()))?; if !self.bytecode_writes_disabled() { crate::pycache::try_write(path, &code); } @@ -8481,6 +11183,48 @@ impl Interpreter { } } +/// `(1-based line, 1-based column, line text without newline)` for a byte +/// offset into `source`. Drives `SyntaxError` `lineno`/`offset`/`text`. +fn line_col_text(source: &str, byte: u32) -> (u32, u32, String) { + let byte = (byte as usize).min(source.len()); + let mut line_start = 0usize; + let mut line = 1u32; + for (i, ch) in source.char_indices() { + if i >= byte { + break; + } + if ch == '\n' { + line += 1; + line_start = i + 1; + } + } + let line_end = source[line_start..] + .find('\n') + .map_or(source.len(), |off| line_start + off); + let col = source[line_start..byte].chars().count() as u32 + 1; + (line, col, source[line_start..line_end].to_owned()) +} + +/// Map a [`weavepy_parser::ParseError`] to a CPython-shaped `SyntaxError`, +/// computing the line/column/text from the error's byte offset. CPython +/// raises `SyntaxError` (not `ValueError`/`ImportError`) from both +/// `compile()` and the import machinery, with `.msg`/`.filename`/`.lineno`/ +/// `.offset` populated. +fn parse_error_to_syntax_error( + err: &weavepy_parser::ParseError, + source: &str, + filename: &str, +) -> RuntimeError { + let (lineno, offset, text) = line_col_text(source, err.byte_offset()); + crate::error::syntax_error_located( + err.syntax_message(), + Some(filename), + Some(lineno), + Some(offset), + Some(&text), + ) +} + /// Read the current module's `__package__` (or fall back to /// `__name__`'s parent) so relative imports can resolve. fn current_package(globals: &Rc>) -> Option { @@ -8694,7 +11438,7 @@ fn apply_slice_deletion(data: &mut Vec, s: &PySlice) -> Result<(), Runti Ok(()) } -fn slice_seq(seq: &[Object], s: &PySlice) -> Result, RuntimeError> { +pub(crate) fn slice_seq(seq: &[Object], s: &PySlice) -> Result, RuntimeError> { let len = seq.len() as i64; let step = match &s.step { Object::None => 1i64, @@ -8792,7 +11536,7 @@ fn path_contains(path: &[Object], needle: &str) -> bool { .any(|o| matches!(o, Object::Str(s) if s.as_ref() == needle)) } -fn normalize_index(i: i64, len: usize) -> Result { +pub(crate) fn normalize_index(i: i64, len: usize) -> Result { let n = len as i64; let idx = if i < 0 { i + n } else { i }; if idx < 0 || idx >= n { @@ -8801,6 +11545,16 @@ fn normalize_index(i: i64, len: usize) -> Result { Ok(idx as usize) } +/// Map a `bool` to the equivalent `Int` (`True`→1, `False`→0), leaving any +/// other object untouched. `bool` is an `int` subclass in Python, so a bool +/// used as a sequence index must act as 0/1. +fn normalize_bool_index(n: Object) -> Object { + match n { + Object::Bool(b) => Object::Int(i64::from(b)), + other => other, + } +} + /// Outcome of executing a single instruction. enum StepOutcome { Continue, @@ -8853,7 +11607,7 @@ fn resolve_metaclass( Ok(winner) } -fn instance_method(obj: &Object, name: &str) -> Option { +pub(crate) fn instance_method(obj: &Object, name: &str) -> Option { let inst = match obj { Object::Instance(i) => i.clone(), _ => return None, @@ -9015,14 +11769,151 @@ fn symmetric_diff_sets(a: &crate::object::SetData, b: &crate::object::SetData) - Object::Set(Rc::new(RefCell::new(out))) } +/// Which built-in types actually implement the format mini-language. Only +/// `str` and the numeric types override `__format__`; for everything else +/// `object.__format__` accepts an *empty* spec (returning `str(self)`) but +/// rejects any non-empty spec with a `TypeError`. +fn supports_format_spec(value: &Object) -> bool { + matches!( + value, + Object::Str(_) + | Object::Int(_) + | Object::Long(_) + | Object::Bool(_) + | Object::Float(_) + | Object::Complex(_) + ) +} + +/// The `TypeError` CPython's `object.__format__` raises for a non-empty spec. +fn unsupported_format_string(value: &Object) -> RuntimeError { + type_error(format!( + "unsupported format string passed to {}.__format__", + value.type_name() + )) +} + +/// Bind `complex(real=0, imag=0)` arguments (positional-or-keyword) into +/// the positional `[real]` / `[real, imag]` vector that `do_complex_call` +/// consumes. Mirrors CPython's `complex_new` keyword handling: at most two +/// positional args, `real`/`imag` keywords (rejecting duplicates and +/// unknown names), with `real` defaulting to `0` when only `imag` is given. +fn bind_complex_args( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result, RuntimeError> { + if args.len() > 2 { + return Err(type_error(format!( + "complex() takes at most 2 arguments ({} given)", + args.len() + ))); + } + let mut real = args.first().cloned(); + let mut imag = args.get(1).cloned(); + for (k, v) in kwargs { + match k.as_str() { + "real" => { + if real.is_some() { + return Err(type_error( + "argument for complex() given by name ('real') and position (1)", + )); + } + real = Some(v.clone()); + } + "imag" => { + if imag.is_some() { + return Err(type_error( + "argument for complex() given by name ('imag') and position (2)", + )); + } + imag = Some(v.clone()); + } + other => { + return Err(type_error(format!( + "complex() got an unexpected keyword argument '{other}'" + ))); + } + } + } + Ok(match (real, imag) { + (None, None) => vec![], + (Some(r), None) => vec![r], + (r, Some(i)) => vec![r.unwrap_or(Object::Int(0)), i], + }) +} + +/// Bind `int(x=0, /, base=10)` arguments. The value is positional-only, so +/// only `base` is accepted as a keyword (CPython: `int(x=1)` raises +/// "'x' is an invalid keyword argument for int()"). Returns the positional +/// argument vector for `do_int_call` (`[]`, `[value]`, or `[value, base]`). +fn bind_int_args( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result, RuntimeError> { + if args.len() > 2 { + return Err(type_error(format!( + "int() takes at most 2 arguments ({} given)", + args.len() + ))); + } + let mut base_kw: Option = None; + for (k, v) in kwargs { + if k == "base" { + base_kw = Some(v.clone()); + } else { + return Err(type_error(format!( + "'{k}' is an invalid keyword argument for int()" + ))); + } + } + let mut out: Vec = Vec::new(); + if let Some(v) = args.first() { + out.push(v.clone()); + } + match (args.get(1).cloned(), base_kw) { + (Some(_), Some(_)) => { + return Err(type_error( + "argument for int() given by name ('base') and position (2)", + )); + } + (Some(b), None) | (None, Some(b)) => { + if out.is_empty() { + // `int(base=10)` — base supplied without a value. + return Err(type_error("int() missing string argument")); + } + out.push(b); + } + (None, None) => {} + } + Ok(out) +} + /// Public entry point shared with the `format` builtin: drive the /// format-spec mini-language without going through `FORMAT_VALUE`. pub(crate) fn format_via_spec(value: &Object, spec: &str) -> Result { + format_via_spec_impl(value, spec, false) +} + +/// `format_via_spec` variant used by the printf (`%`) engine, where a +/// precision on an integer code means "minimum digits" (CPython's +/// `str.format` mini-language forbids it; printf mandates it). +fn format_via_spec_percent(value: &Object, spec: &str) -> Result { + format_via_spec_impl(value, spec, true) +} + +fn format_via_spec_impl( + value: &Object, + spec: &str, + allow_int_precision: bool, +) -> Result { let plain = value.to_str(); if spec.is_empty() { return Ok(plain); } - apply_format_spec(value, spec, &plain) + if !supports_format_spec(value) { + return Err(unsupported_format_string(value)); + } + apply_format_spec_inner(value, spec, &plain, allow_int_precision) } /// Public wrapper for `ascii()`. @@ -9372,45 +12263,65 @@ fn apply_trailer(value: Object, trailer: &str) -> Result { /// Apply Python's `%` formatting (`'%s %d' % (x, y)`, or `'%(k)s' % /// {'k': v}`). -/// Map ``bytes %`` arguments through a latin-1 decoding so that the -/// shared ``percent_format`` engine can substitute them as if they -/// were strings. The output is re-encoded back to bytes by the -/// caller so opaque byte values round-trip unchanged. -fn bytes_percent_args(value: &Object) -> Object { - fn map_one(v: &Object) -> Object { - match v { - Object::Bytes(b) => { - let s: String = b.iter().map(|byte| *byte as char).collect(); - Object::from_str(s) - } - Object::ByteArray(cell) => { - let b = cell.borrow().clone(); - let s: String = b.iter().map(|byte| *byte as char).collect(); - Object::from_str(s) - } - _ => v.clone(), - } - } - match value { - Object::Tuple(items) => { - let mapped: Vec = items.iter().map(map_one).collect(); - Object::Tuple(Rc::from(mapped)) - } - Object::Dict(d) => { - let src = d.borrow(); - let mut out: crate::object::DictData = indexmap::IndexMap::new(); - for (k, v) in src.iter() { - out.insert(k.clone(), map_one(v)); - } - Object::Dict(Rc::new(RefCell::new(out))) - } - other => map_one(other), - } +/// Which `%`-formatting flavour the engine is running: `str % args` or +/// PEP 461 `bytes % args` (also `bytearray`). The two differ in their +/// conversion set (`%b` is bytes-only, `%c` inserts a byte vs. a char) and +/// in error wording ("string" vs "bytes" formatting). +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum PercentMode { + Str, + Bytes, } pub(crate) fn percent_format(template: &str, value: &Object) -> Result { let mut noop = |_: &Object, _: char| Ok(None); - percent_format_with(template, value, &mut noop) + percent_format_with(template, value, PercentMode::Str, &mut noop) +} + +/// `true` for any value `%d`/`%f` accept as a real number. +fn percent_is_real(o: &Object) -> bool { + matches!( + o, + Object::Int(_) | Object::Bool(_) | Object::Long(_) | Object::Float(_) + ) || (matches!(o, Object::Instance(_)) && o.as_f64().is_some()) +} + +/// `true` for any value the integer presentation types (`%x`/`%o`/`%b`) +/// accept — ints and int subclasses, but not floats. +fn percent_is_int(o: &Object) -> bool { + matches!(o, Object::Int(_) | Object::Bool(_) | Object::Long(_)) + || (matches!(o, Object::Instance(_)) && o.as_i64().is_some()) +} + +/// CPython's `unsupported format character '%c' (0x%x) at index %zd` +/// (non-printable chars render as `?` but keep their real codepoint). +fn unsupported_format_char(c: char, index: usize) -> RuntimeError { + let disp = if (0x20..=0x7e).contains(&(c as u32)) { + c + } else { + '?' + }; + value_error(format!( + "unsupported format character '{disp}' (0x{:x}) at index {index}", + c as u32 + )) +} + +/// Pull the integer value of a `*` width/precision argument in printf-style +/// formatting (`%*d`, `%.*f`). CPython requires an `int` and raises +/// `TypeError: * wants int` otherwise; an int too large for a C `int` raises +/// `OverflowError` at the call site. +fn star_arg_int(v: &Object) -> Result { + match v { + Object::Bool(b) => Ok(i64::from(*b)), + Object::Int(n) => Ok(*n), + Object::Long(_) => v + .as_i64() + .ok_or_else(|| overflow_error("Python int too large to convert to C int")), + // An `int` subclass instance (e.g. `IntEnum`) unwraps via `as_i64`; + // anything else is not an int. + _ => v.as_i64().ok_or_else(|| type_error("* wants int")), + } } /// Printf-style `%` formatting with a VM-supplied `resolve` callback. @@ -9421,6 +12332,7 @@ pub(crate) fn percent_format(template: &str, value: &Object) -> Result Result, RuntimeError>, ) -> Result { let mut out = String::new(); @@ -9468,26 +12380,77 @@ pub(crate) fn percent_format_with( i += 1; } let mut width = String::new(); - while i < bytes.len() && bytes[i].is_ascii_digit() { - width.push(bytes[i] as char); + if i < bytes.len() && bytes[i] == b'*' { + // Dynamic width: consume the next positional arg as an int. i += 1; + let v = positional + .get(idx) + .cloned() + .ok_or_else(|| type_error("not enough arguments for format string"))?; + idx += 1; + let n = star_arg_int(&v)?; + if !(i64::from(i32::MIN)..=i64::from(i32::MAX)).contains(&n) { + return Err(overflow_error("Python int too large to convert to C int")); + } + // A negative `*` width left-justifies in abs(width), like `-`. + if n < 0 { + flags.push('-'); + width = (-n).to_string(); + } else { + width = n.to_string(); + } + } else { + while i < bytes.len() && bytes[i].is_ascii_digit() { + width.push(bytes[i] as char); + i += 1; + } } let mut precision: Option = None; if i < bytes.len() && bytes[i] == b'.' { i += 1; - let mut p = String::new(); - while i < bytes.len() && bytes[i].is_ascii_digit() { - p.push(bytes[i] as char); + if i < bytes.len() && bytes[i] == b'*' { + // Dynamic precision: consume the next positional arg. i += 1; + let v = positional + .get(idx) + .cloned() + .ok_or_else(|| type_error("not enough arguments for format string"))?; + idx += 1; + let n = star_arg_int(&v)?; + if n > i64::from(i32::MAX) { + return Err(overflow_error("Python int too large to convert to C int")); + } + // CPython treats a negative `*` precision as unspecified. + if n >= 0 { + precision = Some(n.to_string()); + } + } else { + let mut p = String::new(); + while i < bytes.len() && bytes[i].is_ascii_digit() { + p.push(bytes[i] as char); + i += 1; + } + precision = Some(p); } - precision = Some(p); } if i >= bytes.len() { return Err(value_error("incomplete format")); } + // Codepoint index of the conversion char, for error messages. + let kind_index = template[..i].chars().count(); let kind = bytes[i] as char; i += 1; + // `%%` is a literal percent only when the two `%` are adjacent; + // any intervening flag/width/precision/mapping makes the second + // `%` an (invalid) conversion character, e.g. `'% %s'`. + let had_modifier = mapping_key.is_some() + || !flags.is_empty() + || !width.is_empty() + || precision.is_some(); if kind == '%' { + if had_modifier { + return Err(unsupported_format_char('%', kind_index)); + } out.push('%'); continue; } @@ -9509,33 +12472,37 @@ pub(crate) fn percent_format_with( v }; let mut spec = String::new(); - if !flags.is_empty() { - // Build `[fill][align]`. Zero-pad needs the fill - // char *and* the align char together, e.g. "0=" for - // sign-aware zero padding. Left-align via '-' uses - // explicit '<'. - if flags.contains('-') { + // `%`-formatting right-aligns by default for *every* type — even + // strings, unlike the `str.format` mini-language implemented by + // the engine below (which left-aligns strings). Emit an explicit + // `[fill]align` whenever a width is present so the engine doesn't + // fall back to its own per-type default. + let left = flags.contains('-'); + let zero = flags.contains('0') && !left; + let has_width = !width.is_empty(); + if has_width { + if left { spec.push('<'); - } else if flags.contains('0') { - // ``%05d`` → ``0=05d`` (fill='0', align='=', - // ``0`` flag, width=5, type=d). The ``0`` flag - // is harmless after the align prefix. + } else if zero { + // ``%05d`` → ``0=05d`` (fill='0', align='='). spec.push('0'); spec.push('='); + } else { + spec.push('>'); } - if flags.contains('+') { - spec.push('+'); - } else if flags.contains(' ') { - spec.push(' '); - } - if flags.contains('#') { - spec.push('#'); - } - if flags.contains('0') && !flags.contains('-') { - spec.push('0'); - } } - if !width.is_empty() { + if flags.contains('+') { + spec.push('+'); + } else if flags.contains(' ') { + spec.push(' '); + } + if flags.contains('#') { + spec.push('#'); + } + if zero && has_width { + spec.push('0'); + } + if has_width { spec.push_str(&width); } if let Some(p) = precision { @@ -9544,6 +12511,16 @@ pub(crate) fn percent_format_with( } spec.push(kind); let rendered = match kind { + // `%b` inserts a bytes-like object (bytes mode only); `%s` + // aliases it in bytes mode but is text in str mode. + 'b' if mode == PercentMode::Bytes => { + let latin1 = percent_bytes_arg(&item, resolve)?; + format_via_spec(&Object::from_str(latin1), &spec.replace('b', "s"))? + } + 's' if mode == PercentMode::Bytes => { + let latin1 = percent_bytes_arg(&item, resolve)?; + format_via_spec(&Object::from_str(latin1), &spec)? + } 's' => { let s = match resolve(&item, 's')? { Some(s) => s, @@ -9551,6 +12528,17 @@ pub(crate) fn percent_format_with( }; format_via_spec(&Object::from_str(s), &spec)? } + // `%a` (and `%r` in bytes mode) is the ascii-escaped repr. + 'a' | 'r' if mode == PercentMode::Bytes => { + let r = match resolve(&item, 'r')? { + Some(s) => s, + None => item.repr(), + }; + format_via_spec( + &Object::from_str(ascii_escape(&r)), + &spec.replace(['a', 'r'], "s"), + )? + } 'r' => { let s = match resolve(&item, 'r')? { Some(s) => s, @@ -9563,31 +12551,104 @@ pub(crate) fn percent_format_with( &spec.replace('a', "s"), )?, 'd' | 'i' | 'u' => { - // Unwrap `int` subclasses (enum members, _NamedIntConstant) - // so `%d` sees a real integer rather than the instance. let numeric = match &item { + // `%d` truncates a float toward zero (CPython). + Object::Float(f) => { + if f.is_nan() { + return Err(value_error( + "cannot convert float NaN to integer", + )); + } + if f.is_infinite() { + return Err(overflow_error( + "cannot convert float infinity to integer", + )); + } + Object::int_from_bigint(crate::object::bigint_from_f64_trunc(*f)) + } + // Unwrap `int` subclasses (enum members, + // _NamedIntConstant) so `%d` sees a real integer. Object::Instance(_) => match item.as_i64() { Some(n) => Object::Int(n), None => item.clone(), }, - _ => item.clone(), + _ if percent_is_real(&item) => item.clone(), + _ => { + return Err(type_error(format!( + "%{kind} format: a real number is required, not {}", + item.type_name() + ))) + } + }; + format_via_spec_percent(&numeric, &spec.replace(['i', 'u'], "d"))? + } + // `%b` is bytes-only (handled above); in str mode it is an + // unsupported conversion and falls through to the `_` arm. + 'o' | 'x' | 'X' => { + if !percent_is_int(&item) { + return Err(type_error(format!( + "%{kind} format: an integer is required, not {}", + item.type_name() + ))); + } + format_via_spec_percent(&item, &spec)? + } + 'f' | 'F' | 'e' | 'E' | 'g' | 'G' => { + if !percent_is_real(&item) { + return Err(type_error(match mode { + PercentMode::Bytes => format!( + "float argument required, not {}", + item.type_name() + ), + PercentMode::Str => { + format!("must be real number, not {}", item.type_name()) + } + })); + } + format_via_spec_percent(&item, &spec)? + } + 'c' if mode == PercentMode::Bytes => { + // A byte: an int in range(256), or a length-1 bytes-like. + let byte = match &item { + Object::Bool(b) => u8::from(*b), + Object::Int(_) | Object::Long(_) => match item.as_i64() { + Some(n) if (0..=255).contains(&n) => n as u8, + _ => { + return Err(overflow_error("%c arg not in range(256)")) + } + }, + _ => match item.as_bytes_view() { + Some(b) if b.len() == 1 => b[0], + _ => { + return Err(type_error( + "%c requires an integer in range(256) or a single byte", + )) + } + }, }; - format_via_spec(&numeric, &spec.replace(['i', 'u'], "d"))? + format_via_spec( + &Object::from_str((byte as char).to_string()), + &spec.replace('c', "s"), + )? } - 'b' | 'o' | 'x' | 'X' => format_via_spec(&item, &spec)?, - 'f' | 'F' | 'e' | 'E' | 'g' | 'G' => format_via_spec(&item, &spec)?, - 'c' => match &item { - Object::Int(c) => { - char::from_u32(*c as u32).map_or(String::new(), |c| c.to_string()) - } - Object::Str(s) => s.to_string(), - _ => return Err(type_error("%c requires int or single character")), - }, - _ => { - return Err(value_error(format!( - "unsupported format character '{kind}'" - ))) + 'c' => { + let ch = match &item { + Object::Bool(b) => { + char::from_u32(u32::from(*b)).unwrap().to_string() + } + Object::Int(c) => u32::try_from(*c) + .ok() + .and_then(char::from_u32) + .ok_or_else(|| overflow_error("%c arg not in range(0x110000)"))? + .to_string(), + Object::Str(s) if s.chars().count() == 1 => s.to_string(), + _ => return Err(type_error("%c requires int or char")), + }; + // Apply width/alignment by routing through the string + // formatter (`%5c` right-justifies like `%5s`). + format_via_spec_percent(&Object::from_str(ch), &spec.replace('c', "s"))? } + _ => return Err(unsupported_format_char(kind, kind_index)), }; out.push_str(&rendered); } else { @@ -9597,19 +12658,62 @@ pub(crate) fn percent_format_with( i = end; } } + // Leftover positional arguments are an error (mapping args are exempt: + // a dict may legitimately carry keys the template never references). + if !matches!(value, Object::Dict(_)) && idx < positional.len() { + return Err(type_error(format!( + "not all arguments converted during {} formatting", + match mode { + PercentMode::Bytes => "bytes", + PercentMode::Str => "string", + } + ))); + } Ok(out) } +/// Resolve a `%b`/`%s` argument in `bytes %` mode to its raw bytes, decoded +/// latin-1 so the shared (text) engine can splice them. Accepts any +/// bytes-like object or one implementing `__bytes__` (via `resolve`); +/// anything else is the CPython `TypeError`. +fn percent_bytes_arg( + item: &Object, + resolve: &mut dyn FnMut(&Object, char) -> Result, RuntimeError>, +) -> Result { + if let Some(b) = item.as_bytes_view() { + return Ok(b.iter().map(|byte| *byte as char).collect()); + } + if matches!(item, Object::Instance(_)) { + if let Some(s) = resolve(item, 'b')? { + return Ok(s); + } + } + Err(type_error(format!( + "%b requires a bytes-like object, \ + or an object that implements __bytes__, not '{}'", + item.type_name() + ))) +} + /// `ascii()` builtin: like `repr()` but escapes non-ASCII codepoints. fn ascii_repr(value: &Object) -> String { - let r = value.repr(); + ascii_escape(&value.repr()) +} + +/// Escape every non-ASCII scalar in an already-`repr`'d string the way +/// CPython's `ascii()` does: `\xXX` for U+0080-U+00FF, `\uXXXX` for the +/// BMP and `\UXXXXXXXX` above it. ASCII (incl. the escapes `repr` +/// already produced) passes through untouched. +fn ascii_escape(r: &str) -> String { let mut out = String::with_capacity(r.len()); for c in r.chars() { if c.is_ascii() { out.push(c); } else { let n = c as u32; - if n <= 0xFFFF { + if n <= 0xFF { + out.push_str(&format!("\\x{n:02x}")); + } else if n <= 0xFFFF { out.push_str(&format!("\\u{n:04x}")); } else { out.push_str(&format!("\\U{n:08x}")); @@ -9619,42 +12723,97 @@ fn ascii_repr(value: &Object) -> String { out } +/// The user-visible `__name__`/`__qualname__` for a builtin. Internal +/// builtins registered under a dotted sentinel (e.g. the `str.format` +/// method as `.format`, or `gc.collect` as `.gc.collect`) report their +/// final dotted component — matching CPython, where `gc.collect.__name__` +/// is `'collect'` and `str.format.__name__` is `'format'`. +fn builtin_display_name(name: &'static str) -> &'static str { + match name.strip_prefix('.') { + Some(rest) => rest.rsplit('.').next().unwrap_or(rest), + None => name, + } +} + /// Apply a CPython-style format spec to a value. We implement the /// subset needed by f-strings: fill/align, sign, width, precision, /// type. Anything we don't yet handle falls back to the plain string. -fn apply_format_spec(value: &Object, spec: &str, plain: &str) -> Result { - let parsed = parse_format_spec(spec)?; +fn apply_format_spec_inner( + value: &Object, + spec: &str, + plain: &str, + allow_int_precision: bool, +) -> Result { + let parsed = parse_format_spec(spec, value.type_name())?; + // PEP 682 `z` coercion only makes sense for a floating presentation; + // integer/string types reject it (matching CPython's per-type check). + if parsed.no_neg_zero { + let float_presentation = matches!( + parsed.type_char, + Some('e' | 'E' | 'f' | 'F' | 'g' | 'G' | '%') + ) || (matches!(parsed.type_char, None | Some('n')) + && matches!(value, Object::Float(_) | Object::Complex(_))); + if !float_presentation { + return Err(value_error("Negative zero coercion (z) not allowed")); + } + } + // CPython's `str.format` mini-language forbids a precision on the integer + // presentation types; only the printf engine (`allow_int_precision`) + // accepts it (as a minimum-digit count). + if !allow_int_precision + && parsed.precision.is_some() + && matches!(parsed.type_char, Some('d' | 'b' | 'o' | 'x' | 'X' | 'c' | 'n')) + { + return Err(value_error( + "Precision not allowed in integer format specifier", + )); + } + // A precision wider than a C `int` can't be honoured (and would otherwise + // try to allocate gigabytes); CPython raises here. + if matches!(parsed.precision, Some(prec) if prec > i32::MAX as usize) { + return Err(value_error("precision too big")); + } + // Complex routes through its own full formatter (parentheses + repr for + // the no-type case, `re±imj` for explicit float types). + if let Object::Complex(c) = value { + return format_complex(c.real, c.imag, &parsed); + } // Type-driven formatting first; if no type code, just pad. let formatted = match parsed.type_char { Some('d') => match value { Object::Int(i) => format_int(*i, &parsed), Object::Bool(b) => format_int(i64::from(*b), &parsed), + Object::Long(b) => format_bigint(b, &parsed), _ => return Err(value_error("Unknown format code 'd' for non-integer")), }, Some('b') => match value { Object::Int(i) => format_int_base(*i, 2, &parsed), Object::Bool(b) => format_int_base(i64::from(*b), 2, &parsed), + Object::Long(b) => format_bigint_base(b, 2, &parsed), _ => return Err(value_error("Unknown format code 'b' for non-integer")), }, Some('o') => match value { Object::Int(i) => format_int_base(*i, 8, &parsed), Object::Bool(b) => format_int_base(i64::from(*b), 8, &parsed), + Object::Long(b) => format_bigint_base(b, 8, &parsed), _ => return Err(value_error("Unknown format code 'o' for non-integer")), }, Some('x') => match value { Object::Int(i) => format_int_hex(*i, false, &parsed), Object::Bool(b) => format_int_hex(i64::from(*b), false, &parsed), + Object::Long(b) => format_bigint_hex(b, false, &parsed), _ => return Err(value_error("Unknown format code 'x' for non-integer")), }, Some('X') => match value { Object::Int(i) => format_int_hex(*i, true, &parsed), Object::Bool(b) => format_int_hex(i64::from(*b), true, &parsed), + Object::Long(b) => format_bigint_hex(b, true, &parsed), _ => return Err(value_error("Unknown format code 'X' for non-integer")), }, Some('f') | Some('F') => { let f = obj_as_float(value)?; let prec = parsed.precision.unwrap_or(6); - format_float_fixed(f, prec, &parsed) + format_float_fixed(f, prec, parsed.type_char == Some('F'), &parsed) } Some('e') => { let f = obj_as_float(value)?; @@ -9674,9 +12833,48 @@ fn apply_format_spec(value: &Object, spec: &str, plain: &str) -> Result { let f = obj_as_float(value)?; let prec = parsed.precision.unwrap_or(6); - let body = format_float_fixed(f * 100.0, prec, &parsed); + let body = format_float_fixed(f * 100.0, prec, false, &parsed); format!("{body}%") } + None if matches!(value, Object::Float(_)) => { + let Object::Float(f) = value else { + unreachable!() + }; + format_float_no_type(*f, &parsed) + } + // No-type integers reject a precision (CPython's + // "Precision not allowed in integer format specifier"). + None if parsed.precision.is_some() + && matches!(value, Object::Int(_) | Object::Long(_) | Object::Bool(_)) => + { + return Err(value_error( + "Precision not allowed in integer format specifier", + )); + } + // A no-type integer formats like `d`: in particular it honours the + // thousands separator (`format(1234, ',')` → `'1,234'`), which the + // generic string fall-through below would drop. + None if matches!(value, Object::Int(_) | Object::Long(_) | Object::Bool(_)) => { + match value { + Object::Int(i) => format_int(*i, &parsed), + Object::Bool(b) => format_int(i64::from(*b), &parsed), + Object::Long(b) => format_bigint(b, &parsed), + _ => unreachable!(), + } + } + // `s` (string presentation) is invalid for numeric types — CPython + // raises "Unknown format code 's' for object of type 'float'". + Some('s') + if matches!( + value, + Object::Int(_) | Object::Long(_) | Object::Bool(_) | Object::Float(_) + ) => + { + return Err(value_error(format!( + "Unknown format code 's' for object of type '{}'", + value.type_name() + ))); + } Some('s') | None => { let mut s = plain.to_owned(); if let Some(p) = parsed.precision { @@ -9709,6 +12907,23 @@ fn apply_format_spec(value: &Object, spec: &str, plain: &str) -> Result return Err(value_error("%c requires int or char")), }, + // `n` is locale-aware: integers group like `d`, floats like `g`. + // WeavePy runs in the C locale, so grouping is empty and the output + // matches the non-`n` form (the digits are what tests assert on). + Some('n') => match value { + Object::Int(i) => format_int(*i, &parsed), + Object::Bool(b) => format_int(i64::from(*b), &parsed), + Object::Long(b) => format_bigint(b, &parsed), + Object::Float(f) => { + format_float_general(*f, parsed.precision.unwrap_or(6).max(1), false, &parsed) + } + _ => { + return Err(value_error(format!( + "Unknown format code 'n' for object of type '{}'", + value.type_name() + ))) + } + }, Some(other) => { return Err(value_error(format!( "Unknown format code '{other}' for object of type '{}'", @@ -9719,11 +12934,24 @@ fn apply_format_spec(value: &Object, spec: &str, plain: &str) -> Result, align: Option, sign: Option, + /// PEP 682 `z` option: coerce a negative result that rounds to zero + /// (`-0.0`) into positive zero. + no_neg_zero: bool, alt: bool, zero: bool, width: Option, @@ -9732,7 +12960,18 @@ struct ParsedSpec { type_char: Option, } -fn parse_format_spec(spec: &str) -> Result { +/// Accumulate one decimal digit of a width/precision field, rejecting values +/// that overflow a C `Py_ssize_t` (CPython raises "Too many decimal digits in +/// format string"). Keeps the field from later attempting an unbounded +/// allocation (e.g. `format(x, '.%df' % 2**63)`). +fn accumulate_spec_digit(acc: usize, c: char) -> Result { + acc.checked_mul(10) + .and_then(|v| v.checked_add(c as usize - '0' as usize)) + .filter(|v| *v <= i64::MAX as usize) + .ok_or_else(|| value_error("Too many decimal digits in format string")) +} + +fn parse_format_spec(spec: &str, type_name: &str) -> Result { let mut p = ParsedSpec::default(); let chars: Vec = spec.chars().collect(); let mut i = 0; @@ -9752,6 +12991,11 @@ fn parse_format_spec(spec: &str) -> Result { i += 1; } } + // [z] — PEP 682 negative-zero coercion, between sign and `#`. + if let Some(&'z') = chars.get(i) { + p.no_neg_zero = true; + i += 1; + } // [#] if let Some(&'#') = chars.get(i) { p.alt = true; @@ -9771,7 +13015,7 @@ fn parse_format_spec(spec: &str) -> Result { let mut had_width = false; while let Some(&c) = chars.get(i) { if c.is_ascii_digit() { - width = width * 10 + (c as usize - '0' as usize); + width = accumulate_spec_digit(width, c)?; i += 1; had_width = true; } else { @@ -9786,6 +13030,18 @@ fn parse_format_spec(spec: &str) -> Result { if matches!(c, ',' | '_') { p.grouping = Some(c); i += 1; + // A second grouping char is always an error, with CPython's two + // distinct messages: same char twice ("Cannot specify ',' with + // ','.") vs. mixing the two ("Cannot specify both ',' and '_'."). + if let Some(&c2) = chars.get(i) { + if matches!(c2, ',' | '_') { + return Err(value_error(if c2 == c { + format!("Cannot specify '{c}' with '{c}'.") + } else { + "Cannot specify both ',' and '_'.".to_owned() + })); + } + } } } // [.precision] @@ -9795,7 +13051,7 @@ fn parse_format_spec(spec: &str) -> Result { let mut had_prec = false; while let Some(&c) = chars.get(i) { if c.is_ascii_digit() { - prec = prec * 10 + (c as usize - '0' as usize); + prec = accumulate_spec_digit(prec, c)?; i += 1; had_prec = true; } else { @@ -9814,7 +13070,9 @@ fn parse_format_spec(spec: &str) -> Result { } } if i < chars.len() { - return Err(value_error(format!("invalid format specifier: {spec:?}"))); + return Err(value_error(format!( + "Invalid format specifier '{spec}' for object of type '{type_name}'" + ))); } Ok(p) } @@ -9831,33 +13089,50 @@ fn obj_as_float(v: &Object) -> Result { } } +/// Apply a printf integer precision (`%.Nd` → at least N digits, zero-filled) +/// to a magnitude digit string. CPython still honours a `0` width flag on top +/// of this, so the spec is left untouched. (`str.format` rejects integer +/// precision upstream, so this only fires for the printf engine.) +fn int_precision_apply(core: String, p: &ParsedSpec) -> String { + match p.precision { + Some(prec) if core.len() < prec => { + format!("{}{core}", "0".repeat(prec - core.len())) + } + _ => core, + } +} + fn format_int(i: i64, p: &ParsedSpec) -> String { let mag = i.unsigned_abs(); - let mut body = if let Some(grp) = p.grouping { + let core = if let Some(grp) = p.grouping { group_decimal(mag, grp) } else { mag.to_string() }; - body = with_sign(i < 0, &body, p); + let core = int_precision_apply(core, p); + let body = with_sign(i < 0, &core, p); apply_alignment(&body, p, true) } fn format_int_base(i: i64, base: u32, p: &ParsedSpec) -> String { let mag = i.unsigned_abs(); - let mut body = match base { + let core = match base { 2 => format!("{mag:b}"), 8 => format!("{mag:o}"), 10 => mag.to_string(), _ => mag.to_string(), }; - if p.alt { + let core = int_precision_apply(core, p); + let mut body = if p.alt { let prefix = match base { 2 => "0b", 8 => "0o", _ => "", }; - body = format!("{prefix}{body}"); - } + format!("{prefix}{core}") + } else { + core + }; body = with_sign(i < 0, &body, p); apply_alignment(&body, p, true) } @@ -9869,6 +13144,7 @@ fn format_int_hex(i: i64, upper: bool, p: &ParsedSpec) -> String { } else { format!("{mag:x}") }; + let body_core = int_precision_apply(body_core, p); let mut body = if p.alt { format!("{}{body_core}", if upper { "0X" } else { "0x" }) } else { @@ -9878,46 +13154,192 @@ fn format_int_hex(i: i64, upper: bool, p: &ParsedSpec) -> String { apply_alignment(&body, p, true) } -fn format_float_fixed(f: f64, prec: usize, p: &ParsedSpec) -> String { +/// Group an already-rendered magnitude string into fixed-size runs from +/// the right (e.g. `1234567` → `1,234,567`), for arbitrary digit strings. +fn group_str(s: &str, sep: char, group: usize) -> String { + let bytes = s.as_bytes(); + let mut out = String::with_capacity(s.len() + s.len() / group + 1); + let mut first = bytes.len() % group; + if first == 0 { + first = group; + } + out.push_str(std::str::from_utf8(&bytes[..first]).unwrap()); + let mut i = first; + while i < bytes.len() { + out.push(sep); + out.push_str(std::str::from_utf8(&bytes[i..i + group]).unwrap()); + i += group; + } + out +} + +/// Bignum counterparts of [`format_int`]/[`format_int_base`]/ +/// [`format_int_hex`] so the `d`/`b`/`o`/`x`/`X` presentation types work +/// on arbitrary-precision ints (CPython has no width limit here). +fn format_bigint(b: &num_bigint::BigInt, p: &ParsedSpec) -> String { + use num_traits::Signed; + let neg = b.is_negative(); + let mag = b.abs().to_string(); + let core = if let Some(grp) = p.grouping { + group_str(&mag, grp, 3) + } else { + mag + }; + let core = int_precision_apply(core, p); + let body = with_sign(neg, &core, p); + apply_alignment(&body, p, true) +} + +fn format_bigint_base(b: &num_bigint::BigInt, base: u32, p: &ParsedSpec) -> String { + use num_traits::Signed; + let neg = b.is_negative(); + let abs = b.abs(); + let core = match base { + 2 => format!("{abs:b}"), + 8 => format!("{abs:o}"), + _ => abs.to_string(), + }; + let core = int_precision_apply(core, p); + let mut body = if p.alt { + let prefix = match base { + 2 => "0b", + 8 => "0o", + _ => "", + }; + format!("{prefix}{core}") + } else { + core + }; + body = with_sign(neg, &body, p); + apply_alignment(&body, p, true) +} + +fn format_bigint_hex(b: &num_bigint::BigInt, upper: bool, p: &ParsedSpec) -> String { + use num_traits::Signed; + let neg = b.is_negative(); + let abs = b.abs(); + let core = if upper { + format!("{abs:X}") + } else { + format!("{abs:x}") + }; + let core = int_precision_apply(core, p); + let mut body = if p.alt { + format!("{}{core}", if upper { "0X" } else { "0x" }) + } else { + core + }; + body = with_sign(neg, &body, p); + apply_alignment(&body, p, true) +} + +/// Unsigned fixed-point magnitude digits (no sign, no padding). A finite +/// f64 has at most ~1074 fractional digits, but Rust's formatter panics on +/// extremely large precisions, so we cap the real work and zero-pad the +/// rest (`'%.123456f'` matches CPython instead of aborting). +fn fixed_core(mag: f64, prec: usize) -> String { + const FIXED_PREC_CAP: usize = 1100; + if prec <= FIXED_PREC_CAP { + format!("{mag:.*}", prec) + } else { + let cap = FIXED_PREC_CAP; + let mut s = format!("{mag:.cap$}"); + s.extend(std::iter::repeat('0').take(prec - cap)); + s + } +} + +/// `true` when every decimal digit in `core` is `'0'` (and there is at +/// least one). Used to decide PEP 682 negative-zero coercion: a value that +/// renders with all-zero digits *is* zero at this precision. +fn all_digits_zero(core: &str) -> bool { + let mut any = false; + for b in core.bytes() { + if b.is_ascii_digit() { + any = true; + if b != b'0' { + return false; + } + } + } + any +} + +/// Final sign + alignment pass shared by the float presentation types, +/// applying PEP 682 `z` coercion when the magnitude rounds to zero. +fn finish_float(neg: bool, core: String, p: &ParsedSpec) -> String { + let core = match p.grouping { + Some(sep) => group_float_core(&core, sep), + None => core, + }; + let neg = neg && !(p.no_neg_zero && all_digits_zero(&core)); + let body = with_sign(neg, &core, p); + apply_alignment(&body, p, true) +} + +/// Insert a thousands separator into the integer part of a rendered float +/// magnitude (`"150000000000000000000.00"` → `"150,000,…,000.00"`), +/// leaving any fractional part and exponent untouched. Both `,` and `_` +/// group decimal digits in threes. +fn group_float_core(core: &str, sep: char) -> String { + let bytes = core.as_bytes(); + let mut int_end = 0; + while int_end < bytes.len() && bytes[int_end].is_ascii_digit() { + int_end += 1; + } + if int_end == 0 { + return core.to_owned(); + } + let grouped = group_str(&core[..int_end], sep, 3); + format!("{grouped}{}", &core[int_end..]) +} + +/// Pad and sign an `inf`/`nan` float rendering. Unlike finite values these +/// skip grouping and `z`/neg-zero coercion, but they still honour the sign +/// flag (`+nan`, `-inf`) and the numeric default of right alignment — +/// matching CPython's `format_float_short` special-value branch. +fn finish_special(neg: bool, core: &str, p: &ParsedSpec) -> String { + let body = with_sign(neg, core, p); + apply_alignment(&body, p, true) +} + +fn format_float_fixed(f: f64, prec: usize, upper: bool, p: &ParsedSpec) -> String { if f.is_nan() { - return apply_alignment("nan", p, false); + return finish_special(false, if upper { "NAN" } else { "nan" }, p); } if f.is_infinite() { - let s = if f < 0.0 { "-inf" } else { "inf" }; - return apply_alignment(s, p, false); + return finish_special(f < 0.0, if upper { "INF" } else { "inf" }, p); + } + let core = fixed_core(f.abs(), prec); + let core = if p.alt { ensure_decimal_point(&core) } else { core }; + finish_float(f.is_sign_negative(), core, p) +} + +/// `#`/alt flag for the float presentation types: force a decimal-point +/// character into the mantissa even when no fractional digits follow +/// (`"1"` → `"1."`, `"1e+00"` → `"1.e+00"`), matching CPython's +/// `format_float_short` (`add_dot_0_if_integer` plus the alt path). +fn ensure_decimal_point(core: &str) -> String { + if core.contains('.') { + return core.to_owned(); + } + match core.find(['e', 'E']) { + Some(i) => format!("{}.{}", &core[..i], &core[i..]), + None => format!("{core}."), } - let neg = f.is_sign_negative(); - let mag = f.abs(); - let body = format!("{mag:.*}", prec); - let body = with_sign(neg, &body, p); - apply_alignment(&body, p, true) } fn format_float_scientific(f: f64, prec: usize, upper: bool, p: &ParsedSpec) -> String { if f.is_nan() { - return apply_alignment(if upper { "NAN" } else { "nan" }, p, false); + return finish_special(false, if upper { "NAN" } else { "nan" }, p); } if f.is_infinite() { - let s = if upper { - if f < 0.0 { - "-INF" - } else { - "INF" - } - } else if f < 0.0 { - "-inf" - } else { - "inf" - }; - return apply_alignment(s, p, false); + return finish_special(f < 0.0, if upper { "INF" } else { "inf" }, p); } - let neg = f.is_sign_negative(); - let mag = f.abs(); - let raw = format!("{mag:.*e}", prec); // Rust gives e.g. "1.230000e2"; CPython wants "1.230000e+02". - let body = normalize_exponent(&raw, upper); - let body = with_sign(neg, &body, p); - apply_alignment(&body, p, true) + let core = normalize_exponent(&format!("{:.*e}", prec, f.abs()), upper); + let core = if p.alt { ensure_decimal_point(&core) } else { core }; + finish_float(f.is_sign_negative(), core, p) } fn normalize_exponent(raw: &str, upper: bool) -> String { @@ -9944,16 +13366,172 @@ fn normalize_exponent(raw: &str, upper: bool) -> String { } fn format_float_general(f: f64, prec: usize, upper: bool, p: &ParsedSpec) -> String { - if f == 0.0 || f.is_nan() || f.is_infinite() { - return format_float_fixed(f, prec.saturating_sub(1), p); + if f.is_nan() { + return finish_special(false, if upper { "NAN" } else { "nan" }, p); + } + if f.is_infinite() { + return finish_special(f < 0.0, if upper { "INF" } else { "inf" }, p); + } + let prec = prec.max(1); + // Plain `g`/`G` switch to scientific when the decimal exponent reaches + // the precision (`decpt > precision`). + let core = general_core(f.abs(), prec, upper, prec as i32); + // `g`/`G` strip trailing zeros unless the `#`/alt flag is set, which + // also forces a trailing decimal point. + let core = if p.alt { + ensure_decimal_point(&core) + } else { + strip_g_zeros(&core) + }; + finish_float(f.is_sign_negative(), core, p) +} + +/// The unstripped `g`-format core for a non-negative magnitude: chooses +/// fixed vs scientific from the post-rounding decimal exponent (matching +/// CPython, which keys off the rounded exponent rather than `log10`). +/// `exp_hi` is the exclusive exponent at which scientific notation kicks in +/// (`precision` for `g`, `precision - 1` for the type-omitted format). +fn general_core(mag: f64, prec: usize, upper: bool, exp_hi: i32) -> String { + let prec = prec.max(1); + let sci = format!("{:.*e}", prec - 1, mag); + let exp: i32 = sci + .rfind('e') + .and_then(|k| sci[k + 1..].parse().ok()) + .unwrap_or(0); + if exp < -4 || exp >= exp_hi { + normalize_exponent(&sci, upper) + } else { + fixed_core(mag, (prec as i32 - 1 - exp).max(0) as usize) + } +} + +/// `float.__format__` with the presentation type omitted: shortest `repr` +/// when no precision is given, else `g`-style at that precision. Either way +/// CPython's no-type path sets `ADD_DOT_0`, so an integral result keeps a +/// trailing `.0` (`format(100.0, '.4')` → `'100.0'`). +fn format_float_no_type(f: f64, p: &ParsedSpec) -> String { + if f.is_nan() { + return finish_special(false, "nan", p); + } + if f.is_infinite() { + return finish_special(f < 0.0, "inf", p); } - let exp = f.abs().log10().floor() as i32; - if exp < -4 || exp >= prec as i32 { - format_float_scientific(f, prec.saturating_sub(1), upper, p) + let core = match p.precision { + None => crate::object::float_repr(f.abs()), + // Type omitted: like `g` but scientific kicks in one exponent earlier + // (`decpt > precision - 1`), per CPython's `ADD_DOT_0` path. + Some(prec) => strip_g_zeros(&general_core(f.abs(), prec, false, prec.max(1) as i32 - 1)), + }; + let core = if core.contains(['.', 'e', 'E']) { + core } else { - let digits_after = (prec as i32 - 1 - exp).max(0) as usize; - format_float_fixed(f, digits_after, p) + format!("{core}.0") + }; + finish_float(f.is_sign_negative(), core, p) +} + +/// Format a complex value with an explicit float presentation type +/// (`f`/`e`/`g` family). CPython renders `re±imj` with no parentheses, the +/// imaginary part always carrying an explicit sign, and applies any +/// width/fill/alignment to the *whole* result. +/// Full complex `__format__` (CPython `format_complex_internal`): +/// +/// - With an explicit float type (`e`/`E`/`f`/`F`/`g`/`G`) both parts are +/// rendered with that type and joined `re±imj`, no parentheses. +/// - With the type omitted it behaves like `str(z)`: a `+0.0` real part is +/// dropped (`3j`), otherwise the value is wrapped in parentheses. The +/// components use the shortest `repr` form when no precision is given, +/// and `g` with that precision otherwise. +/// - `'='` alignment and `'0'` zero-padding are rejected; integer +/// presentation types are unknown for `complex`. +fn format_complex(re: f64, im: f64, parsed: &ParsedSpec) -> Result { + if parsed.align == Some('=') { + return Err(value_error( + "'=' alignment not allowed in complex format specifier", + )); + } + if parsed.zero { + return Err(value_error( + "Zero padding is not allowed in complex format specifier", + )); } + match parsed.type_char { + None | Some('e' | 'E' | 'f' | 'F' | 'g' | 'G') => {} + Some(other) => { + return Err(value_error(format!( + "unknown format code '{other}' for object of type 'complex'" + ))) + } + } + let no_type = parsed.type_char.is_none(); + // `str(z)` drops a `+0.0` real part and shows just the imaginary half. + let skip_re = no_type && re == 0.0 && re.is_sign_positive(); + let add_parens = no_type && !skip_re; + + // Render one component. Width/fill/align/zero belong to the combined + // string, never the parts; `force_sign` makes the imaginary half always + // carry a leading sign. + let comp = |v: f64, force_sign: bool| -> String { + let mut cp = parsed.clone(); + cp.fill = None; + cp.align = None; + cp.width = None; + cp.zero = false; + if force_sign { + cp.sign = Some('+'); + } + match parsed.type_char { + Some('e') => format_float_scientific(v, parsed.precision.unwrap_or(6), false, &cp), + Some('E') => format_float_scientific(v, parsed.precision.unwrap_or(6), true, &cp), + Some('f') => format_float_fixed(v, parsed.precision.unwrap_or(6), false, &cp), + Some('F') => format_float_fixed(v, parsed.precision.unwrap_or(6), true, &cp), + Some('g') => format_float_general(v, parsed.precision.unwrap_or(6).max(1), false, &cp), + Some('G') => format_float_general(v, parsed.precision.unwrap_or(6).max(1), true, &cp), + // Type omitted: `g` with the given precision, else shortest repr. + _ => match parsed.precision { + Some(p) => format_float_general(v, p.max(1), false, &cp), + None => { + let r = crate::object::complex_component_repr(v); + if r.starts_with('-') { + r + } else if force_sign { + format!("+{r}") + } else { + match parsed.sign { + Some('+') => format!("+{r}"), + Some(' ') => format!(" {r}"), + _ => r, + } + } + } + }, + } + }; + + let im_s = comp(im, !skip_re); + let body = if skip_re { + format!("{im_s}j") + } else { + format!("{}{im_s}j", comp(re, false)) + }; + let body = if add_parens { format!("({body})") } else { body }; + Ok(apply_alignment(&body, parsed, true)) +} + +/// Strip trailing fractional zeros (and a bare decimal point) from a +/// magnitude string, preserving any exponent suffix. Implements the `g` +/// presentation type's zero trimming. +fn strip_g_zeros(core: &str) -> String { + let (mant, exp) = match core.find(['e', 'E']) { + Some(k) => (&core[..k], &core[k..]), + None => (core, ""), + }; + let mant = if mant.contains('.') { + mant.trim_end_matches('0').trim_end_matches('.') + } else { + mant + }; + format!("{mant}{exp}") } fn with_sign(neg: bool, body: &str, p: &ParsedSpec) -> String { @@ -10006,23 +13584,29 @@ fn apply_alignment(body: &str, p: &ParsedSpec, default_right: bool) -> String { s } '=' => { - // Pad between sign and digits. - let mut chars = body.chars(); - let lead = chars - .next() - .filter(|c| matches!(*c, '+' | '-' | ' ')) - .map_or(String::new(), |c| c.to_string()); - let rest: String = if lead.is_empty() { - body.to_owned() - } else { - chars.collect() - }; + // Pad between the sign (and any `#` base prefix) and the digits, + // so e.g. `%#08x % 255` → `0x0000ff`, not `00000xff`. + let mut rest = body; + let mut lead = String::new(); + if let Some(c) = rest.chars().next() { + if matches!(c, '+' | '-' | ' ') { + lead.push(c); + rest = &rest[c.len_utf8()..]; + } + } + if p.alt && rest.len() >= 2 { + let pfx = &rest[..2]; + if matches!(pfx, "0x" | "0X" | "0o" | "0O" | "0b" | "0B") { + lead.push_str(pfx); + rest = &rest[2..]; + } + } let mut s = String::with_capacity(body.len() + pad); s.push_str(&lead); for _ in 0..pad { s.push(fill); } - s.push_str(&rest); + s.push_str(rest); s } _ => body.to_owned(), @@ -10060,6 +13644,17 @@ fn is_index_error(e: &RuntimeError) -> bool { false } +fn is_type_error(e: &RuntimeError) -> bool { + if let RuntimeError::PyException(pe) = e { + if let Object::Instance(inst) = &pe.instance { + return inst + .class + .is_subclass_of(&crate::builtin_types::builtin_types().type_error); + } + } + false +} + fn binop_dunders(op: BinOpKind) -> (&'static str, &'static str) { use BinOpKind as B; match op { @@ -10275,6 +13870,16 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result Result Ok(O::Float(x.rem_euclid(*y))), - (O::Float(x), O::Float(y), B::Pow) => Ok(O::Float(x.powf(*y))), + (O::Float(x), O::Float(y), B::Mod) => Ok(O::Float(py_float_mod(*x, *y)?)), + (O::Float(x), O::Float(y), B::Pow) => float_pow(*x, *y), (O::Float(x), O::Float(y), B::FloorDiv) => Ok(O::Float((x / y).floor())), (O::Int(x), O::Float(y), op) => binary_op(&O::Float(*x as f64), &O::Float(*y), op), @@ -10328,23 +13933,23 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result Ok(Object::from_str(percent_format(template, v)?)), (O::Bytes(template), v, B::Mod) => { - // PEP 461: ``bytes % args`` reuses the same templating - // engine as ``str % args``. We decode the template as - // latin-1 (raw byte → 1:1 codepoint mapping), substitute, - // and re-encode the same way so opaque bytes round-trip. + // PEP 461: ``bytes % args`` reuses the text ``%``-engine in + // bytes mode over a latin-1 (raw byte → 1:1 codepoint) view, + // then re-encodes. (The VM path in `dispatch_binop` handles + // `__bytes__`; this pure fallback covers bytes-like args.) let s: String = template.iter().map(|b| *b as char).collect(); - let mapped = bytes_percent_args(v); - let rendered = percent_format(&s, &mapped)?; + let mut noop = |_: &Object, _: char| Ok(None); + let rendered = percent_format_with(&s, v, PercentMode::Bytes, &mut noop)?; let out: Vec = rendered.chars().map(|c| c as u8).collect(); Ok(Object::new_bytes(out)) } (O::ByteArray(cell), v, B::Mod) => { let template = cell.borrow().clone(); let s: String = template.iter().map(|b| *b as char).collect(); - let mapped = bytes_percent_args(v); - let rendered = percent_format(&s, &mapped)?; + let mut noop = |_: &Object, _: char| Ok(None); + let rendered = percent_format_with(&s, v, PercentMode::Bytes, &mut noop)?; let out: Vec = rendered.chars().map(|c| c as u8).collect(); - Ok(Object::new_bytes(out)) + Ok(Object::new_bytearray(out)) } (O::Bytes(x), O::Bytes(y), B::Add) => { let mut out = Vec::with_capacity(x.len() + y.len()); @@ -10422,16 +14027,22 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result { + // [`is_pep604_union`]. CPython drives this off `type.__or__` / + // `type.__ror__`, so at least one operand must be a real type + // (or an existing union) — `None | None` raises `TypeError`. + _ if op == B::BitOr + && is_union_eligible(&a) + && is_union_eligible(&b) + && (is_union_initiator(&a) || is_union_initiator(&b)) => + { Ok(make_pep604_union(&a, &b)) } _ => Err(type_error(format!( "unsupported operand type(s) for {}: '{}' and '{}'", op.as_str(), - a.type_name(), - b.type_name() + a.type_name_owned(), + b.type_name_owned() ))), } } @@ -10444,6 +14055,15 @@ fn is_union_eligible(obj: &Object) -> bool { matches!(obj, Object::Type(_) | Object::None) || is_pep604_union(obj).is_some() } +/// A PEP 604 union can only be *initiated* by an operand that carries +/// the `type.__or__`/`__ror__` slots — a real type or an existing +/// union. Bare `None` is union-*eligible* (normalised to `type(None)`) +/// but cannot start one, so `None | None` raises `TypeError` like +/// CPython. +fn is_union_initiator(obj: &Object) -> bool { + matches!(obj, Object::Type(_)) || is_pep604_union(obj).is_some() +} + /// Detect whether `obj` is a PEP 604 union. Returns the flattened /// list of `__args__` if so, else `None`. /// @@ -10539,6 +14159,48 @@ fn normalize_union_arg(x: Object) -> Object { x } +/// Python `float % float`. Unlike Rust's `%` (C `fmod`, sign of the +/// dividend), Python's modulo takes the sign of the *divisor* — e.g. +/// `0.1 % float('-inf') == -inf`. Mirrors CPython's `float_rem`. +fn py_float_mod(x: f64, y: f64) -> Result { + if y == 0.0 { + return Err(zero_division_error("float modulo")); + } + let mut m = x % y; + if m != 0.0 { + // The remainder takes the sign of the divisor. + if (y < 0.0) != (m < 0.0) { + m += y; + } + } else { + // A zero remainder still carries the divisor's sign (CPython uses + // `copysign(0.0, y)`, since `fmod`'s signed zero is unportable). + m = 0.0_f64.copysign(y); + } + Ok(m) +} + +/// Python `float ** float`. A negative base raised to a non-integer +/// exponent is complex (CPython promotes via `complex` rather than +/// returning NaN like C's `pow`). `x = |x|·e^{iπ}` so +/// `x^y = |x|^y·(cos πy + i·sin πy)`. +fn float_pow(x: f64, y: f64) -> Result { + // 0.0 to a finite negative power is a division by zero (C99 still treats + // 0**-inf as +inf, so only finite negative exponents raise). + if x == 0.0 && y < 0.0 && y.is_finite() { + return Err(zero_division_error( + "0.0 cannot be raised to a negative power", + )); + } + if x < 0.0 && y.fract() != 0.0 && x.is_finite() && y.is_finite() { + let magnitude = (-x).powf(y); + let theta = std::f64::consts::PI * y; + Ok(Object::new_complex(magnitude * theta.cos(), magnitude * theta.sin())) + } else { + Ok(Object::Float(x.powf(y))) + } +} + fn unary_op(v: &Object, op: UnaryKind) -> Result { use Object as O; match (op, v) { @@ -10562,11 +14224,70 @@ fn unary_op(v: &Object, op: UnaryKind) -> Result { _ => Err(type_error(format!( "bad operand type for unary {}: '{}'", op.as_str(), - v.type_name() + v.type_name_owned() ))), } } +/// Multiply `mant` by `2**exp`, saturating to `±inf`/`0.0` at the f64 +/// range limits. Applied in <=1000-bit steps so each intermediate power +/// of two is exactly representable. +fn scale_pow2(mant: f64, exp: i64) -> f64 { + if mant == 0.0 || !mant.is_finite() { + return mant; + } + let mut m = mant; + let mut e = exp; + while e > 0 { + let step = e.min(1000); + m *= 2f64.powi(step as i32); + if !m.is_finite() { + return m; + } + e -= step; + } + while e < 0 { + let step = (-e).min(1000); + m *= 2f64.powi(-(step as i32)); + if m == 0.0 { + return 0.0; + } + e += step; + } + m +} + +/// Correctly-scaled `BigInt` true division returning the nearest f64. +/// `y` must be non-zero. Avoids the `inf/inf == NaN` trap of dividing the +/// two operands' (possibly overflowing) f64 approximations. +fn bigint_true_div(x: &num_bigint::BigInt, y: &num_bigint::BigInt) -> f64 { + use num_bigint::Sign; + use num_traits::{ToPrimitive, Zero}; + if x.is_zero() { + return 0.0; + } + let negative = (x.sign() == Sign::Minus) ^ (y.sign() == Sign::Minus); + let xm = x.magnitude(); + let ym = y.magnitude(); + let la = xm.bits() as i64; + let lb = ym.bits() as i64; + // Pick a shift so the integer quotient retains ~64 significant bits. + let shift = 64 - la + lb; + let (num, den) = if shift >= 0 { + (xm.clone() << (shift as usize), ym.clone()) + } else { + (xm.clone(), ym.clone() << ((-shift) as usize)) + }; + let q = num / den; + let qf = q.to_f64().unwrap_or(f64::INFINITY); + let result = scale_pow2(qf, -shift); + if negative { + -result + } else { + result + } +} + /// Bignum-aware integer arithmetic for `int`-flavoured operands. /// Both inputs are guaranteed `int`/`long`/`bool` by the caller; the /// fast path stays in `i64` until an overflow forces promotion. @@ -10591,13 +14312,12 @@ fn bignum_op(a: &Object, b: &Object, op: BinOpKind) -> Result { if y.is_zero() { @@ -10780,6 +14500,115 @@ fn make_generic_alias(origin: Object, params: Object) -> Object { Object::SimpleNamespace(Rc::new(RefCell::new(d))) } +/// True if `info` is a PEP 585 generic alias (a `SimpleNamespace`-shaped +/// object carrying `__origin__`, produced by [`make_generic_alias`]). +/// `isinstance`/`issubclass` route these through the builtin so the alias +/// is unwrapped to its origin class before the MRO walk. +fn is_generic_alias(info: &Object) -> bool { + matches!(info, Object::SimpleNamespace(d) + if d.borrow().get(&DictKey(Object::from_static("__origin__"))).is_some()) +} + +/// A `Long` whose magnitude exceeds what a C double can represent. +/// `num_bigint::BigInt::to_f64` saturates to ±∞ for such values, which +/// is how we detect the overflow CPython reports as `OverflowError`. +fn long_overflows_f64(o: &Object) -> bool { + matches!(o, Object::Long(b) if b.to_f64().is_none_or(f64::is_infinite)) +} + +/// CPython's `_Py_c_quot` (Smith's algorithm): scale by the +/// larger-magnitude component so `(1e200+1e200j)/(1e200+1e200j)` +/// doesn't overflow the naive `|b|²` denominator. Matches +/// `Objects/complexobject.c` exactly. +fn complex_div(ar: f64, ai: f64, br: f64, bi: f64) -> Result<(f64, f64), RuntimeError> { + // Mirrors CPython's `_Py_c_quot`: the three-way magnitude test means a + // NaN component in the denominator falls through to a NaN result (both + // `>=` tests are false), and division-by-zero is signalled *only* when + // the denominator is exactly `0+0j` (first branch with `abs_br == 0`). + let abs_br = br.abs(); + let abs_bi = bi.abs(); + if abs_br >= abs_bi { + if abs_br == 0.0 { + return Err(zero_division_error("complex division by zero")); + } + let ratio = bi / br; + let denom = br + bi * ratio; + Ok(((ar + ai * ratio) / denom, (ai - ar * ratio) / denom)) + } else if abs_bi >= abs_br { + // `abs_bi >= abs_br` and not the first branch ⇒ `abs_bi != 0`. + let ratio = br / bi; + let denom = br * ratio + bi; + Ok(((ar * ratio + ai) / denom, (ai * ratio - ar) / denom)) + } else { + // At least one of `br`/`bi` is NaN. + Ok((f64::NAN, f64::NAN)) + } +} + +/// CPython `_Py_c_pow` — repeated multiplication for small integer +/// exponents (exact for e.g. `(1+1j)**2 == 2j`), polar form otherwise. +/// Raises `ZeroDivisionError` for `0 ** (negative or complex)`, matching +/// `complexobject.c`'s `errno == EDOM` check. +fn complex_pow(ar: f64, ai: f64, br: f64, bi: f64) -> Result<(f64, f64), RuntimeError> { + // `x ** 0 == 1` for every base (checked before the zero-base guard). + if br == 0.0 && bi == 0.0 { + return Ok((1.0, 0.0)); + } + // `0 ** y`: zero for a positive real `y`, otherwise a zero-division + // (CPython sets `errno = EDOM` → "0.0 to a negative or complex power"). + if ar == 0.0 && ai == 0.0 { + if bi != 0.0 || br < 0.0 { + return Err(zero_division_error("0.0 to a negative or complex power")); + } + return Ok((0.0, 0.0)); + } + // Integer real exponent in (-100, 100), zero imaginary part: CPython + // uses `c_powi`/`c_powu` (repeated squaring) so results are exact for + // the common integer-power cases the test-suite checks. + if bi == 0.0 && br.fract() == 0.0 && br.abs() < 100.0 { + let n = br as i64; + let (mut pr, mut pi) = (1.0_f64, 0.0_f64); + let (mut xr, mut xi) = (ar, ai); + let mut k = n.unsigned_abs(); + while k > 0 { + if k & 1 == 1 { + let nr = pr * xr - pi * xi; + let ni = pr * xi + pi * xr; + pr = nr; + pi = ni; + } + let sr = xr * xr - xi * xi; + let si = 2.0 * xr * xi; + xr = sr; + xi = si; + k >>= 1; + } + if n < 0 { + // Reciprocal via Smith's algorithm (base != 0, guarded above). + let (rr, ri) = complex_div(1.0, 0.0, pr, pi)?; + return complex_pow_finish(ar, ai, rr, ri); + } + return complex_pow_finish(ar, ai, pr, pi); + } + let base_mag = ar.hypot(ai); + let base_arg = ai.atan2(ar); + let log_mag = base_mag.ln(); + let new_log = br * log_mag - bi * base_arg; + let new_arg = br * base_arg + bi * log_mag; + let new_mag = new_log.exp(); + complex_pow_finish(ar, ai, new_mag * new_arg.cos(), new_mag * new_arg.sin()) +} + +/// CPython's `_Py_ADJUST_ERANGE2` for `_Py_c_pow`: a non-finite result +/// produced from a finite base is a genuine magnitude overflow → +/// `OverflowError`, matching `complex_pow`'s `errno == ERANGE` check. +fn complex_pow_finish(ar: f64, ai: f64, re: f64, im: f64) -> Result<(f64, f64), RuntimeError> { + if (re.is_infinite() || im.is_infinite()) && ar.is_finite() && ai.is_finite() { + return Err(overflow_error("complex exponentiation")); + } + Ok((re, im)) +} + fn complex_arith( (ar, ai): (f64, f64), (br, bi): (f64, f64), @@ -10791,31 +14620,12 @@ fn complex_arith( B::Sub => Ok(Object::new_complex(ar - br, ai - bi)), B::Mult => Ok(Object::new_complex(ar * br - ai * bi, ar * bi + ai * br)), B::Div => { - let denom = br * br + bi * bi; - if denom == 0.0 { - return Err(zero_division_error("complex division by zero")); - } - Ok(Object::new_complex( - (ar * br + ai * bi) / denom, - (ai * br - ar * bi) / denom, - )) + let (re, im) = complex_div(ar, ai, br, bi)?; + Ok(Object::new_complex(re, im)) } B::Pow => { - // Approximate via polar: r^n * cis(n*θ). Pure real - // exponent is the common case; fall back to numerics. - let base_mag = (ar * ar + ai * ai).sqrt(); - let base_arg = ai.atan2(ar); - let exp_re = br; - let exp_im = bi; - let log_mag = base_mag.ln(); - // (a + bi)^(c + di) = exp((c + di) * (log_mag + i*arg)) - let new_log = exp_re * log_mag - exp_im * base_arg; - let new_arg = exp_re * base_arg + exp_im * log_mag; - let new_mag = new_log.exp(); - Ok(Object::new_complex( - new_mag * new_arg.cos(), - new_mag * new_arg.sin(), - )) + let (re, im) = complex_pow(ar, ai, br, bi)?; + Ok(Object::new_complex(re, im)) } _ => Err(type_error(format!( "unsupported operand type(s) for {}: 'complex' and 'complex'", @@ -10834,6 +14644,14 @@ fn compare_op(a: &Object, b: &Object, op: CompareKind) -> Result Ok(a.eq_value(b)), CompareKind::NotEq => Ok(!a.eq_value(b)), + // Ordering against a NaN is always false in CPython (NaN is + // unordered), rather than the `ValueError` that `Object::cmp` + // returns for an undefined total order. + CompareKind::Lt | CompareKind::LtE | CompareKind::Gt | CompareKind::GtE + if is_nan_value(a) || is_nan_value(b) => + { + Ok(false) + } CompareKind::Lt => Ok(a.cmp(b)?.is_lt()), CompareKind::LtE => Ok(a.cmp(b)?.is_le()), CompareKind::Gt => Ok(a.cmp(b)?.is_gt()), @@ -10841,6 +14659,16 @@ fn compare_op(a: &Object, b: &Object, op: CompareKind) -> Result bool { + match o { + Object::Float(f) => f.is_nan(), + Object::Instance(_) => o.native_value().as_ref().is_some_and(is_nan_value), + _ => false, + } +} + /// Snapshot a ``set``/``frozenset`` payload for subset comparison. /// Returns ``None`` for any non-set so the caller can fall through /// to the generic comparison path. diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 0d5bf39..9b96fa3 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -519,9 +519,69 @@ impl fmt::Debug for PyModule { #[derive(Clone, Debug)] pub struct DictKey(pub Object); +/// Reach for the running interpreter to compute a user instance's Python +/// `__hash__`. `DictKey`'s `Hash`/`Eq` impls have no interpreter handle, so +/// they borrow the thread's published interpreter pointer — the same bridge +/// `_imp`/`_thread`/the C-API iterator use. Returns `None` when no +/// interpreter is active (e.g. a dict built from pure-Rust setup), so the +/// caller falls back to the native structural behaviour. +fn current_interp_hash(obj: &Object) -> Option { + let ptr = crate::vm_singletons::current_interpreter_ptr()?; + // SAFETY: the pointer is published by the bytecode dispatch loop for the + // running thread and used only to re-enter the interpreter synchronously, + // mirroring the established reentrant-callback pattern in `_imp`/`_thread`. + let interp = unsafe { &mut *ptr }; + interp.reentrant_py_hash(obj) +} + +/// Companion to [`current_interp_hash`] for `a == b` via Python `__eq__`. +fn current_interp_eq(a: &Object, b: &Object) -> Option { + let ptr = crate::vm_singletons::current_interpreter_ptr()?; + // SAFETY: see `current_interp_hash`. + let interp = unsafe { &mut *ptr }; + interp.reentrant_py_eq(a, b) +} + +/// True when `obj` is a user instance whose class supplies a *callable* +/// `name` dunder (a real Python `def`, not the inherited identity default). +/// Used to gate the reentrant `__eq__` dispatch so plain instances keep the +/// native identity fast path. +fn instance_has_custom_dunder(obj: &Object, name: &str) -> bool { + matches!( + obj, + Object::Instance(inst) + if matches!( + inst.class.lookup(name), + Some(Object::Function(_) | Object::BoundMethod(_)) + ) + ) +} + impl PartialEq for DictKey { fn eq(&self, other: &Self) -> bool { - self.0.eq_value(&other.0) + // CPython compares dict/set keys with `a is b or a == b`; the identity + // half makes a stored `nan` findable by itself (`{nan}` contains its + // own nan even though `nan != nan`). + if self.0.is_same(&other.0) { + return true; + } + // Native fast path also covers instance *identity* (`Rc::ptr_eq`), + // which is the `a is b` half of CPython's dict-key comparison. + if self.0.eq_value(&other.0) { + return true; + } + // Distinct user instances with a custom `__eq__` compare through it + // so a class defining `__eq__`/`__hash__` works as a `set`/`dict` + // key. Plain instances (no custom `__eq__`) keep identity semantics, + // already decided by the `eq_value` fast path above. + if instance_has_custom_dunder(&self.0, "__eq__") + || instance_has_custom_dunder(&other.0, "__eq__") + { + if let Some(eq) = current_interp_eq(&self.0, &other.0) { + return eq; + } + } + false } } @@ -529,71 +589,18 @@ impl Eq for DictKey {} impl Hash for DictKey { fn hash(&self, state: &mut H) { - // An `int`/`str`/… subclass instance hashes identically to the - // value it wraps, so it can be used interchangeably with that - // value as a dict/set key (CPython invariant). - if let Some(native) = self.0.native_value() { - return DictKey(native).hash(state); - } - match &self.0 { - Object::None => 0u8.hash(state), - Object::Bool(b) => { - 1u8.hash(state); - b.hash(state); - } - Object::Int(i) => { - // Hash compatibly with Long: route through the - // BigInt hash so `Int(0).hash() == Long(0).hash()`. - 2u8.hash(state); - python_int_hash_i64(*i).hash(state); - } - Object::Long(b) => { - 2u8.hash(state); - python_int_hash_bigint(b).hash(state); - } - Object::Float(f) => { - 3u8.hash(state); - if f.fract() == 0.0 && f.is_finite() { - // For values representable as integers, hash - // through the int path so `1 == 1.0` implies - // `hash(1) == hash(1.0)`. - if let Some(as_int) = f64_to_i64_exact(*f) { - 2u8.hash(state); - python_int_hash_i64(as_int).hash(state); - } else { - f.to_bits().hash(state); - } - } else { - f.to_bits().hash(state); - } - } - Object::Complex(c) => { - // Pure-real complex hashes as the underlying float; - // imaginary component contributes a separate - // factor (CPython: `hash(complex) = hash(real) ^ - // (hash(imag) * IMAG)`). Constant good-enough. - 3u8.hash(state); - c.real.to_bits().hash(state); - c.imag.to_bits().hash(state); - } - Object::Str(s) => { - 4u8.hash(state); - s.hash(state); - } - Object::Tuple(items) => { - 5u8.hash(state); - items.len().hash(state); - for x in items.iter() { - DictKey(x.clone()).hash(state); - } - } - _ => { - // Unhashable types — hash to a constant. Python would - // raise TypeError; we keep it well-defined for now and - // let the runtime raise lazily when this key is used. - 255u8.hash(state); - } - } + // Bucket every key by its single canonical Python hash value, so any + // two keys Python deems equal-and-hashable collide here regardless of + // their Rust representation: equal numeric types (`1 == 1.0 == True`), + // an `int`/`str`/… subclass and its wrapped value, and — crucially — + // a custom `__hash__` that returns a built-in value (e.g. + // `hash('halibut')`) and the string itself. `DictKey::eq` then decides + // actual equality within the bucket. Identity-hashable objects + // (functions, types, plain instances, …) fold in their allocation + // identity; truly unhashable keys share a constant bucket and the + // runtime raises lazily when used. + let h = py_hash_value(&self.0).unwrap_or_else(|| identity_hash(&self.0)); + h.hash(state); } } @@ -1099,6 +1106,14 @@ pub enum PyIterator { data: Rc<[u8]>, index: usize, }, + /// Lazy `enumerate(...)`. Holds a *shared* handle to the wrapped + /// iterator so consuming the enumerate also advances the original + /// (CPython: `enumerate(it)` yields from the same `it`, leaving it + /// positioned right after the last item produced). + Enumerate { + inner: Rc>, + count: i64, + }, } impl PyIterator { @@ -1156,6 +1171,44 @@ impl PyIterator { *index += 1; Some(Object::Int(i64::from(v))) } + PyIterator::Enumerate { inner, count } => { + let v = inner.borrow_mut().next_value()?; + let i = *count; + *count += 1; + Some(Object::new_tuple(vec![Object::Int(i), v])) + } + } + } + + /// Number of items remaining, when cheaply known. Backs the + /// `__length_hint__` slot CPython's built-in iterators expose + /// (`operator.length_hint`, list pre-sizing, …). Returns `None` + /// for sources whose remaining length isn't known in O(1). + pub fn remaining(&self) -> Option { + match self { + PyIterator::List { items, index } => { + Some(items.borrow().len().saturating_sub(*index)) + } + PyIterator::Tuple { items, index } => Some(items.len().saturating_sub(*index)), + PyIterator::Str { s, index } => Some(s[(*index).min(s.len())..].chars().count()), + PyIterator::DictKeys { keys, index } => Some(keys.len().saturating_sub(*index)), + PyIterator::Bytes { data, index } => Some(data.len().saturating_sub(*index)), + PyIterator::Enumerate { inner, .. } => inner.borrow().remaining(), + PyIterator::Range { + current, + stop, + step, + } => { + if *step > 0 && *current < *stop { + Some((((*stop - *current) as i128 + i128::from(*step) - 1) + / i128::from(*step)) as usize) + } else if *step < 0 && *current > *stop { + Some((((*current - *stop) as i128 + i128::from(-*step) - 1) + / i128::from(-*step)) as usize) + } else { + Some(0) + } + } } } } @@ -1323,7 +1376,7 @@ impl Object { } (Object::Float(a), Object::Float(b)) => a == b, (Object::Int(a), Object::Float(b)) | (Object::Float(b), Object::Int(a)) => { - (*a as f64) == *b + i64_eq_f64(*a, *b) } (Object::Long(a), Object::Float(b)) | (Object::Float(b), Object::Long(a)) => { bigint_eq_f64(a, *b) @@ -1333,7 +1386,7 @@ impl Object { } (Object::Complex(a), Object::Complex(b)) => a.real == b.real && a.imag == b.imag, (Object::Complex(c), Object::Int(i)) | (Object::Int(i), Object::Complex(c)) => { - c.imag == 0.0 && c.real == (*i as f64) + c.imag == 0.0 && i64_eq_f64(*i, c.real) } (Object::Complex(c), Object::Float(f)) | (Object::Float(f), Object::Complex(c)) => { c.imag == 0.0 && c.real == *f @@ -1342,13 +1395,21 @@ impl Object { c.imag == 0.0 && bigint_eq_f64(b, c.real) } (Object::Str(a), Object::Str(b)) => a == b, + // Sequence comparison is element-wise `PyObject_RichCompareBool`, + // which is identity-first — so `[nan] == [nan]` (same nan) is true. (Object::Tuple(a), Object::Tuple(b)) => { - a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.eq_value(y)) + a.len() == b.len() + && a.iter() + .zip(b.iter()) + .all(|(x, y)| x.is_same(y) || x.eq_value(y)) } (Object::List(a), Object::List(b)) => { let a = a.borrow(); let b = b.borrow(); - a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.eq_value(y)) + a.len() == b.len() + && a.iter() + .zip(b.iter()) + .all(|(x, y)| x.is_same(y) || x.eq_value(y)) } (Object::Dict(a), Object::Dict(b)) => { let a = a.borrow(); @@ -1374,6 +1435,14 @@ impl Object { (Object::Set(a), Object::FrozenSet(b)) | (Object::FrozenSet(b), Object::Set(a)) => { sets_equal(&a.borrow(), b) } + // `slice` objects compare as the `(start, stop, step)` triple + // (CPython's `slice_richcompare`), identity-first per field so + // `slice(None)` fields (NaN-free here, but consistent) match. + (Object::Slice(a), Object::Slice(b)) => { + (a.start.is_same(&b.start) || a.start.eq_value(&b.start)) + && (a.stop.is_same(&b.stop) || a.stop.eq_value(&b.stop)) + && (a.step.is_same(&b.step) || a.step.eq_value(&b.step)) + } // Reference-identity equality for class / module / function // / builtin / method values. CPython falls back to identity // here, and our `in` / dict-key checks rely on it. @@ -1382,6 +1451,13 @@ impl Object { (Object::Function(a), Object::Function(b)) => Rc::ptr_eq(a, b), (Object::Builtin(a), Object::Builtin(b)) => Rc::ptr_eq(a, b), (Object::Instance(a), Object::Instance(b)) => Rc::ptr_eq(a, b), + // Bound methods compare like CPython's `method_richcompare`: + // `__func__` by equality, `__self__` by identity. Two freshly + // bound references to the same method on the same object are + // therefore equal even though they're distinct allocations. + (Object::BoundMethod(a), Object::BoundMethod(b)) => { + a.function.eq_value(&b.function) && a.receiver.is_same(&b.receiver) + } _ => false, } } @@ -1407,12 +1483,8 @@ impl Object { (O::Float(a), O::Float(b)) => Ok(a .partial_cmp(b) .ok_or_else(|| value_error(format!("cannot order {a} and {b} (NaN)")))?), - (O::Int(a), O::Float(b)) => Ok((*a as f64) - .partial_cmp(b) - .ok_or_else(|| value_error("cannot order with NaN"))?), - (O::Float(a), O::Int(b)) => Ok(a - .partial_cmp(&(*b as f64)) - .ok_or_else(|| value_error("cannot order with NaN"))?), + (O::Int(a), O::Float(b)) => i64_cmp_f64(*a, *b), + (O::Float(a), O::Int(b)) => Ok(i64_cmp_f64(*b, *a)?.reverse()), (O::Long(a), O::Float(b)) => Ok(bigint_cmp_f64(a, *b)?), (O::Float(a), O::Long(b)) => Ok(bigint_cmp_f64(b, *a)?.reverse()), (O::Bool(a), O::Bool(b)) => Ok(a.cmp(b)), @@ -1444,8 +1516,13 @@ impl Object { /// Membership: `x in container`. pub fn contains(&self, item: &Self) -> Result { match self { - Object::Tuple(items) => Ok(items.iter().any(|x| x.eq_value(item))), - Object::List(items) => Ok(items.borrow().iter().any(|x| x.eq_value(item))), + // CPython's `PyObject_RichCompareBool` short-circuits on identity + // before `==`, so `nan in [nan]` (the *same* nan) is `True`. + Object::Tuple(items) => Ok(items.iter().any(|x| x.is_same(item) || x.eq_value(item))), + Object::List(items) => Ok(items + .borrow() + .iter() + .any(|x| x.is_same(item) || x.eq_value(item))), Object::Str(haystack) => match item { Object::Str(needle) => Ok(haystack.contains(&**needle)), _ => Err(type_error( @@ -1708,13 +1785,7 @@ impl Object { Object::Int(i) => i.to_string(), Object::Long(b) => b.to_string(), Object::Complex(c) => complex_repr(c.real, c.imag), - Object::Float(f) => { - if f.fract() == 0.0 && f.is_finite() { - format!("{f:.1}") - } else { - f.to_string() - } - } + Object::Float(f) => float_repr(*f), Object::Str(s) => { // CPython quote selection (Objects/unicodeobject.c // `unicode_repr`): use '\'' unless the string contains a @@ -1872,8 +1943,11 @@ impl Object { } } Object::Property(_) => "".to_owned(), - Object::StaticMethod(_) => "".to_owned(), - Object::ClassMethod(_) => "".to_owned(), + // CPython 3.10+: `)>` — the + // wrapped callable's repr is embedded so the address matches + // `'{!r}'.format(func)`. + Object::StaticMethod(inner) => format!("", inner.repr()), + Object::ClassMethod(inner) => format!("", inner.repr()), Object::SlotDescriptor(sd) => { format!("", sd.name, sd.class_name) } @@ -1941,6 +2015,15 @@ impl Object { Object::MemoryView(mv) => Ok(mv.len.get()), Object::MappingProxy(d) => Ok(d.borrow().len()), Object::DictView(v) => Ok(v.dict.borrow().len()), + // A subclass of a built-in container (`class C(list)`, …) + // measures the length of the native payload it wraps. + Object::Instance(inst) => match &inst.native { + Some(native) => native.len(), + None => Err(type_error(format!( + "object of type '{}' has no len()", + self.type_name() + ))), + }, _ => Err(type_error(format!( "object of type '{}' has no len()", self.type_name() @@ -1996,6 +2079,62 @@ pub(crate) fn bigint_eq_f64(a: &BigInt, b: f64) -> bool { *a == bi } +/// Smallest power of two that is *not* exactly representable beyond the +/// f64 integer-precision boundary; `2f64.powi(63)` as a literal so the +/// i64-range checks below stay branch-cheap. +const TWO_POW_63: f64 = 9_223_372_036_854_775_808.0; + +/// Exact `i64 == f64`. A plain `a as f64 == b` loses precision for +/// `|a| > 2**53`, making e.g. `float(2**53 + 1) == 2**53 + 1` wrongly +/// `True`. CPython compares an int and a float *exactly*; this mirrors +/// that without allocating a `BigInt` for the common in-range case. +pub(crate) fn i64_eq_f64(a: i64, b: f64) -> bool { + if !b.is_finite() || b.fract() != 0.0 { + return false; + } + // `b` is integral; it can equal an `i64` only inside `[-2**63, 2**63)`. + if (-TWO_POW_63..TWO_POW_63).contains(&b) { + (b as i64) == a + } else { + false + } +} + +/// Exact `i64` vs `f64` ordering (see [`i64_eq_f64`]). +pub(crate) fn i64_cmp_f64(a: i64, b: f64) -> Result { + if b.is_nan() { + return Err(value_error("cannot order with NaN")); + } + if b == f64::INFINITY { + return Ok(Ordering::Less); + } + if b == f64::NEG_INFINITY { + return Ok(Ordering::Greater); + } + let trunc = b.trunc(); + if (-TWO_POW_63..TWO_POW_63).contains(&trunc) { + let ti = trunc as i64; + match a.cmp(&ti) { + Ordering::Equal => { + let frac = b - trunc; + if frac == 0.0 { + Ok(Ordering::Equal) + } else if frac > 0.0 { + Ok(Ordering::Less) + } else { + Ok(Ordering::Greater) + } + } + other => Ok(other), + } + } else if trunc > 0.0 { + // |b| ≥ 2**63 is larger in magnitude than any i64. + Ok(Ordering::Less) + } else { + Ok(Ordering::Greater) + } +} + /// CPython's hash for ints: `value mod (2**61 - 1)` for 64-bit /// platforms. This keeps `hash(1) == hash(1.0) == hash(True)` and /// also `hash(big) == hash(int(big))` for any big int. @@ -2017,6 +2156,263 @@ pub(crate) fn python_int_hash_bigint(value: &BigInt) -> i64 { rem.to_i64().unwrap_or(0) } +/// Width of the Python numeric hash reduction: `_PyHASH_BITS` (61 on +/// 64-bit, so the modulus is the Mersenne prime `2**61 - 1`). +const PY_HASH_BITS: u32 = 61; +/// `sys.hash_info.inf` — the hash of `±inf` (CPython `_PyHASH_INF`). +pub(crate) const PY_HASH_INF: i64 = 314_159; +/// `sys.hash_info.imag` — the multiplier for a complex's imaginary part. +const PY_HASH_IMAG: u64 = 1_000_003; + +/// C `frexp`: split `x` into `(m, e)` with `x == m * 2**e` and +/// `0.5 <= |m| < 1` (or `m == 0`). Handles subnormals; callers guard +/// against non-finite inputs. +fn py_frexp(x: f64) -> (f64, i32) { + if x == 0.0 || !x.is_finite() { + return (x, 0); + } + let bits = x.to_bits(); + let raw_exp = ((bits >> 52) & 0x7ff) as i32; + if raw_exp == 0 { + // Subnormal: scale into the normal range (× 2**54), then correct. + let (m, e) = py_frexp(x * 18_014_398_509_481_984.0_f64); + return (m, e - 54); + } + // Normal value = ±(1.frac) * 2**(raw_exp-1023). Forcing the stored + // exponent field to 1022 (factor 2**-1) yields a mantissa in [0.5, 1); + // the true binary exponent is then `raw_exp - 1022`. + let e = raw_exp - 1022; + let m = f64::from_bits((bits & 0x800f_ffff_ffff_ffff) | (1022u64 << 52)); + (m, e) +} + +/// CPython `_Py_HashDouble`: the canonical hash of a finite double via +/// reduction modulo `2**61 - 1`, so an integer-valued float hashes equal +/// to the corresponding `int` and a `Fraction`/`Decimal` of equal value. +pub(crate) fn py_hash_double(v: f64) -> i64 { + const MOD: u64 = (1u64 << PY_HASH_BITS) - 1; + if !v.is_finite() { + if v.is_infinite() { + return if v > 0.0 { PY_HASH_INF } else { -PY_HASH_INF }; + } + // NaN. CPython 3.10+ uses the object's identity; for value hashing + // 0 is a stable, collision-tolerant choice (matches sys.hash_info.nan). + return 0; + } + let (mut m, mut e) = py_frexp(v); + let sign: i64 = if m < 0.0 { + m = -m; + -1 + } else { + 1 + }; + // Accumulate 28 bits of mantissa at a time, rotating left within the + // 61-bit field (mirrors the C loop exactly). + let mut x: u64 = 0; + while m != 0.0 { + x = ((x << 28) & MOD) | (x >> (PY_HASH_BITS - 28)); + m *= 268_435_456.0; // 2**28 + e -= 28; + let y = m as u64; + m -= y as f64; + x += y; + if x >= MOD { + x -= MOD; + } + } + // Fold in the leftover power of two via a 61-bit rotate. + let mut e = e % (PY_HASH_BITS as i32); + if e < 0 { + e += PY_HASH_BITS as i32; + } + let e = e as u32; + x = ((x << e) & MOD) | (x >> (PY_HASH_BITS - e)); + let mut res = (x as i64) * sign; + if res == -1 { + res = -2; + } + res +} + +/// CPython `long_hash` for a machine int: `sign * (|n| mod (2**61-1))`, +/// with the reserved `-1` remapped to `-2`. +pub(crate) fn py_hash_long_i64(n: i64) -> i64 { + const MOD: u128 = (1u128 << PY_HASH_BITS) - 1; + let mut x = ((n as i128).unsigned_abs() % MOD) as i64; + if n < 0 { + x = -x; + } + if x == -1 { + x = -2; + } + x +} + +/// CPython `long_hash` for a big int. `BigInt %` is truncating, so it +/// already carries the dividend's sign with magnitude `|n| mod P`. +pub(crate) fn py_hash_long_bigint(value: &BigInt) -> i64 { + let modulus = BigInt::from((1u64 << PY_HASH_BITS) - 1); + let rem = value % &modulus; + let mut x = rem.to_i64().unwrap_or(0); + if x == -1 { + x = -2; + } + x +} + +/// CPython `complex_hash`: `hash(real) + _PyHASH_IMAG * hash(imag)` in +/// wrapping (mod 2**64) arithmetic, with `-1` remapped to `-2`. A +/// zero-imaginary complex therefore hashes equal to the bare float. +pub(crate) fn py_hash_complex(re: f64, im: f64) -> i64 { + let hr = py_hash_double(re) as u64; + let hi = py_hash_double(im) as u64; + let combined = hr.wrapping_add(PY_HASH_IMAG.wrapping_mul(hi)); + let res = combined as i64; + if res == -1 { + -2 + } else { + res + } +} + +/// Exact CPython `hash()` for the built-in numeric types, so that equal +/// values across `bool`/`int`/`float`/`complex` (and the pure-Python +/// `Fraction`/`Decimal`, which implement the same reduction) all agree. +/// Returns `None` for non-numeric objects. +pub(crate) fn numeric_hash(obj: &Object) -> Option { + match obj { + Object::Bool(b) => Some(py_hash_long_i64(i64::from(*b))), + Object::Int(i) => Some(py_hash_long_i64(*i)), + Object::Long(b) => Some(py_hash_long_bigint(b)), + Object::Float(f) => Some(py_hash_double(*f)), + Object::Complex(c) => Some(py_hash_complex(c.real, c.imag)), + _ => None, + } +} + +/// `hash(None)` — CPython 3.12 returns this fixed constant rather than a +/// pointer-derived value. +const PY_HASH_NONE: i64 = 0xFCA8_6420; + +/// Deterministic structural hash for a byte slice (backs both `str` and +/// `bytes`). CPython randomises string hashing per process via SipHash, so +/// we don't need to reproduce its exact output — only to be stable within a +/// run so equal strings bucket together. `hash("") == hash(b"") == 0`, +/// matching CPython, and the reserved `-1` is remapped to `-2`. +fn py_hash_bytes_slice(bytes: &[u8]) -> i64 { + if bytes.is_empty() { + return 0; + } + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + bytes.hash(&mut h); + let v = h.finish() as i64; + if v == -1 { + -2 + } else { + v + } +} + +/// Identity-based hash for objects that hash by allocation identity in +/// CPython (functions, types, modules, plain instances without a custom +/// `__hash__`, …). Mirrors CPython's pointer hash: rotate so the low +/// alignment zero-bits don't waste bucket entropy, remapping `-1` to `-2`. +pub(crate) fn identity_hash(obj: &Object) -> i64 { + fn rot(p: *const ()) -> i64 { + let u = p as usize as u64; + let v = (u >> 4 | u << 60) as i64; + if v == -1 { + -2 + } else { + v + } + } + match obj { + Object::Function(r) => rot(Rc::as_ptr(r).cast()), + Object::Builtin(r) => rot(Rc::as_ptr(r).cast()), + Object::BoundMethod(r) => rot(Rc::as_ptr(r).cast()), + Object::Code(r) => rot(Rc::as_ptr(r).cast()), + Object::Cell(r) => rot(Rc::as_ptr(r).cast()), + Object::Iter(r) => rot(Rc::as_ptr(r).cast()), + Object::Slice(r) => rot(Rc::as_ptr(r).cast()), + Object::Type(r) => rot(Rc::as_ptr(r).cast()), + Object::Instance(r) => rot(Rc::as_ptr(r).cast()), + Object::Module(r) => rot(Rc::as_ptr(r).cast()), + Object::Generator(r) | Object::Coroutine(r) | Object::AsyncGenerator(r) => { + rot(Rc::as_ptr(r).cast()) + } + Object::File(r) => rot(Rc::as_ptr(r).cast()), + Object::Property(r) => rot(Rc::as_ptr(r).cast()), + Object::StaticMethod(r) => rot(Rc::as_ptr(r).cast()), + Object::ClassMethod(r) => rot(Rc::as_ptr(r).cast()), + Object::SlotDescriptor(r) => rot(Rc::as_ptr(r).cast()), + Object::Frame(r) => rot(Rc::as_ptr(r).cast()), + Object::Traceback(r) => rot(Rc::as_ptr(r).cast()), + Object::MemoryView(r) => rot(Rc::as_ptr(r).cast()), + Object::SimpleNamespace(r) => rot(Rc::as_ptr(r).cast()), + // Value-hashable variants never reach here (handled by + // `py_hash_value`); anything else gets a stable constant. + _ => 0, + } +} + +/// Canonical Python `hash(obj)` value, shared by the `hash()` builtin and +/// the [`DictKey`] hasher. Bucketing every key by this single value (rather +/// than a type-tagged structural hash) is what lets objects Python considers +/// equal-and-hashable collide regardless of Rust representation — e.g. a +/// custom `__hash__` returning `hash('halibut')` buckets with the actual +/// string, so a `set`/`dict` can dedup them via [`DictKey::eq`]. +/// +/// Returns `None` for objects with no *value* hash (identity-hashable or +/// unhashable); callers fall back to [`identity_hash`]. +pub(crate) fn py_hash_value(obj: &Object) -> Option { + if let Some(h) = numeric_hash(obj) { + return Some(h); + } + match obj { + Object::None => Some(PY_HASH_NONE), + Object::Str(s) => Some(py_hash_bytes_slice(s.as_bytes())), + Object::Bytes(b) => Some(py_hash_bytes_slice(b)), + Object::Tuple(items) => { + // Order-sensitive mix (FNV-style) over element hashes so equal + // tuples bucket together; unhashable elements would raise at the + // `hash()` builtin, here they just fold their identity in. + let mut acc: u64 = 0x345678; + for x in items.iter() { + let eh = py_hash_value(x).unwrap_or_else(|| identity_hash(x)) as u64; + acc = (acc ^ eh).wrapping_mul(1_000_003).wrapping_add(items.len() as u64); + } + let v = acc as i64; + Some(if v == -1 { -2 } else { v }) + } + Object::FrozenSet(s) => { + // Order-independent: xor scrambled element hashes (mirrors the + // key property of CPython's `frozenset_hash`). + let mut acc: u64 = 0; + for k in s.iter() { + let eh = py_hash_value(&k.0).unwrap_or_else(|| identity_hash(&k.0)) as u64; + acc ^= eh + .wrapping_mul(89_869_747) + .wrapping_add(0x2545_F491_4F6C_DD1D); + } + let v = acc as i64; + Some(if v == -1 { -2 } else { v }) + } + Object::Instance(inst) => { + if let Some(native) = &inst.native { + // int/str/… subclass instance hashes as the wrapped value. + return py_hash_value(native); + } + // Custom `__hash__` via the interpreter; `None` (no active + // interpreter or only the inherited identity hash) falls through + // to `identity_hash` at the call site. + current_interp_hash(obj) + } + _ => None, + } +} + pub(crate) fn f64_to_i64_exact(f: f64) -> Option { if !f.is_finite() { return None; @@ -2055,21 +2451,94 @@ pub(crate) fn bigint_from_f64_trunc(f: f64) -> BigInt { /// Render a `complex` the way CPython does: bare `Xj` if real is /// zero, `(R+Ij)` / `(R-Ij)` otherwise. Special-cases `nan` and /// signed zeros to match CPython's `repr` exactly. -pub(crate) fn complex_repr(real: f64, imag: f64) -> String { - fn fmt_part(p: f64) -> String { - // Unlike `float`, CPython renders integer-valued complex - // components without a trailing `.0` (e.g. `4j`, not `4.0j`). - if p.fract() == 0.0 && p.is_finite() { - format!("{p:.0}") - } else { - format!("{p}") +/// CPython-compatible `repr(float)` — the shortest decimal string that +/// round-trips, switching to exponential notation exactly when CPython +/// does (`decpt <= -4 || decpt > 16`, i.e. magnitudes below 1e-4 or at +/// or above 1e16). Mirrors `float_repr` / +/// `PyOS_double_to_string(v, 'r', 0, Py_DTSF_ADD_DOT_0, ...)`. +/// +/// Rust's `f64::to_string()` is *also* shortest-round-trip, but never +/// uses exponential form, so `1e100` would otherwise print as a 101-digit +/// integer. We recover the shortest digits + decimal exponent from +/// `{:e}` (Ryū) and reassemble them under CPython's rules. +pub(crate) fn float_repr(f: f64) -> String { + if f.is_nan() { + return "nan".to_owned(); + } + if f.is_infinite() { + return if f < 0.0 { "-inf" } else { "inf" }.to_owned(); + } + if f == 0.0 { + return if f.is_sign_negative() { "-0.0" } else { "0.0" }.to_owned(); + } + let neg = f.is_sign_negative(); + let a = f.abs(); + let sci = format!("{a:e}"); + let (mant, exp_str) = sci.split_once('e').expect("scientific form has 'e'"); + let exp: i32 = exp_str.parse().expect("valid exponent"); + let digits: String = mant.chars().filter(|c| *c != '.').collect(); + let ndigits = digits.len() as i32; + let decpt = exp + 1; // count of digits left of the decimal point + let body = if decpt <= -4 || decpt > 16 { + let e = decpt - 1; + let mut s = digits[..1].to_owned(); + if digits.len() > 1 { + s.push('.'); + s.push_str(&digits[1..]); + } + s.push('e'); + s.push(if e < 0 { '-' } else { '+' }); + s.push_str(&format!("{:02}", e.unsigned_abs())); + s + } else if decpt <= 0 { + let mut s = String::from("0."); + for _ in 0..(-decpt) { + s.push('0'); } + s.push_str(&digits); + s + } else if decpt >= ndigits { + let mut s = digits.clone(); + for _ in 0..(decpt - ndigits) { + s.push('0'); + } + s.push_str(".0"); + s + } else { + let d = decpt as usize; + format!("{}.{}", &digits[..d], &digits[d..]) + }; + if neg { + format!("-{body}") + } else { + body + } +} + +/// `repr`-shortest rendering of a single complex component. Unlike +/// `float`, CPython renders integer-valued complex components without a +/// trailing `.0` (e.g. `4j`, not `4.0j`), but otherwise uses the same +/// shortest/exponential rules. +pub(crate) fn complex_component_repr(p: f64) -> String { + let r = float_repr(p); + match r.strip_suffix(".0") { + Some(stripped) => stripped.to_owned(), + None => r, } +} + +pub(crate) fn complex_repr(real: f64, imag: f64) -> String { + let fmt_part = complex_component_repr; if real == 0.0 && real.is_sign_positive() { format!("{}j", fmt_part(imag)) } else { - let sep = if imag.is_sign_negative() { "" } else { "+" }; - format!("({}{sep}{}j)", fmt_part(real), fmt_part(imag)) + // Insert the joining sign based on the *rendered* imaginary part, + // not its raw sign bit: `-nan` keeps a set sign bit yet renders as + // "nan" (no leading '-'), so CPython prints `(nan+nanj)`, and a + // genuine negative like -2.0 renders "-2" and needs no extra '+'. + let im = fmt_part(imag); + let sep = if im.starts_with('-') { "" } else { "+" }; + format!("({}{sep}{im}j)", fmt_part(real)) } } @@ -2320,12 +2789,13 @@ impl Object { } } - /// Try to view this value as bytes (works for both `bytes` and - /// `bytearray`). Returns `None` for any other type. + /// Try to view this value as bytes (works for `bytes`, `bytearray`, + /// and contiguous `memoryview`). Returns `None` for any other type. pub fn as_bytes_view(&self) -> Option> { match self { Object::Bytes(b) => Some(b.to_vec()), Object::ByteArray(b) => Some(b.borrow().clone()), + Object::MemoryView(mv) => Some(mv.to_bytes()), _ => None, } } diff --git a/crates/weavepy-vm/src/recursion.rs b/crates/weavepy-vm/src/recursion.rs new file mode 100644 index 0000000..03d13c9 --- /dev/null +++ b/crates/weavepy-vm/src/recursion.rs @@ -0,0 +1,169 @@ +//! Python-level recursion guard — RFC 0037 (WS1). +//! +//! WeavePy's evaluator is a recursive tree-walker: every Python call +//! activation (`run_until_yield_or_return`) maps onto a native (Rust) +//! stack frame. Without a guard, unbounded Python recursion overflows +//! the native stack and `abort()`s the process (the failure mode RFC +//! 0036 hit on `test_exceptions`). +//! +//! CPython instead raises `RecursionError` once Python call depth +//! crosses `sys.setrecursionlimit` (default 1000). The `weavepy-cli` +//! build reserves enough main-thread stack (8 MiB on Linux/macOS, an +//! explicit 64 MiB reserve on Windows) that the *limit* is reached well +//! before the native stack, so enforcing the limit here is what makes +//! deep recursion fail cleanly and uniformly across platforms. +//! +//! This module owns the process-wide limit (CPython's limit is global — +//! `setrecursionlimit` affects every thread) and a per-thread depth +//! counter, plus a small RAII [`Guard`] the dispatch loop holds so the +//! depth is restored on *every* exit path (return, yield, exception). +//! +//! ## Why we raise on *every* over-limit call +//! +//! CPython raises `RecursionError` on every activation attempted past the +//! limit; its "recursion headroom" is a count of how many times the error +//! machinery may itself recurse before a *fatal* abort — it is **not** a +//! block of extra frames a program may freely use. An earlier design here +//! tolerated 50 free frames once the limit was first exceeded, re-arming +//! the allowance whenever depth dipped back under the ceiling. That turns +//! a function which recurses in *both* its body and its `except` handler +//! (`test_exceptions.test_recursion_in_except_handler`) into an +//! exponential blowup: each partial unwind frees a frame the handler +//! immediately re-consumes, so the stack never actually drains. Raising at +//! the limit every time makes such teardown linear, exactly like CPython. + +use std::cell::Cell; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Default `sys.getrecursionlimit()`. +pub const DEFAULT_RECURSION_LIMIT: usize = 1000; + +/// Process-wide recursion limit. +static RECURSION_LIMIT: AtomicUsize = AtomicUsize::new(DEFAULT_RECURSION_LIMIT); + +thread_local! { + /// Live Python call depth on *this* thread. + static DEPTH: Cell = const { Cell::new(0) }; +} + +/// Current process-wide recursion limit (`sys.getrecursionlimit()`). +pub fn recursion_limit() -> usize { + RECURSION_LIMIT.load(Ordering::Relaxed) +} + +/// Live Python call depth on the calling thread. +pub fn current_depth() -> usize { + DEPTH.with(|d| d.get()) +} + +/// Set a new process-wide limit. Returns `Err(current_depth)` if the +/// requested limit isn't strictly above the calling thread's current +/// depth — CPython raises `RecursionError` in that case so a program +/// can't lower the limit out from under its own live stack. +pub fn set_limit(new_limit: usize) -> Result<(), usize> { + let depth = current_depth(); + if new_limit <= depth { + return Err(depth); + } + RECURSION_LIMIT.store(new_limit, Ordering::Relaxed); + Ok(()) +} + +/// Result of attempting to enter one more activation. +#[derive(Debug)] +pub enum Enter { + /// Proceed; the caller must hold the [`Guard`] for the activation. + Ok(Guard), + /// Limit exceeded — the caller should raise `RecursionError`. + Overflow, +} + +/// RAII handle that restores the per-thread depth on drop. Created only +/// when [`enter`] permits the activation, so the increment and the +/// decrement are always balanced. +#[derive(Debug)] +pub struct Guard { + _private: (), +} + +impl Drop for Guard { + fn drop(&mut self) { + DEPTH.with(|d| d.set(d.get().saturating_sub(1))); + } +} + +/// Enter one Python activation. On [`Enter::Ok`] the returned [`Guard`] +/// must stay alive until the activation finishes; dropping it restores +/// the depth. +/// +/// Returns [`Enter::Overflow`] — and rolls the (un-run) activation back — +/// whenever the new depth would exceed the limit, on *every* such call. +/// See the module docs for why there is no extra-frame headroom. +pub fn enter() -> Enter { + let limit = recursion_limit(); + let depth = DEPTH.with(|d| { + let n = d.get() + 1; + d.set(n); + n + }); + if depth > limit { + DEPTH.with(|d| d.set(d.get().saturating_sub(1))); + return Enter::Overflow; + } + Enter::Ok(Guard { _private: () }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn depth_balances_across_guards() { + assert_eq!(current_depth(), 0); + { + let _g = match enter() { + Enter::Ok(g) => g, + Enter::Overflow => panic!("unexpected overflow"), + }; + assert_eq!(current_depth(), 1); + { + let _g2 = match enter() { + Enter::Ok(g) => g, + Enter::Overflow => panic!("unexpected overflow"), + }; + assert_eq!(current_depth(), 2); + } + assert_eq!(current_depth(), 1); + } + assert_eq!(current_depth(), 0); + } + + #[test] + fn over_limit_raises_every_time_without_inflating_depth() { + // Use a tiny limit on this thread's view via the global atomic. + // (Tests run single-threaded per #[test] body.) + let saved = recursion_limit(); + RECURSION_LIMIT.store(4, Ordering::Relaxed); + + let mut guards = Vec::new(); + for _ in 0..4 { + match enter() { + Enter::Ok(g) => guards.push(g), + Enter::Overflow => panic!("should fit under the limit"), + } + } + assert_eq!(current_depth(), 4); + // Every breach past the limit overflows and leaves depth pinned at + // the limit — no free "headroom" frames a recursing handler could + // re-consume. This is what keeps `recurse_in_body_and_except` + // teardown linear instead of exponential. + for _ in 0..1000 { + assert!(matches!(enter(), Enter::Overflow)); + assert_eq!(current_depth(), 4); + } + + drop(guards); + assert_eq!(current_depth(), 0); + RECURSION_LIMIT.store(saved, Ordering::Relaxed); + } +} diff --git a/crates/weavepy-vm/src/specialize.rs b/crates/weavepy-vm/src/specialize.rs index d0c06db..ae94a7b 100644 --- a/crates/weavepy-vm/src/specialize.rs +++ b/crates/weavepy-vm/src/specialize.rs @@ -319,8 +319,14 @@ fn type_has_attr_override(ty: &Rc) -> bool { if ty.lookup("__getattr__").is_some() { return true; } - if ty.lookup("__getattribute__").is_some() { - return true; + // `object.__getattribute__` lives in `object`'s dict as a sentinel, so a + // bare `is_some()` would match *every* class. Only a genuine user override + // (anything other than that sentinel) should disable the dict-slot fast + // path — the default lookup is exactly what the fast path reproduces. + match ty.lookup("__getattribute__") { + Some(Object::Builtin(b)) if b.name == ".object_getattribute" => {} + Some(_) => return true, + None => {} } if ty.lookup("__setattr__").is_some() { return true; diff --git a/crates/weavepy-vm/src/stdlib/datetime_mod.rs b/crates/weavepy-vm/src/stdlib/datetime_mod.rs index fbe65ca..c4607d4 100644 --- a/crates/weavepy-vm/src/stdlib/datetime_mod.rs +++ b/crates/weavepy-vm/src/stdlib/datetime_mod.rs @@ -248,7 +248,11 @@ fn arg_int(args: &[Object], idx: usize) -> Result { match args.get(idx) { Some(Object::Int(i)) => Ok(*i), Some(Object::Bool(b)) => Ok(i64::from(*b)), - _ => Err(type_error("expected int")), + // Honour the `__index__` protocol so integer-backed instances + // (e.g. `IntEnum` members like `calendar.Month`) are accepted, like + // CPython's `PyDateTime` argument parsing. + Some(o) => crate::builtins::coerce_index_i64(o).map_err(|_| type_error("expected int")), + None => Err(type_error("expected int")), } } @@ -257,7 +261,7 @@ fn arg_int_or(args: &[Object], idx: usize, default: i64) -> Result Ok(default), Some(Object::Int(i)) => Ok(*i), Some(Object::Bool(b)) => Ok(i64::from(*b)), - _ => Err(type_error("expected int")), + Some(o) => crate::builtins::coerce_index_i64(o).map_err(|_| type_error("expected int")), } } diff --git a/crates/weavepy-vm/src/stdlib/io.rs b/crates/weavepy-vm/src/stdlib/io.rs index 59006e8..c995b47 100644 --- a/crates/weavepy-vm/src/stdlib/io.rs +++ b/crates/weavepy-vm/src/stdlib/io.rs @@ -55,13 +55,19 @@ pub fn build(_cache: &ModuleCache) -> Rc { "BufferedWriter", "BufferedRandom", "BufferedRWPair", - "TextIOWrapper", "IncrementalNewlineDecoder", "UnsupportedOperation", ] { let cls = make_io_protocol(name); d.insert(DictKey(Object::from_static(name)), Object::Type(cls)); } + // A functional `TextIOWrapper`: a text layer over a binary buffer + // (e.g. `io.TextIOWrapper(io.BytesIO())`). `write` encodes through to + // the wrapped buffer; `.buffer` exposes it again. + d.insert( + DictKey(Object::from_static("TextIOWrapper")), + Object::Type(make_text_io_wrapper()), + ); } Rc::new(PyModule { name: "io".to_owned(), @@ -133,3 +139,309 @@ fn io_bytesio(args: &[Object]) -> Result { }, )))) } + +// --------------------------------------------------------------------------- +// TextIOWrapper — a text layer over a binary buffer. +// +// `io.TextIOWrapper(buffer, encoding=None, errors=None, newline=None, ...)` +// wraps a binary stream (e.g. `io.BytesIO`) and presents a text interface: +// `write(str)` encodes through to the buffer, `read()` decodes back, and +// `.buffer` re-exposes the wrapped stream. We store the buffer + codec +// settings on the instance `__dict__` so the methods (Rust builtins) can +// recover them from `self`. +// --------------------------------------------------------------------------- + +fn make_text_io_wrapper() -> Rc { + use crate::builtin_types::builtin_types; + use crate::types::{TypeFlags, TypeObject}; + let bt = builtin_types(); + let mut dict = DictData::new(); + let mut method = |name: &'static str, body: fn(&[Object]) -> Result| { + dict.insert(DictKey(Object::from_static(name)), builtin(name, body)); + }; + method("write", tw_write); + method("read", tw_read); + method("readline", tw_readline); + method("flush", tw_flush); + method("close", tw_close); + method("seek", tw_seek); + method("tell", tw_tell); + method("truncate", tw_flush_noop); + method("fileno", tw_fileno); + method("isatty", tw_false); + method("readable", tw_readable); + method("writable", tw_writable); + method("seekable", tw_seekable); + method("detach", tw_detach); + method("__iter__", tw_iter); + method("__next__", tw_next); + method("__enter__", tw_enter); + method("__exit__", tw_exit); + method("reconfigure", tw_reconfigure); + // `__init__` needs keyword arguments (encoding=, errors=, newline=, …). + dict.insert( + DictKey(Object::from_static("__init__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__init__", + call: Box::new(|args| tw_init(args, &[])), + call_kw: Some(Box::new(tw_init)), + })), + ); + TypeObject::new_with_flags( + "TextIOWrapper", + vec![bt.object_.clone()], + dict, + TypeFlags { + is_exception: false, + is_builtin: true, + }, + ) + .expect("TextIOWrapper type") +} + +/// Pull `self` (a `TextIOWrapper` instance) out of the argument list. +fn tw_self(args: &[Object]) -> Result, RuntimeError> { + match args.first() { + Some(Object::Instance(i)) => Ok(i.clone()), + _ => Err(type_error( + "unbound method TextIOWrapper requires a TextIOWrapper instance", + )), + } +} + +fn tw_get(inst: &crate::types::PyInstance, name: &str) -> Option { + inst.dict + .borrow() + .get(&DictKey(Object::from_str(name))) + .cloned() +} + +fn tw_set(inst: &crate::types::PyInstance, name: &'static str, value: Object) { + inst.dict + .borrow_mut() + .insert(DictKey(Object::from_static(name)), value); +} + +fn tw_encoding(inst: &crate::types::PyInstance) -> String { + match tw_get(inst, "encoding") { + Some(Object::Str(s)) => s.to_string(), + _ => "utf-8".to_owned(), + } +} + +fn tw_errors(inst: &crate::types::PyInstance) -> String { + match tw_get(inst, "errors") { + Some(Object::Str(s)) => s.to_string(), + _ => "strict".to_owned(), + } +} + +/// Resolve the wrapped binary buffer to the underlying `PyFile`. +fn tw_buffer(inst: &crate::types::PyInstance) -> Result, RuntimeError> { + match tw_get(inst, "buffer") { + Some(Object::File(f)) => Ok(f), + _ => Err(crate::error::value_error( + "underlying buffer has been detached", + )), + } +} + +fn tw_init(args: &[Object], kwargs: &[(String, Object)]) -> Result { + let inst = tw_self(args)?; + let positional = &args[1..]; + let kw = |name: &str| kwargs.iter().find(|(k, _)| k == name).map(|(_, v)| v.clone()); + let buffer = positional + .first() + .cloned() + .or_else(|| kw("buffer")) + .ok_or_else(|| type_error("TextIOWrapper() missing required argument 'buffer'"))?; + // encoding (positional index 1 or keyword); `None`/missing → utf-8. + let encoding = positional.get(1).cloned().or_else(|| kw("encoding")); + let encoding = match encoding { + Some(Object::Str(s)) => s.to_string(), + _ => "utf-8".to_owned(), + }; + let errors = positional.get(2).cloned().or_else(|| kw("errors")); + let errors = match errors { + Some(Object::Str(s)) => s.to_string(), + _ => "strict".to_owned(), + }; + let newline = positional.get(3).cloned().or_else(|| kw("newline")); + tw_set(&inst, "buffer", buffer); + tw_set(&inst, "encoding", Object::from_str(encoding)); + tw_set(&inst, "errors", Object::from_str(errors)); + tw_set(&inst, "newline", newline.unwrap_or(Object::None)); + tw_set(&inst, "_detached", Object::Bool(false)); + Ok(Object::None) +} + +fn tw_write(args: &[Object]) -> Result { + let inst = tw_self(args)?; + let text = match args.get(1) { + Some(Object::Str(s)) => s.to_string(), + Some(other) => { + return Err(type_error(format!( + "write() argument must be str, not {}", + other.type_name() + ))) + } + None => return Err(type_error("write() takes exactly one argument")), + }; + let encoding = tw_encoding(&inst); + let errors = tw_errors(&inst); + let bytes = crate::stdlib::codecs_mod::encode_str(&text, &encoding, &errors)?; + let file = tw_buffer(&inst)?; + file.write_bytes(&bytes)?; + // TextIOWrapper.write returns the number of characters written. + Ok(Object::Int(text.chars().count() as i64)) +} + +fn tw_read(args: &[Object]) -> Result { + let inst = tw_self(args)?; + let file = tw_buffer(&inst)?; + // Text size is measured in characters; we only support the + // read-everything form (size omitted / None / negative), which is + // what stream-capture helpers use. + let want_all = !matches!(args.get(1), Some(Object::Int(n)) if *n >= 0); + let raw = if want_all { + file.read_bytes(None)? + } else { + // Approximate: read the requested count of bytes. Good enough + // for ASCII-heavy capture; exact char counting would require + // incremental decoding. + match args.get(1) { + Some(Object::Int(n)) => file.read_bytes(Some(*n as usize))?, + _ => file.read_bytes(None)?, + } + }; + let encoding = tw_encoding(&inst); + let errors = tw_errors(&inst); + let text = crate::stdlib::codecs_mod::decode_bytes(&raw, &encoding, &errors)?; + Ok(Object::from_str(text)) +} + +fn tw_readline(args: &[Object]) -> Result { + let inst = tw_self(args)?; + let file = tw_buffer(&inst)?; + // Read byte-by-byte until newline; fine for the small captured + // streams these tests exercise. + let mut line: Vec = Vec::new(); + loop { + let b = file.read_bytes(Some(1))?; + if b.is_empty() { + break; + } + line.push(b[0]); + if b[0] == b'\n' { + break; + } + } + let encoding = tw_encoding(&inst); + let errors = tw_errors(&inst); + let text = crate::stdlib::codecs_mod::decode_bytes(&line, &encoding, &errors)?; + Ok(Object::from_str(text)) +} + +fn tw_flush(args: &[Object]) -> Result { + let inst = tw_self(args)?; + if let Ok(file) = tw_buffer(&inst) { + file.flush()?; + } + Ok(Object::None) +} + +fn tw_flush_noop(_args: &[Object]) -> Result { + Ok(Object::None) +} + +fn tw_close(args: &[Object]) -> Result { + let inst = tw_self(args)?; + if let Ok(file) = tw_buffer(&inst) { + let _ = file.flush(); + file.close(); + } + Ok(Object::None) +} + +fn tw_seek(args: &[Object]) -> Result { + let inst = tw_self(args)?; + let file = tw_buffer(&inst)?; + let offset = match args.get(1) { + Some(Object::Int(n)) => *n as isize, + _ => 0, + }; + let whence = match args.get(2) { + Some(Object::Int(n)) => *n as i32, + _ => 0, + }; + let pos = file.seek(offset, whence)?; + Ok(Object::Int(pos as i64)) +} + +fn tw_tell(args: &[Object]) -> Result { + let inst = tw_self(args)?; + let file = tw_buffer(&inst)?; + // SEEK_CUR with 0 offset reports the current position. + let pos = file.seek(0, 1)?; + Ok(Object::Int(pos as i64)) +} + +fn tw_fileno(_args: &[Object]) -> Result { + Err(value_error("underlying stream has no fileno")) +} + +fn tw_false(_args: &[Object]) -> Result { + Ok(Object::Bool(false)) +} + +fn tw_readable(args: &[Object]) -> Result { + let inst = tw_self(args)?; + Ok(Object::Bool(tw_buffer(&inst).is_ok())) +} + +fn tw_writable(args: &[Object]) -> Result { + let inst = tw_self(args)?; + Ok(Object::Bool(tw_buffer(&inst).is_ok())) +} + +fn tw_seekable(args: &[Object]) -> Result { + let inst = tw_self(args)?; + Ok(Object::Bool(tw_buffer(&inst).is_ok())) +} + +fn tw_detach(args: &[Object]) -> Result { + let inst = tw_self(args)?; + let buffer = tw_buffer(&inst)?; + tw_set(&inst, "_detached", Object::Bool(true)); + inst.dict + .borrow_mut() + .shift_remove(&DictKey(Object::from_static("buffer"))); + Ok(Object::File(buffer)) +} + +fn tw_iter(args: &[Object]) -> Result { + Ok(args[0].clone()) +} + +fn tw_next(args: &[Object]) -> Result { + let line = tw_readline(args)?; + match &line { + Object::Str(s) if s.is_empty() => Err(crate::error::stop_iteration()), + _ => Ok(line), + } +} + +fn tw_enter(args: &[Object]) -> Result { + Ok(args[0].clone()) +} + +fn tw_exit(_args: &[Object]) -> Result { + Ok(Object::None) +} + +fn tw_reconfigure(args: &[Object]) -> Result { + // Accept (and ignore) encoding/newline reconfiguration requests; the + // common case in tests is `reconfigure(newline='')`. + let _ = tw_self(args)?; + Ok(Object::None) +} diff --git a/crates/weavepy-vm/src/stdlib/math.rs b/crates/weavepy-vm/src/stdlib/math.rs index fc56951..6f4916a 100644 --- a/crates/weavepy-vm/src/stdlib/math.rs +++ b/crates/weavepy-vm/src/stdlib/math.rs @@ -280,13 +280,13 @@ fn total_f64() -> &'static [(&'static str, fn(&[Object]) -> Result Result { match args.get(idx) { - Some(Object::Float(f)) => Ok(*f), - Some(Object::Int(i)) => Ok(*i as f64), - Some(Object::Bool(b)) => Ok(if *b { 1.0 } else { 0.0 }), - Some(other) => Err(type_error(format!( - "{func}() argument must be int or float, not '{}'", - other.type_name() - ))), + Some(other) => match crate::builtins::coerce_f64_opt(other)? { + Some(f) => Ok(f), + None => Err(type_error(format!( + "{func}() argument must be int or float, not '{}'", + other.type_name() + ))), + }, None => Err(type_error(format!( "{func}() takes at least {} argument(s)", idx + 1 @@ -309,6 +309,32 @@ fn to_i64(args: &[Object], func: &str, idx: usize) -> Result } } +/// Coerce an argument to an arbitrary-precision integer, accepting the +/// full integer tower (`bool`, `int`, big `int`, and integer-backed +/// subclasses). Mirrors CPython's "object cannot be interpreted as an +/// integer" TypeError for everything else — this is what lets `math.gcd`, +/// `math.lcm`, etc. operate on values that overflow 64 bits (e.g. the +/// `10**23` denominators that `fractions.Fraction.__new__` feeds in). +fn to_bigint(args: &[Object], idx: usize) -> Result { + let obj = args.get(idx); + if let Some(o) = obj { + if let Some(bi) = o.as_bigint() { + return Ok(bi); + } + // Honor int subclasses whose native payload is itself an integer. + if let Some(native) = o.native_value() { + if let Some(bi) = native.as_bigint() { + return Ok(bi); + } + } + return Err(type_error(format!( + "'{}' object cannot be interpreted as an integer", + o.type_name() + ))); + } + Err(type_error("expected at least one integer argument")) +} + fn math_sqrt(args: &[Object]) -> Result { let x = to_f64(args, "sqrt", 0)?; if x < 0.0 { @@ -386,19 +412,73 @@ fn math_pow(args: &[Object]) -> Result { Ok(Object::Float(x.powf(y))) } +/// Convert an (already integral) `f64` to a Python int, promoting to a +/// big integer when the value exceeds the 64-bit range so we never wrap. +fn float_to_int_obj(f: f64) -> Result { + use num_traits::FromPrimitive; + if !f.is_finite() { + return Err(value_error("cannot convert float infinity to integer")); + } + if (i64::MIN as f64..=i64::MAX as f64).contains(&f) { + Ok(Object::Int(f as i64)) + } else { + let big = num_bigint::BigInt::from_f64(f) + .ok_or_else(|| value_error("cannot convert float to integer"))?; + Ok(Object::int_from_bigint(big)) + } +} + +/// Shared core for `math.floor`/`ceil`/`trunc`. CPython dispatches the +/// matching dunder (`type(x).__floor__(x)`, …) for non-float arguments, +/// which is exactly how `fractions.Fraction`, `decimal.Decimal`, and any +/// user numeric type participate. Integers floor/ceil/trunc to themselves; +/// floats use the native rounding op. +fn floor_ceil_trunc( + args: &[Object], + func: &str, + dunder: &str, + op: fn(f64) -> f64, +) -> Result { + match args.first() { + Some(Object::Int(i)) => Ok(Object::Int(*i)), + Some(Object::Bool(b)) => Ok(Object::Int(i64::from(*b))), + Some(Object::Long(b)) => Ok(Object::Long(b.clone())), + Some(Object::Float(f)) => float_to_int_obj(op(*f)), + Some(obj @ Object::Instance(_)) => { + if let Some(method) = crate::instance_method(obj, dunder) { + let ptr = crate::vm_singletons::current_interpreter_ptr().ok_or_else(|| { + type_error(format!("{func}() requires an active interpreter")) + })?; + // SAFETY: the pointer was published by an enclosing VM call + // frame still live on this thread's stack; the GIL makes the + // mutable access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + return interp.call_object_with_globals(&method, &[], &[], &globals); + } + Err(type_error(format!( + "type {} doesn't define {} method", + obj.type_name(), + dunder + ))) + } + Some(_) => float_to_int_obj(op(to_f64(args, func, 0)?)), + None => Err(type_error(format!( + "{func}() takes exactly one argument (0 given)" + ))), + } +} + fn math_floor(args: &[Object]) -> Result { - let x = to_f64(args, "floor", 0)?; - Ok(Object::Int(x.floor() as i64)) + floor_ceil_trunc(args, "floor", "__floor__", f64::floor) } fn math_ceil(args: &[Object]) -> Result { - let x = to_f64(args, "ceil", 0)?; - Ok(Object::Int(x.ceil() as i64)) + floor_ceil_trunc(args, "ceil", "__ceil__", f64::ceil) } fn math_trunc(args: &[Object]) -> Result { - let x = to_f64(args, "trunc", 0)?; - Ok(Object::Int(x.trunc() as i64)) + floor_ceil_trunc(args, "trunc", "__trunc__", f64::trunc) } fn math_isnan(args: &[Object]) -> Result { @@ -429,32 +509,27 @@ fn math_fmod(args: &[Object]) -> Result { } fn math_gcd(args: &[Object]) -> Result { - if args.is_empty() { - return Ok(Object::Int(0)); - } - let mut acc: i64 = 0; - for (i, _) in args.iter().enumerate() { - let v = to_i64(args, "gcd", i)?.unsigned_abs() as i64; - acc = gcd_i64(acc, v); + use num_integer::Integer; + let mut acc = num_bigint::BigInt::from(0); + for i in 0..args.len() { + let v = to_bigint(args, i)?; + acc = acc.gcd(&v); } - Ok(Object::Int(acc)) + Ok(Object::int_from_bigint(acc)) } fn math_lcm(args: &[Object]) -> Result { - if args.is_empty() { - return Ok(Object::Int(1)); - } - let mut acc: i64 = 1; - for (i, _) in args.iter().enumerate() { - let v = to_i64(args, "lcm", i)?.unsigned_abs() as i64; - if v == 0 { + use num_integer::Integer; + use num_traits::Zero; + let mut acc = num_bigint::BigInt::from(1); + for i in 0..args.len() { + let v = to_bigint(args, i)?; + if v.is_zero() { return Ok(Object::Int(0)); } - let g = gcd_i64(acc, v); - // acc * v / g - acc = (acc / g).saturating_mul(v); + acc = acc.lcm(&v); } - Ok(Object::Int(acc)) + Ok(Object::int_from_bigint(acc)) } fn math_factorial(args: &[Object]) -> Result { @@ -462,11 +537,13 @@ fn math_factorial(args: &[Object]) -> Result { if n < 0 { return Err(value_error("factorial() not defined for negative values")); } - let mut acc: i64 = 1; - for i in 1..=n { - acc = acc.saturating_mul(i); + // Accumulate in arbitrary precision: a plain `i64` overflows past 20!, + // which silently produced wrong answers under the old `saturating_mul`. + let mut acc = num_bigint::BigInt::from(1); + for i in 2..=n { + acc *= i; } - Ok(Object::Int(acc)) + Ok(Object::int_from_bigint(acc)) } /// `math.isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0)` implementing @@ -502,17 +579,6 @@ fn math_isclose(args: &[Object]) -> Result { Ok(Object::Bool(diff <= tol)) } -fn gcd_i64(a: i64, b: i64) -> i64 { - let mut a = a.unsigned_abs(); - let mut b = b.unsigned_abs(); - while b != 0 { - let t = b; - b = a % b; - a = t; - } - a as i64 -} - // --------------------------------------------------------------------- // Math additions (RFC 0030) // --------------------------------------------------------------------- @@ -649,7 +715,43 @@ fn math_log1p(args: &[Object]) -> Result { fn math_ldexp(args: &[Object]) -> Result { let x = to_f64(args, "ldexp", 0)?; let i = to_i64(args, "ldexp", 1)?; - Ok(Object::Float(x * 2f64.powi(i as i32))) + // Saturate the exponent: anything past these bounds overflows to ±inf or + // underflows to ±0 anyway, and keeps `ldexp`'s `i32` happy. + let n = i.clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32; + Ok(Object::Float(ldexp(x, n))) +} + +/// Correctly-rounded `x * 2**n` (C `scalbn`/`ldexp`), including the +/// subnormal range — `2f64.powi(n)` underflows to 0 for `n < -1022`, so a +/// naive `x * 2f64.powi(n)` cannot produce subnormals like `ldexp(1.0, +/// -1074)` (the smallest positive double). Mirrors musl's `scalbn`. +pub(crate) fn ldexp(mut x: f64, mut n: i32) -> f64 { + let p1023 = 2f64.powi(1023); + // 2**-1022 * 2**53 == 2**-969, applied in steps so the running value + // never underflows before the final scaling (avoids double rounding). + let p_minus_969 = 2f64.powi(-969); + if n > 1023 { + x *= p1023; + n -= 1023; + if n > 1023 { + x *= p1023; + n -= 1023; + if n > 1023 { + n = 1023; + } + } + } else if n < -1022 { + x *= p_minus_969; + n += 969; + if n < -1022 { + x *= p_minus_969; + n += 969; + if n < -1022 { + n = -1022; + } + } + } + x * f64::from_bits(((0x3ff + n) as u64) << 52) } fn math_frexp(args: &[Object]) -> Result { @@ -854,20 +956,27 @@ fn math_lgamma(args: &[Object]) -> Result { } fn math_isqrt(args: &[Object]) -> Result { - let n = to_i64(args, "isqrt", 0)?; - if n < 0 { - return Err(value_error("isqrt() argument must be non-negative")); - } - let approx = (n as f64).sqrt().floor() as i64; - // Adjust for rounding error at the boundary. - let mut root = approx; - while root > 0 && root * root > n { - root -= 1; - } - while (root + 1) * (root + 1) <= n { - root += 1; + use num_bigint::BigInt; + use num_traits::Signed; + // Accept any integer, including arbitrary-precision values, matching + // CPython. A float-based approximation overflows for large inputs, so + // we compute the exact integer square root over BigInt. + let n: BigInt = match args.first() { + Some(Object::Int(i)) => BigInt::from(*i), + Some(Object::Bool(b)) => BigInt::from(i64::from(*b)), + Some(Object::Long(b)) => (**b).clone(), + Some(other) => { + return Err(type_error(format!( + "'{}' object cannot be interpreted as an integer", + other.type_name() + ))) + } + None => return Err(type_error("isqrt() takes exactly one argument (0 given)")), + }; + if n.is_negative() { + return Err(value_error("isqrt() argument must be nonnegative")); } - Ok(Object::Int(root)) + Ok(Object::int_from_bigint(n.sqrt())) } fn math_cbrt(args: &[Object]) -> Result { diff --git a/crates/weavepy-vm/src/stdlib/mod.rs b/crates/weavepy-vm/src/stdlib/mod.rs index 0d2fdc0..d9deb85 100644 --- a/crates/weavepy-vm/src/stdlib/mod.rs +++ b/crates/weavepy-vm/src/stdlib/mod.rs @@ -171,9 +171,60 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/builtins.py"), is_package: false, }, + // `collections` is a package so `collections.abc` resolves; the + // verbatim CPython `_collections_abc` carries the ABC definitions + // and `collections.abc` re-exports them (RFC 0037 WS8). FrozenSource { name: "collections", source: include_str!("python/collections.py"), + is_package: true, + }, + FrozenSource { + name: "_collections_abc", + source: include_str!("python/_collections_abc.py"), + is_package: false, + }, + // `_weakrefset` (verbatim CPython): the `WeakSet` source module + // that `abc`/`_py_abc` import directly to back the ABC virtual- + // subclass registry/caches (RFC 0037 WS8). + FrozenSource { + name: "_weakrefset", + source: include_str!("python/_weakrefset.py"), + is_package: false, + }, + // `_py_abc` (verbatim CPython): the pure-Python `ABCMeta` + // reference implementation. `test_abc` imports it directly to + // exercise the Python ABC machinery alongside the C `_abc` path. + FrozenSource { + name: "_py_abc", + source: include_str!("python/_py_abc.py"), + is_package: false, + }, + // `_colorize`: CPython 3.13's ANSI-colour helper (verbatim). Imported + // by `traceback`/`test_traceback` (and the 3.13 REPL); honours + // NO_COLOR/FORCE_COLOR and TTY detection. + FrozenSource { + name: "_colorize", + source: include_str!("python/_colorize.py"), + is_package: false, + }, + // `__future__`: the feature-flag table (verbatim CPython 3.13). + // `from __future__ import annotations` is a compiler directive, but + // the module must still be importable because real modules read its + // `_Feature` objects (e.g. `__future__.annotations`). + FrozenSource { + name: "__future__", + source: include_str!("python/future_module.py"), + is_package: false, + }, + FrozenSource { + name: "collections.abc", + source: include_str!("python/collections_abc.py"), + is_package: false, + }, + FrozenSource { + name: "_collections_user", + source: include_str!("python/_collections_user.py"), is_package: false, }, // RFC 0036 — `string` (constants + `Template` + `Formatter` over @@ -199,6 +250,26 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/functools.py"), is_package: false, }, + // RFC 0037 WS8 verbatim/faithful module ports that gate import-time + // clusters: `cmath` (pure-Python over the `math` core) unblocks + // `test_fractions`; the C-locale `locale` unblocks `test_format` + // and backs `calendar`'s `LocaleTextCalendar`; `calendar` is the + // verbatim CPython 3.13 module. + FrozenSource { + name: "cmath", + source: include_str!("python/cmath.py"), + is_package: false, + }, + FrozenSource { + name: "locale", + source: include_str!("python/locale.py"), + is_package: false, + }, + FrozenSource { + name: "calendar", + source: include_str!("python/calendar.py"), + is_package: false, + }, FrozenSource { name: "contextlib", source: include_str!("python/contextlib.py"), @@ -332,6 +403,11 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/html_parser.py"), is_package: false, }, + FrozenSource { + name: "html.entities", + source: include_str!("python/html_entities.py"), + is_package: false, + }, // `urllib` is a package containing three submodules. FrozenSource { name: "urllib", @@ -447,6 +523,11 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/linecache.py"), is_package: false, }, + FrozenSource { + name: "reprlib", + source: include_str!("python/reprlib.py"), + is_package: false, + }, FrozenSource { name: "warnings", source: include_str!("python/warnings.py"), @@ -538,6 +619,20 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/test_support_socket_helper.py"), is_package: false, }, + // `test.support.hashlib_helper` (verbatim) — `requires_hashdigest` + // gate used by test_hmac and friends. + FrozenSource { + name: "test.support.hashlib_helper", + source: include_str!("python/test_support_hashlib_helper.py"), + is_package: false, + }, + // `test.support.i18n_helper` — minimal shim (snapshot tests skip) so + // test_getopt/test_optparse import; their own tests still run. + FrozenSource { + name: "test.support.i18n_helper", + source: include_str!("python/test_support_i18n_helper.py"), + is_package: false, + }, // RFC 0036 — two more 3.13 helper submodules carried verbatim: // `testcase` (ExceptionIsLikeMixin + float/complex assertions used // by test_float/test_complex) and `numbers` (the numeric-tower @@ -552,6 +647,47 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/test_support_numbers.py"), is_package: false, }, + // `test.tokenizedata`: vendored lexer/tokenizer fixtures. + // `test_unicode_identifiers` imports `badsyntax_3131` to assert the + // exact `SyntaxError` for an invalid PEP 3131 identifier (`€`). + FrozenSource { + name: "test.tokenizedata", + source: include_str!("python/test_tokenizedata_init.py"), + is_package: true, + }, + FrozenSource { + name: "test.tokenizedata.badsyntax_3131", + source: include_str!("python/test_tokenizedata_badsyntax_3131.py"), + is_package: false, + }, + // `test.string_tests`: the shared CommonTest/MixinStrUnicodeUserStringTest + // base classes that `test_bytes`/`test_bytearray`/`test_str` derive + // from. Carried verbatim from CPython 3.13. + FrozenSource { + name: "test.string_tests", + source: include_str!("python/test_string_tests.py"), + is_package: false, + }, + // `test.seq_tests` / `test.list_tests`: shared sequence/list test + // mixins (verbatim CPython 3.13) that `test_bytes`/`test_list`/ + // `test_tuple`/`test_deque` and friends import. + FrozenSource { + name: "test.seq_tests", + source: include_str!("python/test_seq_tests.py"), + is_package: false, + }, + FrozenSource { + name: "test.list_tests", + source: include_str!("python/test_list_tests.py"), + is_package: false, + }, + // `test.pickletester`: only `ExtensionSaver` is carried (test_copyreg + // imports it); the full CPython file is ~4900 lines of pickle matrix. + FrozenSource { + name: "test.pickletester", + source: include_str!("python/test_pickletester.py"), + is_package: false, + }, // `test.__main__` / `test.regrtest`: drive `weavepy -m test` and // `weavepy -m test.regrtest`. The runner itself lives in the // `test.libregrtest` package below. @@ -671,6 +807,14 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/decimal.py"), is_package: false, }, + // Full CPython pure-Python decimal (IEEE 754-2008: NaN/Infinity, + // contexts, traps, exact float/Decimal comparison + hashing). The + // `decimal` shim above re-exports this via `sys.modules` like CPython. + FrozenSource { + name: "_pydecimal", + source: include_str!("python/_pydecimal.py"), + is_package: false, + }, FrozenSource { name: "py_compile", source: include_str!("python/py_compile.py"), diff --git a/crates/weavepy-vm/src/stdlib/os.rs b/crates/weavepy-vm/src/stdlib/os.rs index e559a4e..1fd5650 100644 --- a/crates/weavepy-vm/src/stdlib/os.rs +++ b/crates/weavepy-vm/src/stdlib/os.rs @@ -143,6 +143,14 @@ pub fn build(cache: &ModuleCache) -> Rc { DictKey(Object::from_static("fspath")), builtin("fspath", os_fspath), ); + d.insert( + DictKey(Object::from_static("fsdecode")), + builtin("fsdecode", os_fsdecode), + ); + d.insert( + DictKey(Object::from_static("fsencode")), + builtin("fsencode", os_fsencode), + ); d.insert( DictKey(Object::from_static("walk")), builtin("walk", os_walk), @@ -305,6 +313,10 @@ pub fn build_path(_cache: &ModuleCache) -> Rc { DictKey(Object::from_static("splitext")), builtin("splitext", path_splitext), ); + d.insert( + DictKey(Object::from_static("splitdrive")), + builtin("splitdrive", path_splitdrive), + ); d.insert( DictKey(Object::from_static("basename")), builtin("basename", path_basename), @@ -723,6 +735,54 @@ fn os_fspath(args: &[Object]) -> Result { } } +/// Reduce a path-like argument to a `str` or `bytes` object, mirroring +/// CPython's `os.fspath`: `str`/`bytes` pass through, an `str`/`bytes` +/// subclass instance reduces to its native value. Used by `fsdecode`/ +/// `fsencode` (which themselves only special-case the str/bytes split). +fn fspath_to_str_or_bytes(obj: &Object, func: &str) -> Result { + match obj { + Object::Str(_) | Object::Bytes(_) => Ok(obj.clone()), + Object::Instance(_) => match obj.native_value() { + Some(n @ (Object::Str(_) | Object::Bytes(_))) => Ok(n), + _ => Err(type_error(format!( + "expected str, bytes or os.PathLike object, not {}", + obj.type_name() + ))), + }, + other => Err(type_error(format!( + "{}() argument must be str, bytes, or os.PathLike object, not {}", + func, + other.type_name() + ))), + } +} + +/// `os.fsdecode(filename)` — decode a `bytes` path to `str` (the filesystem +/// encoding is UTF-8 here), pass a `str` through unchanged. +fn os_fsdecode(args: &[Object]) -> Result { + let obj = args + .first() + .ok_or_else(|| type_error("fsdecode() takes exactly one argument (0 given)"))?; + match fspath_to_str_or_bytes(obj, "fsdecode")? { + s @ Object::Str(_) => Ok(s), + Object::Bytes(b) => Ok(Object::from_str(String::from_utf8_lossy(&b).into_owned())), + _ => unreachable!("fspath_to_str_or_bytes returns only str/bytes"), + } +} + +/// `os.fsencode(filename)` — encode a `str` path to `bytes` (UTF-8), pass a +/// `bytes` through unchanged. +fn os_fsencode(args: &[Object]) -> Result { + let obj = args + .first() + .ok_or_else(|| type_error("fsencode() takes exactly one argument (0 given)"))?; + match fspath_to_str_or_bytes(obj, "fsencode")? { + Object::Str(s) => Ok(Object::Bytes(Rc::from(s.as_bytes()))), + b @ Object::Bytes(_) => Ok(b), + _ => unreachable!("fspath_to_str_or_bytes returns only str/bytes"), + } +} + fn os_walk(args: &[Object]) -> Result { let p = first_path(args, "walk")?; let mut out = Vec::new(); @@ -1293,6 +1353,18 @@ fn path_splitext(args: &[Object]) -> Result { } } +/// `os.path.splitdrive(p)` — on POSIX the drive component is always empty, +/// so this returns `("", p)` (matching `posixpath.splitdrive`). Paths here +/// are already `str` by the time callers reach this (e.g. `mimetypes` +/// `fsdecode`s first), so we reuse the `first_path` string coercion. +fn path_splitdrive(args: &[Object]) -> Result { + let s = first_path(args, "splitdrive")?; + Ok(Object::new_tuple(vec![ + Object::from_static(""), + Object::from_str(s), + ])) +} + /// Mirror CPython's `os.path.splitext`: split on the *last* dot, but /// only when that dot follows a non-dot character (`.profile` keeps /// the leading dot). diff --git a/crates/weavepy-vm/src/stdlib/python/_collections_abc.py b/crates/weavepy-vm/src/stdlib/python/_collections_abc.py new file mode 100644 index 0000000..147324a --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_collections_abc.py @@ -0,0 +1,1195 @@ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) for collections, according to PEP 3119. + +Unit tests are in test_collections. +""" + +############ Maintenance notes ######################################### +# +# ABCs are different from other standard library modules in that they +# specify compliance tests. In general, once an ABC has been published, +# new methods (either abstract or concrete) cannot be added. +# +# Though classes that inherit from an ABC would automatically receive a +# new mixin method, registered classes would become non-compliant and +# violate the contract promised by ``isinstance(someobj, SomeABC)``. +# +# Though irritating, the correct procedure for adding new abstract or +# mixin methods is to create a new ABC as a subclass of the previous +# ABC. For example, union(), intersection(), and difference() cannot +# be added to Set but could go into a new ABC that extends Set. +# +# Because they are so hard to change, new ABCs should have their APIs +# carefully thought through prior to publication. +# +# Since ABCMeta only checks for the presence of methods, it is possible +# to alter the signature of a method by adding optional arguments +# or changing parameters names. This is still a bit dubious but at +# least it won't cause isinstance() to return an incorrect result. +# +# +####################################################################### + +from abc import ABCMeta, abstractmethod +import sys + +GenericAlias = type(list[int]) +EllipsisType = type(...) +def _f(): pass +FunctionType = type(_f) +del _f + +__all__ = ["Awaitable", "Coroutine", + "AsyncIterable", "AsyncIterator", "AsyncGenerator", + "Hashable", "Iterable", "Iterator", "Generator", "Reversible", + "Sized", "Container", "Callable", "Collection", + "Set", "MutableSet", + "Mapping", "MutableMapping", + "MappingView", "KeysView", "ItemsView", "ValuesView", + "Sequence", "MutableSequence", + "ByteString", "Buffer", + ] + +# This module has been renamed from collections.abc to _collections_abc to +# speed up interpreter startup. Some of the types such as MutableMapping are +# required early but collections module imports a lot of other modules. +# See issue #19218 +__name__ = "collections.abc" + +# Private list of types that we want to register with the various ABCs +# so that they will pass tests like: +# it = iter(somebytearray) +# assert isinstance(it, Iterable) +# Note: in other implementations, these types might not be distinct +# and they may have their own implementation specific types that +# are not included on this list. +bytes_iterator = type(iter(b'')) +bytearray_iterator = type(iter(bytearray())) +#callable_iterator = ??? +dict_keyiterator = type(iter({}.keys())) +dict_valueiterator = type(iter({}.values())) +dict_itemiterator = type(iter({}.items())) +list_iterator = type(iter([])) +list_reverseiterator = type(iter(reversed([]))) +range_iterator = type(iter(range(0))) +# WeavePy: ``range`` does not accept arbitrary-precision (big int) bounds +# yet, so ``range(1 << 1000)`` raises. Every ``range`` shares one iterator +# type here, so reuse it rather than probing an out-of-range bound. +try: + longrange_iterator = type(iter(range(1 << 1000))) +except (TypeError, OverflowError): + longrange_iterator = range_iterator +set_iterator = type(iter(set())) +str_iterator = type(iter("")) +tuple_iterator = type(iter(())) +zip_iterator = type(iter(zip())) +## views ## +dict_keys = type({}.keys()) +dict_values = type({}.values()) +dict_items = type({}.items()) +## misc ## +mappingproxy = type(type.__dict__) +# WeavePy: frame ``f_locals`` is a plain ``dict`` rather than a distinct +# ``FrameLocalsProxy`` type (PEP 667), so fall back to that. The registry +# below registers it with ``Mapping``; ``dict`` is already a Mapping, so +# the duplicate registration is harmless. +def _get_framelocalsproxy(): + try: + return type(sys._getframe().f_locals) + except (AttributeError, TypeError): + return dict +framelocalsproxy = _get_framelocalsproxy() +del _get_framelocalsproxy +generator = type((lambda: (yield))()) +## coroutine ## +async def _coro(): pass +_coro = _coro() +coroutine = type(_coro) +_coro.close() # Prevent ResourceWarning +del _coro +## asynchronous generator ## +async def _ag(): yield +_ag = _ag() +async_generator = type(_ag) +del _ag + + +### ONE-TRICK PONIES ### + +def _check_methods(C, *methods): + mro = C.__mro__ + for method in methods: + for B in mro: + if method in B.__dict__: + if B.__dict__[method] is None: + return NotImplemented + break + else: + return NotImplemented + return True + +class Hashable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __hash__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Hashable: + return _check_methods(C, "__hash__") + return NotImplemented + + +class Awaitable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __await__(self): + yield + + @classmethod + def __subclasshook__(cls, C): + if cls is Awaitable: + return _check_methods(C, "__await__") + return NotImplemented + + __class_getitem__ = classmethod(GenericAlias) + + +class Coroutine(Awaitable): + + __slots__ = () + + @abstractmethod + def send(self, value): + """Send a value into the coroutine. + Return next yielded value or raise StopIteration. + """ + raise StopIteration + + @abstractmethod + def throw(self, typ, val=None, tb=None): + """Raise an exception in the coroutine. + Return next yielded value or raise StopIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + def close(self): + """Raise GeneratorExit inside coroutine. + """ + try: + self.throw(GeneratorExit) + except (GeneratorExit, StopIteration): + pass + else: + raise RuntimeError("coroutine ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is Coroutine: + return _check_methods(C, '__await__', 'send', 'throw', 'close') + return NotImplemented + + +Coroutine.register(coroutine) + + +class AsyncIterable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __aiter__(self): + return AsyncIterator() + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncIterable: + return _check_methods(C, "__aiter__") + return NotImplemented + + __class_getitem__ = classmethod(GenericAlias) + + +class AsyncIterator(AsyncIterable): + + __slots__ = () + + @abstractmethod + async def __anext__(self): + """Return the next item or raise StopAsyncIteration when exhausted.""" + raise StopAsyncIteration + + def __aiter__(self): + return self + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncIterator: + return _check_methods(C, "__anext__", "__aiter__") + return NotImplemented + + +class AsyncGenerator(AsyncIterator): + + __slots__ = () + + async def __anext__(self): + """Return the next item from the asynchronous generator. + When exhausted, raise StopAsyncIteration. + """ + return await self.asend(None) + + @abstractmethod + async def asend(self, value): + """Send a value into the asynchronous generator. + Return next yielded value or raise StopAsyncIteration. + """ + raise StopAsyncIteration + + @abstractmethod + async def athrow(self, typ, val=None, tb=None): + """Raise an exception in the asynchronous generator. + Return next yielded value or raise StopAsyncIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + async def aclose(self): + """Raise GeneratorExit inside coroutine. + """ + try: + await self.athrow(GeneratorExit) + except (GeneratorExit, StopAsyncIteration): + pass + else: + raise RuntimeError("asynchronous generator ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncGenerator: + return _check_methods(C, '__aiter__', '__anext__', + 'asend', 'athrow', 'aclose') + return NotImplemented + + +AsyncGenerator.register(async_generator) + + +class Iterable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __iter__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterable: + return _check_methods(C, "__iter__") + return NotImplemented + + __class_getitem__ = classmethod(GenericAlias) + + +class Iterator(Iterable): + + __slots__ = () + + @abstractmethod + def __next__(self): + 'Return the next item from the iterator. When exhausted, raise StopIteration' + raise StopIteration + + def __iter__(self): + return self + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterator: + return _check_methods(C, '__iter__', '__next__') + return NotImplemented + + +Iterator.register(bytes_iterator) +Iterator.register(bytearray_iterator) +#Iterator.register(callable_iterator) +Iterator.register(dict_keyiterator) +Iterator.register(dict_valueiterator) +Iterator.register(dict_itemiterator) +Iterator.register(list_iterator) +Iterator.register(list_reverseiterator) +Iterator.register(range_iterator) +Iterator.register(longrange_iterator) +Iterator.register(set_iterator) +Iterator.register(str_iterator) +Iterator.register(tuple_iterator) +Iterator.register(zip_iterator) + + +class Reversible(Iterable): + + __slots__ = () + + @abstractmethod + def __reversed__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Reversible: + return _check_methods(C, "__reversed__", "__iter__") + return NotImplemented + + +class Generator(Iterator): + + __slots__ = () + + def __next__(self): + """Return the next item from the generator. + When exhausted, raise StopIteration. + """ + return self.send(None) + + @abstractmethod + def send(self, value): + """Send a value into the generator. + Return next yielded value or raise StopIteration. + """ + raise StopIteration + + @abstractmethod + def throw(self, typ, val=None, tb=None): + """Raise an exception in the generator. + Return next yielded value or raise StopIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + def close(self): + """Raise GeneratorExit inside generator. + """ + try: + self.throw(GeneratorExit) + except (GeneratorExit, StopIteration): + pass + else: + raise RuntimeError("generator ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is Generator: + return _check_methods(C, '__iter__', '__next__', + 'send', 'throw', 'close') + return NotImplemented + + +Generator.register(generator) + + +class Sized(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __len__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Sized: + return _check_methods(C, "__len__") + return NotImplemented + + +class Container(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __contains__(self, x): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Container: + return _check_methods(C, "__contains__") + return NotImplemented + + __class_getitem__ = classmethod(GenericAlias) + + +class Collection(Sized, Iterable, Container): + + __slots__ = () + + @classmethod + def __subclasshook__(cls, C): + if cls is Collection: + return _check_methods(C, "__len__", "__iter__", "__contains__") + return NotImplemented + + +class Buffer(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __buffer__(self, flags: int, /) -> memoryview: + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, C): + if cls is Buffer: + return _check_methods(C, "__buffer__") + return NotImplemented + + +class _CallableGenericAlias(GenericAlias): + """ Represent `Callable[argtypes, resulttype]`. + + This sets ``__args__`` to a tuple containing the flattened ``argtypes`` + followed by ``resulttype``. + + Example: ``Callable[[int, str], float]`` sets ``__args__`` to + ``(int, str, float)``. + """ + + __slots__ = () + + def __new__(cls, origin, args): + if not (isinstance(args, tuple) and len(args) == 2): + raise TypeError( + "Callable must be used as Callable[[arg, ...], result].") + t_args, t_result = args + if isinstance(t_args, (tuple, list)): + args = (*t_args, t_result) + elif not _is_param_expr(t_args): + raise TypeError(f"Expected a list of types, an ellipsis, " + f"ParamSpec, or Concatenate. Got {t_args}") + return super().__new__(cls, origin, args) + + def __repr__(self): + if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): + return super().__repr__() + return (f'collections.abc.Callable' + f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' + f'{_type_repr(self.__args__[-1])}]') + + def __reduce__(self): + args = self.__args__ + if not (len(args) == 2 and _is_param_expr(args[0])): + args = list(args[:-1]), args[-1] + return _CallableGenericAlias, (Callable, args) + + def __getitem__(self, item): + # Called during TypeVar substitution, returns the custom subclass + # rather than the default types.GenericAlias object. Most of the + # code is copied from typing's _GenericAlias and the builtin + # types.GenericAlias. + if not isinstance(item, tuple): + item = (item,) + + new_args = super().__getitem__(item).__args__ + + # args[0] occurs due to things like Z[[int, str, bool]] from PEP 612 + if not isinstance(new_args[0], (tuple, list)): + t_result = new_args[-1] + t_args = new_args[:-1] + new_args = (t_args, t_result) + return _CallableGenericAlias(Callable, tuple(new_args)) + +def _is_param_expr(obj): + """Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or + ``_ConcatenateGenericAlias`` from typing.py + """ + if obj is Ellipsis: + return True + if isinstance(obj, list): + return True + obj = type(obj) + names = ('ParamSpec', '_ConcatenateGenericAlias') + return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names) + +def _type_repr(obj): + """Return the repr() of an object, special-casing types (internal helper). + + Copied from :mod:`typing` since collections.abc + shouldn't depend on that module. + (Keep this roughly in sync with the typing version.) + """ + if isinstance(obj, type): + if obj.__module__ == 'builtins': + return obj.__qualname__ + return f'{obj.__module__}.{obj.__qualname__}' + if obj is Ellipsis: + return '...' + if isinstance(obj, FunctionType): + return obj.__name__ + return repr(obj) + + +class Callable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __call__(self, *args, **kwds): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Callable: + return _check_methods(C, "__call__") + return NotImplemented + + __class_getitem__ = classmethod(_CallableGenericAlias) + + +### SETS ### + + +class Set(Collection): + """A set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__ and __len__. + + To override the comparisons (presumably for speed, as the + semantics are fixed), redefine __le__ and __ge__, + then the other operations will automatically follow suit. + """ + + __slots__ = () + + def __le__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) > len(other): + return False + for elem in self: + if elem not in other: + return False + return True + + def __lt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) < len(other) and self.__le__(other) + + def __gt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) > len(other) and self.__ge__(other) + + def __ge__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) < len(other): + return False + for elem in other: + if elem not in self: + return False + return True + + def __eq__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) == len(other) and self.__le__(other) + + @classmethod + def _from_iterable(cls, it): + '''Construct an instance of the class from any iterable input. + + Must override this method if the class constructor signature + does not accept an iterable for an input. + ''' + return cls(it) + + def __and__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + return self._from_iterable(value for value in other if value in self) + + __rand__ = __and__ + + def isdisjoint(self, other): + 'Return True if two sets have a null intersection.' + for value in other: + if value in self: + return False + return True + + def __or__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + chain = (e for s in (self, other) for e in s) + return self._from_iterable(chain) + + __ror__ = __or__ + + def __sub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in self + if value not in other) + + def __rsub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in other + if value not in self) + + def __xor__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return (self - other) | (other - self) + + __rxor__ = __xor__ + + def _hash(self): + """Compute the hash value of a set. + + Note that we don't define __hash__: not all sets are hashable. + But if you define a hashable set type, its __hash__ should + call this function. + + This must be compatible __eq__. + + All sets ought to compare equal if they contain the same + elements, regardless of how they are implemented, and + regardless of the order of the elements; so there's not much + freedom for __eq__ or __hash__. We match the algorithm used + by the built-in frozenset type. + """ + MAX = sys.maxsize + MASK = 2 * MAX + 1 + n = len(self) + h = 1927868237 * (n + 1) + h &= MASK + for x in self: + hx = hash(x) + h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 + h &= MASK + h ^= (h >> 11) ^ (h >> 25) + h = h * 69069 + 907133923 + h &= MASK + if h > MAX: + h -= MASK + 1 + if h == -1: + h = 590923713 + return h + + +Set.register(frozenset) + + +class MutableSet(Set): + """A mutable set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__, __len__, + add(), and discard(). + + To override the comparisons (presumably for speed, as the + semantics are fixed), all you have to do is redefine __le__ and + then the other operations will automatically follow suit. + """ + + __slots__ = () + + @abstractmethod + def add(self, value): + """Add an element.""" + raise NotImplementedError + + @abstractmethod + def discard(self, value): + """Remove an element. Do not raise an exception if absent.""" + raise NotImplementedError + + def remove(self, value): + """Remove an element. If not a member, raise a KeyError.""" + if value not in self: + raise KeyError(value) + self.discard(value) + + def pop(self): + """Return the popped value. Raise KeyError if empty.""" + it = iter(self) + try: + value = next(it) + except StopIteration: + raise KeyError from None + self.discard(value) + return value + + def clear(self): + """This is slow (creates N new iterators!) but effective.""" + try: + while True: + self.pop() + except KeyError: + pass + + def __ior__(self, it): + for value in it: + self.add(value) + return self + + def __iand__(self, it): + for value in (self - it): + self.discard(value) + return self + + def __ixor__(self, it): + if it is self: + self.clear() + else: + if not isinstance(it, Set): + it = self._from_iterable(it) + for value in it: + if value in self: + self.discard(value) + else: + self.add(value) + return self + + def __isub__(self, it): + if it is self: + self.clear() + else: + for value in it: + self.discard(value) + return self + + +MutableSet.register(set) + + +### MAPPINGS ### + +class Mapping(Collection): + """A Mapping is a generic container for associating key/value + pairs. + + This class provides concrete generic implementations of all + methods except for __getitem__, __iter__, and __len__. + """ + + __slots__ = () + + # Tell ABCMeta.__new__ that this class should have TPFLAGS_MAPPING set. + __abc_tpflags__ = 1 << 6 # Py_TPFLAGS_MAPPING + + @abstractmethod + def __getitem__(self, key): + raise KeyError + + def get(self, key, default=None): + 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.' + try: + return self[key] + except KeyError: + return default + + def __contains__(self, key): + try: + self[key] + except KeyError: + return False + else: + return True + + def keys(self): + "D.keys() -> a set-like object providing a view on D's keys" + return KeysView(self) + + def items(self): + "D.items() -> a set-like object providing a view on D's items" + return ItemsView(self) + + def values(self): + "D.values() -> an object providing a view on D's values" + return ValuesView(self) + + def __eq__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + return dict(self.items()) == dict(other.items()) + + __reversed__ = None + +Mapping.register(mappingproxy) +Mapping.register(framelocalsproxy) + + +class MappingView(Sized): + + __slots__ = '_mapping', + + def __init__(self, mapping): + self._mapping = mapping + + def __len__(self): + return len(self._mapping) + + def __repr__(self): + return '{0.__class__.__name__}({0._mapping!r})'.format(self) + + __class_getitem__ = classmethod(GenericAlias) + + +class KeysView(MappingView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(cls, it): + return set(it) + + def __contains__(self, key): + return key in self._mapping + + def __iter__(self): + yield from self._mapping + + +KeysView.register(dict_keys) + + +class ItemsView(MappingView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(cls, it): + return set(it) + + def __contains__(self, item): + key, value = item + try: + v = self._mapping[key] + except KeyError: + return False + else: + return v is value or v == value + + def __iter__(self): + for key in self._mapping: + yield (key, self._mapping[key]) + + +ItemsView.register(dict_items) + + +class ValuesView(MappingView, Collection): + + __slots__ = () + + def __contains__(self, value): + for key in self._mapping: + v = self._mapping[key] + if v is value or v == value: + return True + return False + + def __iter__(self): + for key in self._mapping: + yield self._mapping[key] + + +ValuesView.register(dict_values) + + +class MutableMapping(Mapping): + """A MutableMapping is a generic container for associating + key/value pairs. + + This class provides concrete generic implementations of all + methods except for __getitem__, __setitem__, __delitem__, + __iter__, and __len__. + """ + + __slots__ = () + + @abstractmethod + def __setitem__(self, key, value): + raise KeyError + + @abstractmethod + def __delitem__(self, key): + raise KeyError + + __marker = object() + + def pop(self, key, default=__marker): + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + ''' + try: + value = self[key] + except KeyError: + if default is self.__marker: + raise + return default + else: + del self[key] + return value + + def popitem(self): + '''D.popitem() -> (k, v), remove and return some (key, value) pair + as a 2-tuple; but raise KeyError if D is empty. + ''' + try: + key = next(iter(self)) + except StopIteration: + raise KeyError from None + value = self[key] + del self[key] + return key, value + + def clear(self): + 'D.clear() -> None. Remove all items from D.' + try: + while True: + self.popitem() + except KeyError: + pass + + def update(self, other=(), /, **kwds): + ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. + If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] + If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v + In either case, this is followed by: for k, v in F.items(): D[k] = v + ''' + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + def setdefault(self, key, default=None): + 'D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D' + try: + return self[key] + except KeyError: + self[key] = default + return default + + +MutableMapping.register(dict) + + +### SEQUENCES ### + +class Sequence(Reversible, Collection): + """All the operations on a read-only sequence. + + Concrete subclasses must override __new__ or __init__, + __getitem__, and __len__. + """ + + __slots__ = () + + # Tell ABCMeta.__new__ that this class should have TPFLAGS_SEQUENCE set. + __abc_tpflags__ = 1 << 5 # Py_TPFLAGS_SEQUENCE + + @abstractmethod + def __getitem__(self, index): + raise IndexError + + def __iter__(self): + i = 0 + try: + while True: + v = self[i] + yield v + i += 1 + except IndexError: + return + + def __contains__(self, value): + for v in self: + if v is value or v == value: + return True + return False + + def __reversed__(self): + for i in reversed(range(len(self))): + yield self[i] + + def index(self, value, start=0, stop=None): + '''S.index(value, [start, [stop]]) -> integer -- return first index of value. + Raises ValueError if the value is not present. + + Supporting start and stop arguments is optional, but + recommended. + ''' + if start is not None and start < 0: + start = max(len(self) + start, 0) + if stop is not None and stop < 0: + stop += len(self) + + i = start + while stop is None or i < stop: + try: + v = self[i] + except IndexError: + break + if v is value or v == value: + return i + i += 1 + raise ValueError + + def count(self, value): + 'S.count(value) -> integer -- return number of occurrences of value' + return sum(1 for v in self if v is value or v == value) + +Sequence.register(tuple) +Sequence.register(str) +Sequence.register(range) +Sequence.register(memoryview) + +class _DeprecateByteStringMeta(ABCMeta): + def __new__(cls, name, bases, namespace, **kwargs): + if name != "ByteString": + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__new__(cls, name, bases, namespace, **kwargs) + + def __instancecheck__(cls, instance): + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__instancecheck__(instance) + +class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): + """Deprecated ABC serving as a common supertype of ``bytes`` and ``bytearray``. + + This ABC is scheduled for removal in Python 3.17. + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the buffer protocol at runtime. For use in type annotations, + either use ``Buffer`` or a union that explicitly specifies the types your + code supports (e.g., ``bytes | bytearray | memoryview``). + """ + + __slots__ = () + +ByteString.register(bytes) +ByteString.register(bytearray) + + +class MutableSequence(Sequence): + """All the operations on a read-write sequence. + + Concrete subclasses must provide __new__ or __init__, + __getitem__, __setitem__, __delitem__, __len__, and insert(). + """ + + __slots__ = () + + @abstractmethod + def __setitem__(self, index, value): + raise IndexError + + @abstractmethod + def __delitem__(self, index): + raise IndexError + + @abstractmethod + def insert(self, index, value): + 'S.insert(index, value) -- insert value before index' + raise IndexError + + def append(self, value): + 'S.append(value) -- append value to the end of the sequence' + self.insert(len(self), value) + + def clear(self): + 'S.clear() -> None -- remove all items from S' + try: + while True: + self.pop() + except IndexError: + pass + + def reverse(self): + 'S.reverse() -- reverse *IN PLACE*' + n = len(self) + for i in range(n//2): + self[i], self[n-i-1] = self[n-i-1], self[i] + + def extend(self, values): + 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + if values is self: + values = list(values) + for v in values: + self.append(v) + + def pop(self, index=-1): + '''S.pop([index]) -> item -- remove and return item at index (default last). + Raise IndexError if list is empty or index is out of range. + ''' + v = self[index] + del self[index] + return v + + def remove(self, value): + '''S.remove(value) -- remove first occurrence of value. + Raise ValueError if the value is not present. + ''' + del self[self.index(value)] + + def __iadd__(self, values): + self.extend(values) + return self + + +MutableSequence.register(list) +MutableSequence.register(bytearray) # Multiply inheriting, see ByteString diff --git a/crates/weavepy-vm/src/stdlib/python/_collections_user.py b/crates/weavepy-vm/src/stdlib/python/_collections_user.py new file mode 100644 index 0000000..4397a0a --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_collections_user.py @@ -0,0 +1,491 @@ +"""``UserDict`` / ``UserList`` / ``UserString`` — carried verbatim from +CPython 3.13's ``collections/__init__.py`` (RFC 0037 WS8). + +These live in a sibling frozen module rather than inline in ``collections`` +so the package ``__init__`` (WeavePy's own deque/Counter/... implementation) +stays untouched; ``collections`` re-exports the three names from here. +""" + +import _collections_abc +import sys as _sys # CPython's collections/__init__.py aliases sys this way + +class UserDict(_collections_abc.MutableMapping): + + # Start by filling-out the abstract methods + def __init__(self, dict=None, /, **kwargs): + self.data = {} + if dict is not None: + self.update(dict) + if kwargs: + self.update(kwargs) + + def __len__(self): + return len(self.data) + + def __getitem__(self, key): + if key in self.data: + return self.data[key] + if hasattr(self.__class__, "__missing__"): + return self.__class__.__missing__(self, key) + raise KeyError(key) + + def __setitem__(self, key, item): + self.data[key] = item + + def __delitem__(self, key): + del self.data[key] + + def __iter__(self): + return iter(self.data) + + # Modify __contains__ and get() to work like dict + # does when __missing__ is present. + def __contains__(self, key): + return key in self.data + + def get(self, key, default=None): + if key in self: + return self[key] + return default + + + # Now, add the methods in dicts but not in MutableMapping + def __repr__(self): + return repr(self.data) + + def __or__(self, other): + if isinstance(other, UserDict): + return self.__class__(self.data | other.data) + if isinstance(other, dict): + return self.__class__(self.data | other) + return NotImplemented + + def __ror__(self, other): + if isinstance(other, UserDict): + return self.__class__(other.data | self.data) + if isinstance(other, dict): + return self.__class__(other | self.data) + return NotImplemented + + def __ior__(self, other): + if isinstance(other, UserDict): + self.data |= other.data + else: + self.data |= other + return self + + def __copy__(self): + inst = self.__class__.__new__(self.__class__) + inst.__dict__.update(self.__dict__) + # Create a copy and avoid triggering descriptors + inst.__dict__["data"] = self.__dict__["data"].copy() + return inst + + def copy(self): + if self.__class__ is UserDict: + return UserDict(self.data.copy()) + import copy + data = self.data + try: + self.data = {} + c = copy.copy(self) + finally: + self.data = data + c.update(self) + return c + + @classmethod + def fromkeys(cls, iterable, value=None): + d = cls() + for key in iterable: + d[key] = value + return d + + +################################################################################ +### UserList +################################################################################ + +class UserList(_collections_abc.MutableSequence): + """A more or less complete user-defined wrapper around list objects.""" + + def __init__(self, initlist=None): + self.data = [] + if initlist is not None: + # XXX should this accept an arbitrary sequence? + if type(initlist) == type(self.data): + self.data[:] = initlist + elif isinstance(initlist, UserList): + self.data[:] = initlist.data[:] + else: + self.data = list(initlist) + + def __repr__(self): + return repr(self.data) + + def __lt__(self, other): + return self.data < self.__cast(other) + + def __le__(self, other): + return self.data <= self.__cast(other) + + def __eq__(self, other): + return self.data == self.__cast(other) + + def __gt__(self, other): + return self.data > self.__cast(other) + + def __ge__(self, other): + return self.data >= self.__cast(other) + + def __cast(self, other): + return other.data if isinstance(other, UserList) else other + + def __contains__(self, item): + return item in self.data + + def __len__(self): + return len(self.data) + + def __getitem__(self, i): + if isinstance(i, slice): + return self.__class__(self.data[i]) + else: + return self.data[i] + + def __setitem__(self, i, item): + self.data[i] = item + + def __delitem__(self, i): + del self.data[i] + + def __add__(self, other): + if isinstance(other, UserList): + return self.__class__(self.data + other.data) + elif isinstance(other, type(self.data)): + return self.__class__(self.data + other) + return self.__class__(self.data + list(other)) + + def __radd__(self, other): + if isinstance(other, UserList): + return self.__class__(other.data + self.data) + elif isinstance(other, type(self.data)): + return self.__class__(other + self.data) + return self.__class__(list(other) + self.data) + + def __iadd__(self, other): + if isinstance(other, UserList): + self.data += other.data + elif isinstance(other, type(self.data)): + self.data += other + else: + self.data += list(other) + return self + + def __mul__(self, n): + return self.__class__(self.data * n) + + __rmul__ = __mul__ + + def __imul__(self, n): + self.data *= n + return self + + def __copy__(self): + inst = self.__class__.__new__(self.__class__) + inst.__dict__.update(self.__dict__) + # Create a copy and avoid triggering descriptors + inst.__dict__["data"] = self.__dict__["data"][:] + return inst + + def append(self, item): + self.data.append(item) + + def insert(self, i, item): + self.data.insert(i, item) + + def pop(self, i=-1): + return self.data.pop(i) + + def remove(self, item): + self.data.remove(item) + + def clear(self): + self.data.clear() + + def copy(self): + return self.__class__(self) + + def count(self, item): + return self.data.count(item) + + def index(self, item, *args): + return self.data.index(item, *args) + + def reverse(self): + self.data.reverse() + + def sort(self, /, *args, **kwds): + self.data.sort(*args, **kwds) + + def extend(self, other): + if isinstance(other, UserList): + self.data.extend(other.data) + else: + self.data.extend(other) + + +################################################################################ +### UserString +################################################################################ + +class UserString(_collections_abc.Sequence): + + def __init__(self, seq): + if isinstance(seq, str): + self.data = seq + elif isinstance(seq, UserString): + self.data = seq.data[:] + else: + self.data = str(seq) + + def __str__(self): + return str(self.data) + + def __repr__(self): + return repr(self.data) + + def __int__(self): + return int(self.data) + + def __float__(self): + return float(self.data) + + def __complex__(self): + return complex(self.data) + + def __hash__(self): + return hash(self.data) + + def __getnewargs__(self): + return (self.data[:],) + + def __eq__(self, string): + if isinstance(string, UserString): + return self.data == string.data + return self.data == string + + def __lt__(self, string): + if isinstance(string, UserString): + return self.data < string.data + return self.data < string + + def __le__(self, string): + if isinstance(string, UserString): + return self.data <= string.data + return self.data <= string + + def __gt__(self, string): + if isinstance(string, UserString): + return self.data > string.data + return self.data > string + + def __ge__(self, string): + if isinstance(string, UserString): + return self.data >= string.data + return self.data >= string + + def __contains__(self, char): + if isinstance(char, UserString): + char = char.data + return char in self.data + + def __len__(self): + return len(self.data) + + def __getitem__(self, index): + return self.__class__(self.data[index]) + + def __add__(self, other): + if isinstance(other, UserString): + return self.__class__(self.data + other.data) + elif isinstance(other, str): + return self.__class__(self.data + other) + return self.__class__(self.data + str(other)) + + def __radd__(self, other): + if isinstance(other, str): + return self.__class__(other + self.data) + return self.__class__(str(other) + self.data) + + def __mul__(self, n): + return self.__class__(self.data * n) + + __rmul__ = __mul__ + + def __mod__(self, args): + return self.__class__(self.data % args) + + def __rmod__(self, template): + return self.__class__(str(template) % self) + + # the following methods are defined in alphabetical order: + def capitalize(self): + return self.__class__(self.data.capitalize()) + + def casefold(self): + return self.__class__(self.data.casefold()) + + def center(self, width, *args): + return self.__class__(self.data.center(width, *args)) + + def count(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.count(sub, start, end) + + def removeprefix(self, prefix, /): + if isinstance(prefix, UserString): + prefix = prefix.data + return self.__class__(self.data.removeprefix(prefix)) + + def removesuffix(self, suffix, /): + if isinstance(suffix, UserString): + suffix = suffix.data + return self.__class__(self.data.removesuffix(suffix)) + + def encode(self, encoding='utf-8', errors='strict'): + encoding = 'utf-8' if encoding is None else encoding + errors = 'strict' if errors is None else errors + return self.data.encode(encoding, errors) + + def endswith(self, suffix, start=0, end=_sys.maxsize): + return self.data.endswith(suffix, start, end) + + def expandtabs(self, tabsize=8): + return self.__class__(self.data.expandtabs(tabsize)) + + def find(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.find(sub, start, end) + + def format(self, /, *args, **kwds): + return self.data.format(*args, **kwds) + + def format_map(self, mapping): + return self.data.format_map(mapping) + + def index(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.index(sub, start, end) + + def isalpha(self): + return self.data.isalpha() + + def isalnum(self): + return self.data.isalnum() + + def isascii(self): + return self.data.isascii() + + def isdecimal(self): + return self.data.isdecimal() + + def isdigit(self): + return self.data.isdigit() + + def isidentifier(self): + return self.data.isidentifier() + + def islower(self): + return self.data.islower() + + def isnumeric(self): + return self.data.isnumeric() + + def isprintable(self): + return self.data.isprintable() + + def isspace(self): + return self.data.isspace() + + def istitle(self): + return self.data.istitle() + + def isupper(self): + return self.data.isupper() + + def join(self, seq): + return self.data.join(seq) + + def ljust(self, width, *args): + return self.__class__(self.data.ljust(width, *args)) + + def lower(self): + return self.__class__(self.data.lower()) + + def lstrip(self, chars=None): + return self.__class__(self.data.lstrip(chars)) + + maketrans = str.maketrans + + def partition(self, sep): + return self.data.partition(sep) + + def replace(self, old, new, maxsplit=-1): + if isinstance(old, UserString): + old = old.data + if isinstance(new, UserString): + new = new.data + return self.__class__(self.data.replace(old, new, maxsplit)) + + def rfind(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.rfind(sub, start, end) + + def rindex(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.rindex(sub, start, end) + + def rjust(self, width, *args): + return self.__class__(self.data.rjust(width, *args)) + + def rpartition(self, sep): + return self.data.rpartition(sep) + + def rstrip(self, chars=None): + return self.__class__(self.data.rstrip(chars)) + + def split(self, sep=None, maxsplit=-1): + return self.data.split(sep, maxsplit) + + def rsplit(self, sep=None, maxsplit=-1): + return self.data.rsplit(sep, maxsplit) + + def splitlines(self, keepends=False): + return self.data.splitlines(keepends) + + def startswith(self, prefix, start=0, end=_sys.maxsize): + return self.data.startswith(prefix, start, end) + + def strip(self, chars=None): + return self.__class__(self.data.strip(chars)) + + def swapcase(self): + return self.__class__(self.data.swapcase()) + + def title(self): + return self.__class__(self.data.title()) + + def translate(self, *args): + return self.__class__(self.data.translate(*args)) + + def upper(self): + return self.__class__(self.data.upper()) + + def zfill(self, width): + return self.__class__(self.data.zfill(width)) diff --git a/crates/weavepy-vm/src/stdlib/python/_colorize.py b/crates/weavepy-vm/src/stdlib/python/_colorize.py new file mode 100644 index 0000000..8263d2d --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_colorize.py @@ -0,0 +1,119 @@ +from __future__ import annotations +import os +import sys + +COLORIZE = True + +# types +if False: + from typing import IO + + +class ANSIColors: + RESET = "\x1b[0m" + + BLACK = "\x1b[30m" + BLUE = "\x1b[34m" + CYAN = "\x1b[36m" + GREEN = "\x1b[32m" + MAGENTA = "\x1b[35m" + RED = "\x1b[31m" + WHITE = "\x1b[37m" # more like LIGHT GRAY + YELLOW = "\x1b[33m" + + BOLD_BLACK = "\x1b[1;30m" # DARK GRAY + BOLD_BLUE = "\x1b[1;34m" + BOLD_CYAN = "\x1b[1;36m" + BOLD_GREEN = "\x1b[1;32m" + BOLD_MAGENTA = "\x1b[1;35m" + BOLD_RED = "\x1b[1;31m" + BOLD_WHITE = "\x1b[1;37m" # actual WHITE + BOLD_YELLOW = "\x1b[1;33m" + + # intense = like bold but without being bold + INTENSE_BLACK = "\x1b[90m" + INTENSE_BLUE = "\x1b[94m" + INTENSE_CYAN = "\x1b[96m" + INTENSE_GREEN = "\x1b[92m" + INTENSE_MAGENTA = "\x1b[95m" + INTENSE_RED = "\x1b[91m" + INTENSE_WHITE = "\x1b[97m" + INTENSE_YELLOW = "\x1b[93m" + + BACKGROUND_BLACK = "\x1b[40m" + BACKGROUND_BLUE = "\x1b[44m" + BACKGROUND_CYAN = "\x1b[46m" + BACKGROUND_GREEN = "\x1b[42m" + BACKGROUND_MAGENTA = "\x1b[45m" + BACKGROUND_RED = "\x1b[41m" + BACKGROUND_WHITE = "\x1b[47m" + BACKGROUND_YELLOW = "\x1b[43m" + + INTENSE_BACKGROUND_BLACK = "\x1b[100m" + INTENSE_BACKGROUND_BLUE = "\x1b[104m" + INTENSE_BACKGROUND_CYAN = "\x1b[106m" + INTENSE_BACKGROUND_GREEN = "\x1b[102m" + INTENSE_BACKGROUND_MAGENTA = "\x1b[105m" + INTENSE_BACKGROUND_RED = "\x1b[101m" + INTENSE_BACKGROUND_WHITE = "\x1b[107m" + INTENSE_BACKGROUND_YELLOW = "\x1b[103m" + + +NoColors = ANSIColors() + +for attr in dir(NoColors): + if not attr.startswith("__"): + setattr(NoColors, attr, "") + + +def get_colors( + colorize: bool = False, *, file: IO[str] | IO[bytes] | None = None +) -> ANSIColors: + if colorize or can_colorize(file=file): + return ANSIColors() + else: + return NoColors + + +def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: + + def _safe_getenv(k: str, fallback: str | None = None) -> str | None: + """Exception-safe environment retrieval. See gh-128636.""" + try: + return os.environ.get(k, fallback) + except Exception: + return fallback + + if file is None: + file = sys.stdout + + if not sys.flags.ignore_environment: + if _safe_getenv("PYTHON_COLORS") == "0": + return False + if _safe_getenv("PYTHON_COLORS") == "1": + return True + if _safe_getenv("NO_COLOR"): + return False + if not COLORIZE: + return False + if _safe_getenv("FORCE_COLOR"): + return True + if _safe_getenv("TERM") == "dumb": + return False + + if not hasattr(file, "fileno"): + return False + + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + + try: + return os.isatty(file.fileno()) + except OSError: + return hasattr(file, "isatty") and file.isatty() diff --git a/crates/weavepy-vm/src/stdlib/python/_py_abc.py b/crates/weavepy-vm/src/stdlib/python/_py_abc.py new file mode 100644 index 0000000..c870ae9 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_py_abc.py @@ -0,0 +1,147 @@ +from _weakrefset import WeakSet + + +def get_cache_token(): + """Returns the current ABC cache token. + + The token is an opaque object (supporting equality testing) identifying the + current version of the ABC cache for virtual subclasses. The token changes + with every call to ``register()`` on any ABC. + """ + return ABCMeta._abc_invalidation_counter + + +class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + + # A global counter that is incremented each time a class is + # registered as a virtual subclass of anything. It forces the + # negative cache to be cleared before its next use. + # Note: this counter is private. Use `abc.get_cache_token()` for + # external code. + _abc_invalidation_counter = 0 + + def __new__(mcls, name, bases, namespace, /, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + # Compute set of abstract method names + abstracts = {name + for name, value in namespace.items() + if getattr(value, "__isabstractmethod__", False)} + for base in bases: + for name in getattr(base, "__abstractmethods__", set()): + value = getattr(cls, name, None) + if getattr(value, "__isabstractmethod__", False): + abstracts.add(name) + cls.__abstractmethods__ = frozenset(abstracts) + # Set up inheritance registry + cls._abc_registry = WeakSet() + cls._abc_cache = WeakSet() + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + if not isinstance(subclass, type): + raise TypeError("Can only register classes") + if issubclass(subclass, cls): + return subclass # Already a subclass + # Subtle: test for cycles *after* testing for "already a subclass"; + # this means we allow X.register(X) and interpret it as a no-op. + if issubclass(cls, subclass): + # This would create a cycle, which is bad for the algorithm below + raise RuntimeError("Refusing to create an inheritance cycle") + cls._abc_registry.add(subclass) + ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache + return subclass + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + for name in cls.__dict__: + if name.startswith("_abc_"): + value = getattr(cls, name) + if isinstance(value, WeakSet): + value = set(value) + print(f"{name}: {value!r}", file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + cls._abc_registry.clear() + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + cls._abc_cache.clear() + cls._abc_negative_cache.clear() + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + # Inline the cache checking + subclass = instance.__class__ + if subclass in cls._abc_cache: + return True + subtype = type(instance) + if subtype is subclass: + if (cls._abc_negative_cache_version == + ABCMeta._abc_invalidation_counter and + subclass in cls._abc_negative_cache): + return False + # Fall back to the subclass check. + return cls.__subclasscheck__(subclass) + return any(cls.__subclasscheck__(c) for c in (subclass, subtype)) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + if not isinstance(subclass, type): + raise TypeError('issubclass() arg 1 must be a class') + # Check cache + if subclass in cls._abc_cache: + return True + # Check negative cache; may have to invalidate + if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: + # Invalidate the negative cache + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + elif subclass in cls._abc_negative_cache: + return False + # Check the subclass hook + ok = cls.__subclasshook__(subclass) + if ok is not NotImplemented: + assert isinstance(ok, bool) + if ok: + cls._abc_cache.add(subclass) + else: + cls._abc_negative_cache.add(subclass) + return ok + # Check if it's a direct subclass + if cls in getattr(subclass, '__mro__', ()): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a registered class (recursive) + for rcls in cls._abc_registry: + if issubclass(subclass, rcls): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a subclass (recursive) + for scls in cls.__subclasses__(): + if issubclass(subclass, scls): + cls._abc_cache.add(subclass) + return True + # No dice; update negative cache + cls._abc_negative_cache.add(subclass) + return False diff --git a/crates/weavepy-vm/src/stdlib/python/_pydecimal.py b/crates/weavepy-vm/src/stdlib/python/_pydecimal.py new file mode 100644 index 0000000..49119ed --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_pydecimal.py @@ -0,0 +1,6351 @@ +# Copyright (c) 2004 Python Software Foundation. +# All rights reserved. + +# Written by Eric Price +# and Facundo Batista +# and Raymond Hettinger +# and Aahz +# and Tim Peters + +# This module should be kept in sync with the latest updates of the +# IBM specification as it evolves. Those updates will be treated +# as bug fixes (deviation from the spec is a compatibility, usability +# bug) and will be backported. At this point the spec is stabilizing +# and the updates are becoming fewer, smaller, and less significant. + +"""Python decimal arithmetic module""" + +__all__ = [ + # Two major classes + 'Decimal', 'Context', + + # Named tuple representation + 'DecimalTuple', + + # Contexts + 'DefaultContext', 'BasicContext', 'ExtendedContext', + + # Exceptions + 'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero', + 'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow', + 'FloatOperation', + + # Exceptional conditions that trigger InvalidOperation + 'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined', + + # Constants for use in setting up contexts + 'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING', + 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', + + # Functions for manipulating contexts + 'setcontext', 'getcontext', 'localcontext', + + # Limits for the C version for compatibility + 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', + + # C version: compile time choice that enables the thread local context (deprecated, now always true) + 'HAVE_THREADS', + + # C version: compile time choice that enables the coroutine local context + 'HAVE_CONTEXTVAR' +] + +__xname__ = __name__ # sys.modules lookup (--without-threads) +__name__ = 'decimal' # For pickling +__version__ = '1.70' # Highest version of the spec this complies with + # See http://speleotrove.com/decimal/ +__libmpdec_version__ = "2.4.2" # compatible libmpdec version + +import math as _math +import numbers as _numbers +import sys + +try: + from collections import namedtuple as _namedtuple + DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent', module='decimal') +except ImportError: + DecimalTuple = lambda *args: args + +# Rounding +ROUND_DOWN = 'ROUND_DOWN' +ROUND_HALF_UP = 'ROUND_HALF_UP' +ROUND_HALF_EVEN = 'ROUND_HALF_EVEN' +ROUND_CEILING = 'ROUND_CEILING' +ROUND_FLOOR = 'ROUND_FLOOR' +ROUND_UP = 'ROUND_UP' +ROUND_HALF_DOWN = 'ROUND_HALF_DOWN' +ROUND_05UP = 'ROUND_05UP' + +# Compatibility with the C version +HAVE_THREADS = True +HAVE_CONTEXTVAR = True +if sys.maxsize == 2**63-1: + MAX_PREC = 999999999999999999 + MAX_EMAX = 999999999999999999 + MIN_EMIN = -999999999999999999 +else: + MAX_PREC = 425000000 + MAX_EMAX = 425000000 + MIN_EMIN = -425000000 + +MIN_ETINY = MIN_EMIN - (MAX_PREC-1) + +# Errors + +class DecimalException(ArithmeticError): + """Base exception class. + + Used exceptions derive from this. + If an exception derives from another exception besides this (such as + Underflow (Inexact, Rounded, Subnormal)) that indicates that it is only + called if the others are present. This isn't actually used for + anything, though. + + handle -- Called when context._raise_error is called and the + trap_enabler is not set. First argument is self, second is the + context. More arguments can be given, those being after + the explanation in _raise_error (For example, + context._raise_error(NewError, '(-x)!', self._sign) would + call NewError().handle(context, self._sign).) + + To define a new exception, it should be sufficient to have it derive + from DecimalException. + """ + def handle(self, context, *args): + pass + + +class Clamped(DecimalException): + """Exponent of a 0 changed to fit bounds. + + This occurs and signals clamped if the exponent of a result has been + altered in order to fit the constraints of a specific concrete + representation. This may occur when the exponent of a zero result would + be outside the bounds of a representation, or when a large normal + number would have an encoded exponent that cannot be represented. In + this latter case, the exponent is reduced to fit and the corresponding + number of zero digits are appended to the coefficient ("fold-down"). + """ + +class InvalidOperation(DecimalException): + """An invalid operation was performed. + + Various bad things cause this: + + Something creates a signaling NaN + -INF + INF + 0 * (+-)INF + (+-)INF / (+-)INF + x % 0 + (+-)INF % x + x._rescale( non-integer ) + sqrt(-x) , x > 0 + 0 ** 0 + x ** (non-integer) + x ** (+-)INF + An operand is invalid + + The result of the operation after this is a quiet positive NaN, + except when the cause is a signaling NaN, in which case the result is + also a quiet NaN, but with the original sign, and an optional + diagnostic information. + """ + def handle(self, context, *args): + if args: + ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True) + return ans._fix_nan(context) + return _NaN + +class ConversionSyntax(InvalidOperation): + """Trying to convert badly formed string. + + This occurs and signals invalid-operation if a string is being + converted to a number and it does not conform to the numeric string + syntax. The result is [0,qNaN]. + """ + def handle(self, context, *args): + return _NaN + +class DivisionByZero(DecimalException, ZeroDivisionError): + """Division by 0. + + This occurs and signals division-by-zero if division of a finite number + by zero was attempted (during a divide-integer or divide operation, or a + power operation with negative right-hand operand), and the dividend was + not zero. + + The result of the operation is [sign,inf], where sign is the exclusive + or of the signs of the operands for divide, or is 1 for an odd power of + -0, for power. + """ + + def handle(self, context, sign, *args): + return _SignedInfinity[sign] + +class DivisionImpossible(InvalidOperation): + """Cannot perform the division adequately. + + This occurs and signals invalid-operation if the integer result of a + divide-integer or remainder operation had too many digits (would be + longer than precision). The result is [0,qNaN]. + """ + + def handle(self, context, *args): + return _NaN + +class DivisionUndefined(InvalidOperation, ZeroDivisionError): + """Undefined result of division. + + This occurs and signals invalid-operation if division by zero was + attempted (during a divide-integer, divide, or remainder operation), and + the dividend is also zero. The result is [0,qNaN]. + """ + + def handle(self, context, *args): + return _NaN + +class Inexact(DecimalException): + """Had to round, losing information. + + This occurs and signals inexact whenever the result of an operation is + not exact (that is, it needed to be rounded and any discarded digits + were non-zero), or if an overflow or underflow condition occurs. The + result in all cases is unchanged. + + The inexact signal may be tested (or trapped) to determine if a given + operation (or sequence of operations) was inexact. + """ + +class InvalidContext(InvalidOperation): + """Invalid context. Unknown rounding, for example. + + This occurs and signals invalid-operation if an invalid context was + detected during an operation. This can occur if contexts are not checked + on creation and either the precision exceeds the capability of the + underlying concrete representation or an unknown or unsupported rounding + was specified. These aspects of the context need only be checked when + the values are required to be used. The result is [0,qNaN]. + """ + + def handle(self, context, *args): + return _NaN + +class Rounded(DecimalException): + """Number got rounded (not necessarily changed during rounding). + + This occurs and signals rounded whenever the result of an operation is + rounded (that is, some zero or non-zero digits were discarded from the + coefficient), or if an overflow or underflow condition occurs. The + result in all cases is unchanged. + + The rounded signal may be tested (or trapped) to determine if a given + operation (or sequence of operations) caused a loss of precision. + """ + +class Subnormal(DecimalException): + """Exponent < Emin before rounding. + + This occurs and signals subnormal whenever the result of a conversion or + operation is subnormal (that is, its adjusted exponent is less than + Emin, before any rounding). The result in all cases is unchanged. + + The subnormal signal may be tested (or trapped) to determine if a given + or operation (or sequence of operations) yielded a subnormal result. + """ + +class Overflow(Inexact, Rounded): + """Numerical overflow. + + This occurs and signals overflow if the adjusted exponent of a result + (from a conversion or from an operation that is not an attempt to divide + by zero), after rounding, would be greater than the largest value that + can be handled by the implementation (the value Emax). + + The result depends on the rounding mode: + + For round-half-up and round-half-even (and for round-half-down and + round-up, if implemented), the result of the operation is [sign,inf], + where sign is the sign of the intermediate result. For round-down, the + result is the largest finite number that can be represented in the + current precision, with the sign of the intermediate result. For + round-ceiling, the result is the same as for round-down if the sign of + the intermediate result is 1, or is [0,inf] otherwise. For round-floor, + the result is the same as for round-down if the sign of the intermediate + result is 0, or is [1,inf] otherwise. In all cases, Inexact and Rounded + will also be raised. + """ + + def handle(self, context, sign, *args): + if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN, + ROUND_HALF_DOWN, ROUND_UP): + return _SignedInfinity[sign] + if sign == 0: + if context.rounding == ROUND_CEILING: + return _SignedInfinity[sign] + return _dec_from_triple(sign, '9'*context.prec, + context.Emax-context.prec+1) + if sign == 1: + if context.rounding == ROUND_FLOOR: + return _SignedInfinity[sign] + return _dec_from_triple(sign, '9'*context.prec, + context.Emax-context.prec+1) + + +class Underflow(Inexact, Rounded, Subnormal): + """Numerical underflow with result rounded to 0. + + This occurs and signals underflow if a result is inexact and the + adjusted exponent of the result would be smaller (more negative) than + the smallest value that can be handled by the implementation (the value + Emin). That is, the result is both inexact and subnormal. + + The result after an underflow will be a subnormal number rounded, if + necessary, so that its exponent is not less than Etiny. This may result + in 0 with the sign of the intermediate result and an exponent of Etiny. + + In all cases, Inexact, Rounded, and Subnormal will also be raised. + """ + +class FloatOperation(DecimalException, TypeError): + """Enable stricter semantics for mixing floats and Decimals. + + If the signal is not trapped (default), mixing floats and Decimals is + permitted in the Decimal() constructor, context.create_decimal() and + all comparison operators. Both conversion and comparisons are exact. + Any occurrence of a mixed operation is silently recorded by setting + FloatOperation in the context flags. Explicit conversions with + Decimal.from_float() or context.create_decimal_from_float() do not + set the flag. + + Otherwise (the signal is trapped), only equality comparisons and explicit + conversions are silent. All other mixed operations raise FloatOperation. + """ + +# List of public traps and flags +_signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded, + Underflow, InvalidOperation, Subnormal, FloatOperation] + +# Map conditions (per the spec) to signals +_condition_map = {ConversionSyntax:InvalidOperation, + DivisionImpossible:InvalidOperation, + DivisionUndefined:InvalidOperation, + InvalidContext:InvalidOperation} + +# Valid rounding modes +_rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING, + ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP) + +##### Context Functions ################################################## + +# The getcontext() and setcontext() function manage access to a thread-local +# current context. + +import contextvars + +_current_context_var = contextvars.ContextVar('decimal_context') + +_context_attributes = frozenset( + ['prec', 'Emin', 'Emax', 'capitals', 'clamp', 'rounding', 'flags', 'traps'] +) + +def getcontext(): + """Returns this thread's context. + + If this thread does not yet have a context, returns + a new context and sets this thread's context. + New contexts are copies of DefaultContext. + """ + try: + return _current_context_var.get() + except LookupError: + context = Context() + _current_context_var.set(context) + return context + +def setcontext(context): + """Set this thread's context to context.""" + if context in (DefaultContext, BasicContext, ExtendedContext): + context = context.copy() + context.clear_flags() + _current_context_var.set(context) + +del contextvars # Don't contaminate the namespace + +def localcontext(ctx=None, **kwargs): + """Return a context manager for a copy of the supplied context + + Uses a copy of the current context if no context is specified + The returned context manager creates a local decimal context + in a with statement: + def sin(x): + with localcontext() as ctx: + ctx.prec += 2 + # Rest of sin calculation algorithm + # uses a precision 2 greater than normal + return +s # Convert result to normal precision + + def sin(x): + with localcontext(ExtendedContext): + # Rest of sin calculation algorithm + # uses the Extended Context from the + # General Decimal Arithmetic Specification + return +s # Convert result to normal context + + >>> setcontext(DefaultContext) + >>> print(getcontext().prec) + 28 + >>> with localcontext(): + ... ctx = getcontext() + ... ctx.prec += 2 + ... print(ctx.prec) + ... + 30 + >>> with localcontext(ExtendedContext): + ... print(getcontext().prec) + ... + 9 + >>> print(getcontext().prec) + 28 + """ + if ctx is None: + ctx = getcontext() + ctx_manager = _ContextManager(ctx) + for key, value in kwargs.items(): + if key not in _context_attributes: + raise TypeError(f"'{key}' is an invalid keyword argument for this function") + setattr(ctx_manager.new_context, key, value) + return ctx_manager + + +##### Decimal class ####################################################### + +# Do not subclass Decimal from numbers.Real and do not register it as such +# (because Decimals are not interoperable with floats). See the notes in +# numbers.py for more detail. + +class Decimal(object): + """Floating-point class for decimal arithmetic.""" + + __slots__ = ('_exp','_int','_sign', '_is_special') + # Generally, the value of the Decimal instance is given by + # (-1)**_sign * _int * 10**_exp + # Special values are signified by _is_special == True + + # We're immutable, so use __new__ not __init__ + def __new__(cls, value="0", context=None): + """Create a decimal point instance. + + >>> Decimal('3.14') # string input + Decimal('3.14') + >>> Decimal((0, (3, 1, 4), -2)) # tuple (sign, digit_tuple, exponent) + Decimal('3.14') + >>> Decimal(314) # int + Decimal('314') + >>> Decimal(Decimal(314)) # another decimal instance + Decimal('314') + >>> Decimal(' 3.14 \\n') # leading and trailing whitespace okay + Decimal('3.14') + """ + + # Note that the coefficient, self._int, is actually stored as + # a string rather than as a tuple of digits. This speeds up + # the "digits to integer" and "integer to digits" conversions + # that are used in almost every arithmetic operation on + # Decimals. This is an internal detail: the as_tuple function + # and the Decimal constructor still deal with tuples of + # digits. + + self = object.__new__(cls) + + # From a string + # REs insist on real strings, so we can too. + if isinstance(value, str): + m = _parser(value.strip().replace("_", "")) + if m is None: + if context is None: + context = getcontext() + return context._raise_error(ConversionSyntax, + "Invalid literal for Decimal: %r" % value) + + if m.group('sign') == "-": + self._sign = 1 + else: + self._sign = 0 + intpart = m.group('int') + if intpart is not None: + # finite number + fracpart = m.group('frac') or '' + exp = int(m.group('exp') or '0') + self._int = str(int(intpart+fracpart)) + self._exp = exp - len(fracpart) + self._is_special = False + else: + diag = m.group('diag') + if diag is not None: + # NaN + self._int = str(int(diag or '0')).lstrip('0') + if m.group('signal'): + self._exp = 'N' + else: + self._exp = 'n' + else: + # infinity + self._int = '0' + self._exp = 'F' + self._is_special = True + return self + + # From an integer + if isinstance(value, int): + if value >= 0: + self._sign = 0 + else: + self._sign = 1 + self._exp = 0 + self._int = str(abs(value)) + self._is_special = False + return self + + # From another decimal + if isinstance(value, Decimal): + self._exp = value._exp + self._sign = value._sign + self._int = value._int + self._is_special = value._is_special + return self + + # From an internal working value + if isinstance(value, _WorkRep): + self._sign = value.sign + self._int = str(value.int) + self._exp = int(value.exp) + self._is_special = False + return self + + # tuple/list conversion (possibly from as_tuple()) + if isinstance(value, (list,tuple)): + if len(value) != 3: + raise ValueError('Invalid tuple size in creation of Decimal ' + 'from list or tuple. The list or tuple ' + 'should have exactly three elements.') + # process sign. The isinstance test rejects floats + if not (isinstance(value[0], int) and value[0] in (0,1)): + raise ValueError("Invalid sign. The first value in the tuple " + "should be an integer; either 0 for a " + "positive number or 1 for a negative number.") + self._sign = value[0] + if value[2] == 'F': + # infinity: value[1] is ignored + self._int = '0' + self._exp = value[2] + self._is_special = True + else: + # process and validate the digits in value[1] + digits = [] + for digit in value[1]: + if isinstance(digit, int) and 0 <= digit <= 9: + # skip leading zeros + if digits or digit != 0: + digits.append(digit) + else: + raise ValueError("The second value in the tuple must " + "be composed of integers in the range " + "0 through 9.") + if value[2] in ('n', 'N'): + # NaN: digits form the diagnostic + self._int = ''.join(map(str, digits)) + self._exp = value[2] + self._is_special = True + elif isinstance(value[2], int): + # finite number: digits give the coefficient + self._int = ''.join(map(str, digits or [0])) + self._exp = value[2] + self._is_special = False + else: + raise ValueError("The third value in the tuple must " + "be an integer, or one of the " + "strings 'F', 'n', 'N'.") + return self + + if isinstance(value, float): + if context is None: + context = getcontext() + context._raise_error(FloatOperation, + "strict semantics for mixing floats and Decimals are " + "enabled") + value = Decimal.from_float(value) + self._exp = value._exp + self._sign = value._sign + self._int = value._int + self._is_special = value._is_special + return self + + raise TypeError("Cannot convert %r to Decimal" % value) + + @classmethod + def from_float(cls, f): + """Converts a float to a decimal number, exactly. + + Note that Decimal.from_float(0.1) is not the same as Decimal('0.1'). + Since 0.1 is not exactly representable in binary floating point, the + value is stored as the nearest representable value which is + 0x1.999999999999ap-4. The exact equivalent of the value in decimal + is 0.1000000000000000055511151231257827021181583404541015625. + + >>> Decimal.from_float(0.1) + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_float(float('nan')) + Decimal('NaN') + >>> Decimal.from_float(float('inf')) + Decimal('Infinity') + >>> Decimal.from_float(-float('inf')) + Decimal('-Infinity') + >>> Decimal.from_float(-0.0) + Decimal('-0') + + """ + if isinstance(f, int): # handle integer inputs + sign = 0 if f >= 0 else 1 + k = 0 + coeff = str(abs(f)) + elif isinstance(f, float): + if _math.isinf(f) or _math.isnan(f): + return cls(repr(f)) + if _math.copysign(1.0, f) == 1.0: + sign = 0 + else: + sign = 1 + n, d = abs(f).as_integer_ratio() + k = d.bit_length() - 1 + coeff = str(n*5**k) + else: + raise TypeError("argument must be int or float.") + + result = _dec_from_triple(sign, coeff, -k) + if cls is Decimal: + return result + else: + return cls(result) + + def _isnan(self): + """Returns whether the number is not actually one. + + 0 if a number + 1 if NaN + 2 if sNaN + """ + if self._is_special: + exp = self._exp + if exp == 'n': + return 1 + elif exp == 'N': + return 2 + return 0 + + def _isinfinity(self): + """Returns whether the number is infinite + + 0 if finite or not a number + 1 if +INF + -1 if -INF + """ + if self._exp == 'F': + if self._sign: + return -1 + return 1 + return 0 + + def _check_nans(self, other=None, context=None): + """Returns whether the number is not actually one. + + if self, other are sNaN, signal + if self, other are NaN return nan + return 0 + + Done before operations. + """ + + self_is_nan = self._isnan() + if other is None: + other_is_nan = False + else: + other_is_nan = other._isnan() + + if self_is_nan or other_is_nan: + if context is None: + context = getcontext() + + if self_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + self) + if other_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + other) + if self_is_nan: + return self._fix_nan(context) + + return other._fix_nan(context) + return 0 + + def _compare_check_nans(self, other, context): + """Version of _check_nans used for the signaling comparisons + compare_signal, __le__, __lt__, __ge__, __gt__. + + Signal InvalidOperation if either self or other is a (quiet + or signaling) NaN. Signaling NaNs take precedence over quiet + NaNs. + + Return 0 if neither operand is a NaN. + + """ + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + if self.is_snan(): + return context._raise_error(InvalidOperation, + 'comparison involving sNaN', + self) + elif other.is_snan(): + return context._raise_error(InvalidOperation, + 'comparison involving sNaN', + other) + elif self.is_qnan(): + return context._raise_error(InvalidOperation, + 'comparison involving NaN', + self) + elif other.is_qnan(): + return context._raise_error(InvalidOperation, + 'comparison involving NaN', + other) + return 0 + + def __bool__(self): + """Return True if self is nonzero; otherwise return False. + + NaNs and infinities are considered nonzero. + """ + return self._is_special or self._int != '0' + + def _cmp(self, other): + """Compare the two non-NaN decimal instances self and other. + + Returns -1 if self < other, 0 if self == other and 1 + if self > other. This routine is for internal use only.""" + + if self._is_special or other._is_special: + self_inf = self._isinfinity() + other_inf = other._isinfinity() + if self_inf == other_inf: + return 0 + elif self_inf < other_inf: + return -1 + else: + return 1 + + # check for zeros; Decimal('0') == Decimal('-0') + if not self: + if not other: + return 0 + else: + return -((-1)**other._sign) + if not other: + return (-1)**self._sign + + # If different signs, neg one is less + if other._sign < self._sign: + return -1 + if self._sign < other._sign: + return 1 + + self_adjusted = self.adjusted() + other_adjusted = other.adjusted() + if self_adjusted == other_adjusted: + self_padded = self._int + '0'*(self._exp - other._exp) + other_padded = other._int + '0'*(other._exp - self._exp) + if self_padded == other_padded: + return 0 + elif self_padded < other_padded: + return -(-1)**self._sign + else: + return (-1)**self._sign + elif self_adjusted > other_adjusted: + return (-1)**self._sign + else: # self_adjusted < other_adjusted + return -((-1)**self._sign) + + # Note: The Decimal standard doesn't cover rich comparisons for + # Decimals. In particular, the specification is silent on the + # subject of what should happen for a comparison involving a NaN. + # We take the following approach: + # + # == comparisons involving a quiet NaN always return False + # != comparisons involving a quiet NaN always return True + # == or != comparisons involving a signaling NaN signal + # InvalidOperation, and return False or True as above if the + # InvalidOperation is not trapped. + # <, >, <= and >= comparisons involving a (quiet or signaling) + # NaN signal InvalidOperation, and return False if the + # InvalidOperation is not trapped. + # + # This behavior is designed to conform as closely as possible to + # that specified by IEEE 754. + + def __eq__(self, other, context=None): + self, other = _convert_for_comparison(self, other, equality_op=True) + if other is NotImplemented: + return other + if self._check_nans(other, context): + return False + return self._cmp(other) == 0 + + def __lt__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) < 0 + + def __le__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) <= 0 + + def __gt__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) > 0 + + def __ge__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) >= 0 + + def compare(self, other, context=None): + """Compare self to other. Return a decimal value: + + a or b is a NaN ==> Decimal('NaN') + a < b ==> Decimal('-1') + a == b ==> Decimal('0') + a > b ==> Decimal('1') + """ + other = _convert_other(other, raiseit=True) + + # Compare(NaN, NaN) = NaN + if (self._is_special or other and other._is_special): + ans = self._check_nans(other, context) + if ans: + return ans + + return Decimal(self._cmp(other)) + + def __hash__(self): + """x.__hash__() <==> hash(x)""" + + # In order to make sure that the hash of a Decimal instance + # agrees with the hash of a numerically equal integer, float + # or Fraction, we follow the rules for numeric hashes outlined + # in the documentation. (See library docs, 'Built-in Types'). + if self._is_special: + if self.is_snan(): + raise TypeError('Cannot hash a signaling NaN value.') + elif self.is_nan(): + return object.__hash__(self) + else: + if self._sign: + return -_PyHASH_INF + else: + return _PyHASH_INF + + if self._exp >= 0: + exp_hash = pow(10, self._exp, _PyHASH_MODULUS) + else: + exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MODULUS) + hash_ = int(self._int) * exp_hash % _PyHASH_MODULUS + ans = hash_ if self >= 0 else -hash_ + return -2 if ans == -1 else ans + + def as_tuple(self): + """Represents the number as a triple tuple. + + To show the internals exactly as they are. + """ + return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) + + def as_integer_ratio(self): + """Express a finite Decimal instance in the form n / d. + + Returns a pair (n, d) of integers. When called on an infinity + or NaN, raises OverflowError or ValueError respectively. + + >>> Decimal('3.14').as_integer_ratio() + (157, 50) + >>> Decimal('-123e5').as_integer_ratio() + (-12300000, 1) + >>> Decimal('0.00').as_integer_ratio() + (0, 1) + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("cannot convert NaN to integer ratio") + else: + raise OverflowError("cannot convert Infinity to integer ratio") + + if not self: + return 0, 1 + + # Find n, d in lowest terms such that abs(self) == n / d; + # we'll deal with the sign later. + n = int(self._int) + if self._exp >= 0: + # self is an integer. + n, d = n * 10**self._exp, 1 + else: + # Find d2, d5 such that abs(self) = n / (2**d2 * 5**d5). + d5 = -self._exp + while d5 > 0 and n % 5 == 0: + n //= 5 + d5 -= 1 + + # (n & -n).bit_length() - 1 counts trailing zeros in binary + # representation of n (provided n is nonzero). + d2 = -self._exp + shift2 = min((n & -n).bit_length() - 1, d2) + if shift2: + n >>= shift2 + d2 -= shift2 + + d = 5**d5 << d2 + + if self._sign: + n = -n + return n, d + + def __repr__(self): + """Represents the number as an instance of Decimal.""" + # Invariant: eval(repr(d)) == d + return "Decimal('%s')" % str(self) + + def __str__(self, eng=False, context=None): + """Return string representation of the number in scientific notation. + + Captures all of the information in the underlying representation. + """ + + sign = ['', '-'][self._sign] + if self._is_special: + if self._exp == 'F': + return sign + 'Infinity' + elif self._exp == 'n': + return sign + 'NaN' + self._int + else: # self._exp == 'N' + return sign + 'sNaN' + self._int + + # number of digits of self._int to left of decimal point + leftdigits = self._exp + len(self._int) + + # dotplace is number of digits of self._int to the left of the + # decimal point in the mantissa of the output string (that is, + # after adjusting the exponent) + if self._exp <= 0 and leftdigits > -6: + # no exponent required + dotplace = leftdigits + elif not eng: + # usual scientific notation: 1 digit on left of the point + dotplace = 1 + elif self._int == '0': + # engineering notation, zero + dotplace = (leftdigits + 1) % 3 - 1 + else: + # engineering notation, nonzero + dotplace = (leftdigits - 1) % 3 + 1 + + if dotplace <= 0: + intpart = '0' + fracpart = '.' + '0'*(-dotplace) + self._int + elif dotplace >= len(self._int): + intpart = self._int+'0'*(dotplace-len(self._int)) + fracpart = '' + else: + intpart = self._int[:dotplace] + fracpart = '.' + self._int[dotplace:] + if leftdigits == dotplace: + exp = '' + else: + if context is None: + context = getcontext() + exp = ['e', 'E'][context.capitals] + "%+d" % (leftdigits-dotplace) + + return sign + intpart + fracpart + exp + + def to_eng_string(self, context=None): + """Convert to a string, using engineering notation if an exponent is needed. + + Engineering notation has an exponent which is a multiple of 3. This + can leave up to 3 digits to the left of the decimal place and may + require the addition of either one or two trailing zeros. + """ + return self.__str__(eng=True, context=context) + + def __neg__(self, context=None): + """Returns a copy with the sign switched. + + Rounds, if it has reason. + """ + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # -Decimal('0') is Decimal('0'), not Decimal('-0'), except + # in ROUND_FLOOR rounding mode. + ans = self.copy_abs() + else: + ans = self.copy_negate() + + return ans._fix(context) + + def __pos__(self, context=None): + """Returns a copy, unless it is a sNaN. + + Rounds the number (if more than precision digits) + """ + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # + (-0) = 0, except in ROUND_FLOOR rounding mode. + ans = self.copy_abs() + else: + ans = Decimal(self) + + return ans._fix(context) + + def __abs__(self, round=True, context=None): + """Returns the absolute value of self. + + If the keyword argument 'round' is false, do not round. The + expression self.__abs__(round=False) is equivalent to + self.copy_abs(). + """ + if not round: + return self.copy_abs() + + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if self._sign: + ans = self.__neg__(context=context) + else: + ans = self.__pos__(context=context) + + return ans + + def __add__(self, other, context=None): + """Returns self + other. + + -INF + INF (or the reverse) cause InvalidOperation errors. + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + # If both INF, same sign => same as both, opposite => error. + if self._sign != other._sign and other._isinfinity(): + return context._raise_error(InvalidOperation, '-INF + INF') + return Decimal(self) + if other._isinfinity(): + return Decimal(other) # Can't both be infinity here + + exp = min(self._exp, other._exp) + negativezero = 0 + if context.rounding == ROUND_FLOOR and self._sign != other._sign: + # If the answer is 0, the sign should be negative, in this case. + negativezero = 1 + + if not self and not other: + sign = min(self._sign, other._sign) + if negativezero: + sign = 1 + ans = _dec_from_triple(sign, '0', exp) + ans = ans._fix(context) + return ans + if not self: + exp = max(exp, other._exp - context.prec-1) + ans = other._rescale(exp, context.rounding) + ans = ans._fix(context) + return ans + if not other: + exp = max(exp, self._exp - context.prec-1) + ans = self._rescale(exp, context.rounding) + ans = ans._fix(context) + return ans + + op1 = _WorkRep(self) + op2 = _WorkRep(other) + op1, op2 = _normalize(op1, op2, context.prec) + + result = _WorkRep() + if op1.sign != op2.sign: + # Equal and opposite + if op1.int == op2.int: + ans = _dec_from_triple(negativezero, '0', exp) + ans = ans._fix(context) + return ans + if op1.int < op2.int: + op1, op2 = op2, op1 + # OK, now abs(op1) > abs(op2) + if op1.sign == 1: + result.sign = 1 + op1.sign, op2.sign = op2.sign, op1.sign + else: + result.sign = 0 + # So we know the sign, and op1 > 0. + elif op1.sign == 1: + result.sign = 1 + op1.sign, op2.sign = (0, 0) + else: + result.sign = 0 + # Now, op1 > abs(op2) > 0 + + if op2.sign == 0: + result.int = op1.int + op2.int + else: + result.int = op1.int - op2.int + + result.exp = op1.exp + ans = Decimal(result) + ans = ans._fix(context) + return ans + + __radd__ = __add__ + + def __sub__(self, other, context=None): + """Return self - other""" + other = _convert_other(other) + if other is NotImplemented: + return other + + if self._is_special or other._is_special: + ans = self._check_nans(other, context=context) + if ans: + return ans + + # self - other is computed as self + other.copy_negate() + return self.__add__(other.copy_negate(), context=context) + + def __rsub__(self, other, context=None): + """Return other - self""" + other = _convert_other(other) + if other is NotImplemented: + return other + + return other.__sub__(self, context=context) + + def __mul__(self, other, context=None): + """Return self * other. + + (+-) INF * 0 (or its reverse) raise InvalidOperation. + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + resultsign = self._sign ^ other._sign + + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + if not other: + return context._raise_error(InvalidOperation, '(+-)INF * 0') + return _SignedInfinity[resultsign] + + if other._isinfinity(): + if not self: + return context._raise_error(InvalidOperation, '0 * (+-)INF') + return _SignedInfinity[resultsign] + + resultexp = self._exp + other._exp + + # Special case for multiplying by zero + if not self or not other: + ans = _dec_from_triple(resultsign, '0', resultexp) + # Fixing in case the exponent is out of bounds + ans = ans._fix(context) + return ans + + # Special case for multiplying by power of 10 + if self._int == '1': + ans = _dec_from_triple(resultsign, other._int, resultexp) + ans = ans._fix(context) + return ans + if other._int == '1': + ans = _dec_from_triple(resultsign, self._int, resultexp) + ans = ans._fix(context) + return ans + + op1 = _WorkRep(self) + op2 = _WorkRep(other) + + ans = _dec_from_triple(resultsign, str(op1.int * op2.int), resultexp) + ans = ans._fix(context) + + return ans + __rmul__ = __mul__ + + def __truediv__(self, other, context=None): + """Return self / other.""" + other = _convert_other(other) + if other is NotImplemented: + return NotImplemented + + if context is None: + context = getcontext() + + sign = self._sign ^ other._sign + + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity() and other._isinfinity(): + return context._raise_error(InvalidOperation, '(+-)INF/(+-)INF') + + if self._isinfinity(): + return _SignedInfinity[sign] + + if other._isinfinity(): + context._raise_error(Clamped, 'Division by infinity') + return _dec_from_triple(sign, '0', context.Etiny()) + + # Special cases for zeroes + if not other: + if not self: + return context._raise_error(DivisionUndefined, '0 / 0') + return context._raise_error(DivisionByZero, 'x / 0', sign) + + if not self: + exp = self._exp - other._exp + coeff = 0 + else: + # OK, so neither = 0, INF or NaN + shift = len(other._int) - len(self._int) + context.prec + 1 + exp = self._exp - other._exp - shift + op1 = _WorkRep(self) + op2 = _WorkRep(other) + if shift >= 0: + coeff, remainder = divmod(op1.int * 10**shift, op2.int) + else: + coeff, remainder = divmod(op1.int, op2.int * 10**-shift) + if remainder: + # result is not exact; adjust to ensure correct rounding + if coeff % 5 == 0: + coeff += 1 + else: + # result is exact; get as close to ideal exponent as possible + ideal_exp = self._exp - other._exp + while exp < ideal_exp and coeff % 10 == 0: + coeff //= 10 + exp += 1 + + ans = _dec_from_triple(sign, str(coeff), exp) + return ans._fix(context) + + def _divide(self, other, context): + """Return (self // other, self % other), to context.prec precision. + + Assumes that neither self nor other is a NaN, that self is not + infinite and that other is nonzero. + """ + sign = self._sign ^ other._sign + if other._isinfinity(): + ideal_exp = self._exp + else: + ideal_exp = min(self._exp, other._exp) + + expdiff = self.adjusted() - other.adjusted() + if not self or other._isinfinity() or expdiff <= -2: + return (_dec_from_triple(sign, '0', 0), + self._rescale(ideal_exp, context.rounding)) + if expdiff <= context.prec: + op1 = _WorkRep(self) + op2 = _WorkRep(other) + if op1.exp >= op2.exp: + op1.int *= 10**(op1.exp - op2.exp) + else: + op2.int *= 10**(op2.exp - op1.exp) + q, r = divmod(op1.int, op2.int) + if q < 10**context.prec: + return (_dec_from_triple(sign, str(q), 0), + _dec_from_triple(self._sign, str(r), ideal_exp)) + + # Here the quotient is too large to be representable + ans = context._raise_error(DivisionImpossible, + 'quotient too large in //, % or divmod') + return ans, ans + + def __rtruediv__(self, other, context=None): + """Swaps self/other and returns __truediv__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__truediv__(self, context=context) + + def __divmod__(self, other, context=None): + """ + Return (self // other, self % other) + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return (ans, ans) + + sign = self._sign ^ other._sign + if self._isinfinity(): + if other._isinfinity(): + ans = context._raise_error(InvalidOperation, 'divmod(INF, INF)') + return ans, ans + else: + return (_SignedInfinity[sign], + context._raise_error(InvalidOperation, 'INF % x')) + + if not other: + if not self: + ans = context._raise_error(DivisionUndefined, 'divmod(0, 0)') + return ans, ans + else: + return (context._raise_error(DivisionByZero, 'x // 0', sign), + context._raise_error(InvalidOperation, 'x % 0')) + + quotient, remainder = self._divide(other, context) + remainder = remainder._fix(context) + return quotient, remainder + + def __rdivmod__(self, other, context=None): + """Swaps self/other and returns __divmod__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__divmod__(self, context=context) + + def __mod__(self, other, context=None): + """ + self % other + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + return context._raise_error(InvalidOperation, 'INF % x') + elif not other: + if self: + return context._raise_error(InvalidOperation, 'x % 0') + else: + return context._raise_error(DivisionUndefined, '0 % 0') + + remainder = self._divide(other, context)[1] + remainder = remainder._fix(context) + return remainder + + def __rmod__(self, other, context=None): + """Swaps self/other and returns __mod__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__mod__(self, context=context) + + def remainder_near(self, other, context=None): + """ + Remainder nearest to 0- abs(remainder-near) <= other/2 + """ + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + # self == +/-infinity -> InvalidOperation + if self._isinfinity(): + return context._raise_error(InvalidOperation, + 'remainder_near(infinity, x)') + + # other == 0 -> either InvalidOperation or DivisionUndefined + if not other: + if self: + return context._raise_error(InvalidOperation, + 'remainder_near(x, 0)') + else: + return context._raise_error(DivisionUndefined, + 'remainder_near(0, 0)') + + # other = +/-infinity -> remainder = self + if other._isinfinity(): + ans = Decimal(self) + return ans._fix(context) + + # self = 0 -> remainder = self, with ideal exponent + ideal_exponent = min(self._exp, other._exp) + if not self: + ans = _dec_from_triple(self._sign, '0', ideal_exponent) + return ans._fix(context) + + # catch most cases of large or small quotient + expdiff = self.adjusted() - other.adjusted() + if expdiff >= context.prec + 1: + # expdiff >= prec+1 => abs(self/other) > 10**prec + return context._raise_error(DivisionImpossible) + if expdiff <= -2: + # expdiff <= -2 => abs(self/other) < 0.1 + ans = self._rescale(ideal_exponent, context.rounding) + return ans._fix(context) + + # adjust both arguments to have the same exponent, then divide + op1 = _WorkRep(self) + op2 = _WorkRep(other) + if op1.exp >= op2.exp: + op1.int *= 10**(op1.exp - op2.exp) + else: + op2.int *= 10**(op2.exp - op1.exp) + q, r = divmod(op1.int, op2.int) + # remainder is r*10**ideal_exponent; other is +/-op2.int * + # 10**ideal_exponent. Apply correction to ensure that + # abs(remainder) <= abs(other)/2 + if 2*r + (q&1) > op2.int: + r -= op2.int + q += 1 + + if q >= 10**context.prec: + return context._raise_error(DivisionImpossible) + + # result has same sign as self unless r is negative + sign = self._sign + if r < 0: + sign = 1-sign + r = -r + + ans = _dec_from_triple(sign, str(r), ideal_exponent) + return ans._fix(context) + + def __floordiv__(self, other, context=None): + """self // other""" + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + if other._isinfinity(): + return context._raise_error(InvalidOperation, 'INF // INF') + else: + return _SignedInfinity[self._sign ^ other._sign] + + if not other: + if self: + return context._raise_error(DivisionByZero, 'x // 0', + self._sign ^ other._sign) + else: + return context._raise_error(DivisionUndefined, '0 // 0') + + return self._divide(other, context)[0] + + def __rfloordiv__(self, other, context=None): + """Swaps self/other and returns __floordiv__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__floordiv__(self, context=context) + + def __float__(self): + """Float representation.""" + if self._isnan(): + if self.is_snan(): + raise ValueError("Cannot convert signaling NaN to float") + s = "-nan" if self._sign else "nan" + else: + s = str(self) + return float(s) + + def __int__(self): + """Converts self to an int, truncating if necessary.""" + if self._is_special: + if self._isnan(): + raise ValueError("Cannot convert NaN to integer") + elif self._isinfinity(): + raise OverflowError("Cannot convert infinity to integer") + s = (-1)**self._sign + if self._exp >= 0: + return s*int(self._int)*10**self._exp + else: + return s*int(self._int[:self._exp] or '0') + + __trunc__ = __int__ + + @property + def real(self): + return self + + @property + def imag(self): + return Decimal(0) + + def conjugate(self): + return self + + def __complex__(self): + return complex(float(self)) + + def _fix_nan(self, context): + """Decapitate the payload of a NaN to fit the context""" + payload = self._int + + # maximum length of payload is precision if clamp=0, + # precision-1 if clamp=1. + max_payload_len = context.prec - context.clamp + if len(payload) > max_payload_len: + payload = payload[len(payload)-max_payload_len:].lstrip('0') + return _dec_from_triple(self._sign, payload, self._exp, True) + return Decimal(self) + + def _fix(self, context): + """Round if it is necessary to keep self within prec precision. + + Rounds and fixes the exponent. Does not raise on a sNaN. + + Arguments: + self - Decimal instance + context - context used. + """ + + if self._is_special: + if self._isnan(): + # decapitate payload if necessary + return self._fix_nan(context) + else: + # self is +/-Infinity; return unaltered + return Decimal(self) + + # if self is zero then exponent should be between Etiny and + # Emax if clamp==0, and between Etiny and Etop if clamp==1. + Etiny = context.Etiny() + Etop = context.Etop() + if not self: + exp_max = [context.Emax, Etop][context.clamp] + new_exp = min(max(self._exp, Etiny), exp_max) + if new_exp != self._exp: + context._raise_error(Clamped) + return _dec_from_triple(self._sign, '0', new_exp) + else: + return Decimal(self) + + # exp_min is the smallest allowable exponent of the result, + # equal to max(self.adjusted()-context.prec+1, Etiny) + exp_min = len(self._int) + self._exp - context.prec + if exp_min > Etop: + # overflow: exp_min > Etop iff self.adjusted() > Emax + ans = context._raise_error(Overflow, 'above Emax', self._sign) + context._raise_error(Inexact) + context._raise_error(Rounded) + return ans + + self_is_subnormal = exp_min < Etiny + if self_is_subnormal: + exp_min = Etiny + + # round if self has too many digits + if self._exp < exp_min: + digits = len(self._int) + self._exp - exp_min + if digits < 0: + self = _dec_from_triple(self._sign, '1', exp_min-1) + digits = 0 + rounding_method = self._pick_rounding_function[context.rounding] + changed = rounding_method(self, digits) + coeff = self._int[:digits] or '0' + if changed > 0: + coeff = str(int(coeff)+1) + if len(coeff) > context.prec: + coeff = coeff[:-1] + exp_min += 1 + + # check whether the rounding pushed the exponent out of range + if exp_min > Etop: + ans = context._raise_error(Overflow, 'above Emax', self._sign) + else: + ans = _dec_from_triple(self._sign, coeff, exp_min) + + # raise the appropriate signals, taking care to respect + # the precedence described in the specification + if changed and self_is_subnormal: + context._raise_error(Underflow) + if self_is_subnormal: + context._raise_error(Subnormal) + if changed: + context._raise_error(Inexact) + context._raise_error(Rounded) + if not ans: + # raise Clamped on underflow to 0 + context._raise_error(Clamped) + return ans + + if self_is_subnormal: + context._raise_error(Subnormal) + + # fold down if clamp == 1 and self has too few digits + if context.clamp == 1 and self._exp > Etop: + context._raise_error(Clamped) + self_padded = self._int + '0'*(self._exp - Etop) + return _dec_from_triple(self._sign, self_padded, Etop) + + # here self was representable to begin with; return unchanged + return Decimal(self) + + # for each of the rounding functions below: + # self is a finite, nonzero Decimal + # prec is an integer satisfying 0 <= prec < len(self._int) + # + # each function returns either -1, 0, or 1, as follows: + # 1 indicates that self should be rounded up (away from zero) + # 0 indicates that self should be truncated, and that all the + # digits to be truncated are zeros (so the value is unchanged) + # -1 indicates that there are nonzero digits to be truncated + + def _round_down(self, prec): + """Also known as round-towards-0, truncate.""" + if _all_zeros(self._int, prec): + return 0 + else: + return -1 + + def _round_up(self, prec): + """Rounds away from 0.""" + return -self._round_down(prec) + + def _round_half_up(self, prec): + """Rounds 5 up (away from 0)""" + if self._int[prec] in '56789': + return 1 + elif _all_zeros(self._int, prec): + return 0 + else: + return -1 + + def _round_half_down(self, prec): + """Round 5 down""" + if _exact_half(self._int, prec): + return -1 + else: + return self._round_half_up(prec) + + def _round_half_even(self, prec): + """Round 5 to even, rest to nearest.""" + if _exact_half(self._int, prec) and \ + (prec == 0 or self._int[prec-1] in '02468'): + return -1 + else: + return self._round_half_up(prec) + + def _round_ceiling(self, prec): + """Rounds up (not away from 0 if negative.)""" + if self._sign: + return self._round_down(prec) + else: + return -self._round_down(prec) + + def _round_floor(self, prec): + """Rounds down (not towards 0 if negative)""" + if not self._sign: + return self._round_down(prec) + else: + return -self._round_down(prec) + + def _round_05up(self, prec): + """Round down unless digit prec-1 is 0 or 5.""" + if prec and self._int[prec-1] not in '05': + return self._round_down(prec) + else: + return -self._round_down(prec) + + _pick_rounding_function = dict( + ROUND_DOWN = _round_down, + ROUND_UP = _round_up, + ROUND_HALF_UP = _round_half_up, + ROUND_HALF_DOWN = _round_half_down, + ROUND_HALF_EVEN = _round_half_even, + ROUND_CEILING = _round_ceiling, + ROUND_FLOOR = _round_floor, + ROUND_05UP = _round_05up, + ) + + def __round__(self, n=None): + """Round self to the nearest integer, or to a given precision. + + If only one argument is supplied, round a finite Decimal + instance self to the nearest integer. If self is infinite or + a NaN then a Python exception is raised. If self is finite + and lies exactly halfway between two integers then it is + rounded to the integer with even last digit. + + >>> round(Decimal('123.456')) + 123 + >>> round(Decimal('-456.789')) + -457 + >>> round(Decimal('-3.0')) + -3 + >>> round(Decimal('2.5')) + 2 + >>> round(Decimal('3.5')) + 4 + >>> round(Decimal('Inf')) + Traceback (most recent call last): + ... + OverflowError: cannot round an infinity + >>> round(Decimal('NaN')) + Traceback (most recent call last): + ... + ValueError: cannot round a NaN + + If a second argument n is supplied, self is rounded to n + decimal places using the rounding mode for the current + context. + + For an integer n, round(self, -n) is exactly equivalent to + self.quantize(Decimal('1En')). + + >>> round(Decimal('123.456'), 0) + Decimal('123') + >>> round(Decimal('123.456'), 2) + Decimal('123.46') + >>> round(Decimal('123.456'), -2) + Decimal('1E+2') + >>> round(Decimal('-Infinity'), 37) + Decimal('NaN') + >>> round(Decimal('sNaN123'), 0) + Decimal('NaN123') + + """ + if n is not None: + # two-argument form: use the equivalent quantize call + if not isinstance(n, int): + raise TypeError('Second argument to round should be integral') + exp = _dec_from_triple(0, '1', -n) + return self.quantize(exp) + + # one-argument form + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + else: + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_HALF_EVEN)) + + def __floor__(self): + """Return the floor of self, as an integer. + + For a finite Decimal instance self, return the greatest + integer n such that n <= self. If self is infinite or a NaN + then a Python exception is raised. + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + else: + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_FLOOR)) + + def __ceil__(self): + """Return the ceiling of self, as an integer. + + For a finite Decimal instance self, return the least integer n + such that n >= self. If self is infinite or a NaN then a + Python exception is raised. + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + else: + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_CEILING)) + + def fma(self, other, third, context=None): + """Fused multiply-add. + + Returns self*other+third with no rounding of the intermediate + product self*other. + + self and other are multiplied together, with no rounding of + the result. The third operand is then added to the result, + and a single final rounding is performed. + """ + + other = _convert_other(other, raiseit=True) + third = _convert_other(third, raiseit=True) + + # compute product; raise InvalidOperation if either operand is + # a signaling NaN or if the product is zero times infinity. + if self._is_special or other._is_special: + if context is None: + context = getcontext() + if self._exp == 'N': + return context._raise_error(InvalidOperation, 'sNaN', self) + if other._exp == 'N': + return context._raise_error(InvalidOperation, 'sNaN', other) + if self._exp == 'n': + product = self + elif other._exp == 'n': + product = other + elif self._exp == 'F': + if not other: + return context._raise_error(InvalidOperation, + 'INF * 0 in fma') + product = _SignedInfinity[self._sign ^ other._sign] + elif other._exp == 'F': + if not self: + return context._raise_error(InvalidOperation, + '0 * INF in fma') + product = _SignedInfinity[self._sign ^ other._sign] + else: + product = _dec_from_triple(self._sign ^ other._sign, + str(int(self._int) * int(other._int)), + self._exp + other._exp) + + return product.__add__(third, context) + + def _power_modulo(self, other, modulo, context=None): + """Three argument version of __pow__""" + + other = _convert_other(other) + if other is NotImplemented: + return other + modulo = _convert_other(modulo) + if modulo is NotImplemented: + return modulo + + if context is None: + context = getcontext() + + # deal with NaNs: if there are any sNaNs then first one wins, + # (i.e. behaviour for NaNs is identical to that of fma) + self_is_nan = self._isnan() + other_is_nan = other._isnan() + modulo_is_nan = modulo._isnan() + if self_is_nan or other_is_nan or modulo_is_nan: + if self_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + self) + if other_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + other) + if modulo_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + modulo) + if self_is_nan: + return self._fix_nan(context) + if other_is_nan: + return other._fix_nan(context) + return modulo._fix_nan(context) + + # check inputs: we apply same restrictions as Python's pow() + if not (self._isinteger() and + other._isinteger() and + modulo._isinteger()): + return context._raise_error(InvalidOperation, + 'pow() 3rd argument not allowed ' + 'unless all arguments are integers') + if other < 0: + return context._raise_error(InvalidOperation, + 'pow() 2nd argument cannot be ' + 'negative when 3rd argument specified') + if not modulo: + return context._raise_error(InvalidOperation, + 'pow() 3rd argument cannot be 0') + + # additional restriction for decimal: the modulus must be less + # than 10**prec in absolute value + if modulo.adjusted() >= context.prec: + return context._raise_error(InvalidOperation, + 'insufficient precision: pow() 3rd ' + 'argument must not have more than ' + 'precision digits') + + # define 0**0 == NaN, for consistency with two-argument pow + # (even though it hurts!) + if not other and not self: + return context._raise_error(InvalidOperation, + 'at least one of pow() 1st argument ' + 'and 2nd argument must be nonzero; ' + '0**0 is not defined') + + # compute sign of result + if other._iseven(): + sign = 0 + else: + sign = self._sign + + # convert modulo to a Python integer, and self and other to + # Decimal integers (i.e. force their exponents to be >= 0) + modulo = abs(int(modulo)) + base = _WorkRep(self.to_integral_value()) + exponent = _WorkRep(other.to_integral_value()) + + # compute result using integer pow() + base = (base.int % modulo * pow(10, base.exp, modulo)) % modulo + for i in range(exponent.exp): + base = pow(base, 10, modulo) + base = pow(base, exponent.int, modulo) + + return _dec_from_triple(sign, str(base), 0) + + def _power_exact(self, other, p): + """Attempt to compute self**other exactly. + + Given Decimals self and other and an integer p, attempt to + compute an exact result for the power self**other, with p + digits of precision. Return None if self**other is not + exactly representable in p digits. + + Assumes that elimination of special cases has already been + performed: self and other must both be nonspecial; self must + be positive and not numerically equal to 1; other must be + nonzero. For efficiency, other._exp should not be too large, + so that 10**abs(other._exp) is a feasible calculation.""" + + # In the comments below, we write x for the value of self and y for the + # value of other. Write x = xc*10**xe and abs(y) = yc*10**ye, with xc + # and yc positive integers not divisible by 10. + + # The main purpose of this method is to identify the *failure* + # of x**y to be exactly representable with as little effort as + # possible. So we look for cheap and easy tests that + # eliminate the possibility of x**y being exact. Only if all + # these tests are passed do we go on to actually compute x**y. + + # Here's the main idea. Express y as a rational number m/n, with m and + # n relatively prime and n>0. Then for x**y to be exactly + # representable (at *any* precision), xc must be the nth power of a + # positive integer and xe must be divisible by n. If y is negative + # then additionally xc must be a power of either 2 or 5, hence a power + # of 2**n or 5**n. + # + # There's a limit to how small |y| can be: if y=m/n as above + # then: + # + # (1) if xc != 1 then for the result to be representable we + # need xc**(1/n) >= 2, and hence also xc**|y| >= 2. So + # if |y| <= 1/nbits(xc) then xc < 2**nbits(xc) <= + # 2**(1/|y|), hence xc**|y| < 2 and the result is not + # representable. + # + # (2) if xe != 0, |xe|*(1/n) >= 1, so |xe|*|y| >= 1. Hence if + # |y| < 1/|xe| then the result is not representable. + # + # Note that since x is not equal to 1, at least one of (1) and + # (2) must apply. Now |y| < 1/nbits(xc) iff |yc|*nbits(xc) < + # 10**-ye iff len(str(|yc|*nbits(xc)) <= -ye. + # + # There's also a limit to how large y can be, at least if it's + # positive: the normalized result will have coefficient xc**y, + # so if it's representable then xc**y < 10**p, and y < + # p/log10(xc). Hence if y*log10(xc) >= p then the result is + # not exactly representable. + + # if len(str(abs(yc*xe)) <= -ye then abs(yc*xe) < 10**-ye, + # so |y| < 1/xe and the result is not representable. + # Similarly, len(str(abs(yc)*xc_bits)) <= -ye implies |y| + # < 1/nbits(xc). + + x = _WorkRep(self) + xc, xe = x.int, x.exp + while xc % 10 == 0: + xc //= 10 + xe += 1 + + y = _WorkRep(other) + yc, ye = y.int, y.exp + while yc % 10 == 0: + yc //= 10 + ye += 1 + + # case where xc == 1: result is 10**(xe*y), with xe*y + # required to be an integer + if xc == 1: + xe *= yc + # result is now 10**(xe * 10**ye); xe * 10**ye must be integral + while xe % 10 == 0: + xe //= 10 + ye += 1 + if ye < 0: + return None + exponent = xe * 10**ye + if y.sign == 1: + exponent = -exponent + # if other is a nonnegative integer, use ideal exponent + if other._isinteger() and other._sign == 0: + ideal_exponent = self._exp*int(other) + zeros = min(exponent-ideal_exponent, p-1) + else: + zeros = 0 + return _dec_from_triple(0, '1' + '0'*zeros, exponent-zeros) + + # case where y is negative: xc must be either a power + # of 2 or a power of 5. + if y.sign == 1: + last_digit = xc % 10 + if last_digit in (2,4,6,8): + # quick test for power of 2 + if xc & -xc != xc: + return None + # now xc is a power of 2; e is its exponent + e = _nbits(xc)-1 + + # We now have: + # + # x = 2**e * 10**xe, e > 0, and y < 0. + # + # The exact result is: + # + # x**y = 5**(-e*y) * 10**(e*y + xe*y) + # + # provided that both e*y and xe*y are integers. Note that if + # 5**(-e*y) >= 10**p, then the result can't be expressed + # exactly with p digits of precision. + # + # Using the above, we can guard against large values of ye. + # 93/65 is an upper bound for log(10)/log(5), so if + # + # ye >= len(str(93*p//65)) + # + # then + # + # -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5), + # + # so 5**(-e*y) >= 10**p, and the coefficient of the result + # can't be expressed in p digits. + + # emax >= largest e such that 5**e < 10**p. + emax = p*93//65 + if ye >= len(str(emax)): + return None + + # Find -e*y and -xe*y; both must be integers + e = _decimal_lshift_exact(e * yc, ye) + xe = _decimal_lshift_exact(xe * yc, ye) + if e is None or xe is None: + return None + + if e > emax: + return None + xc = 5**e + + elif last_digit == 5: + # e >= log_5(xc) if xc is a power of 5; we have + # equality all the way up to xc=5**2658 + e = _nbits(xc)*28//65 + xc, remainder = divmod(5**e, xc) + if remainder: + return None + while xc % 5 == 0: + xc //= 5 + e -= 1 + + # Guard against large values of ye, using the same logic as in + # the 'xc is a power of 2' branch. 10/3 is an upper bound for + # log(10)/log(2). + emax = p*10//3 + if ye >= len(str(emax)): + return None + + e = _decimal_lshift_exact(e * yc, ye) + xe = _decimal_lshift_exact(xe * yc, ye) + if e is None or xe is None: + return None + + if e > emax: + return None + xc = 2**e + else: + return None + + # An exact power of 10 is representable, but can convert to a + # string of any length. But an exact power of 10 shouldn't be + # possible at this point. + assert xc > 1, self + assert xc % 10 != 0, self + strxc = str(xc) + if len(strxc) > p: + return None + xe = -e-xe + return _dec_from_triple(0, strxc, xe) + + # now y is positive; find m and n such that y = m/n + if ye >= 0: + m, n = yc*10**ye, 1 + else: + if xe != 0 and len(str(abs(yc*xe))) <= -ye: + return None + xc_bits = _nbits(xc) + if len(str(abs(yc)*xc_bits)) <= -ye: + return None + m, n = yc, 10**(-ye) + while m % 2 == n % 2 == 0: + m //= 2 + n //= 2 + while m % 5 == n % 5 == 0: + m //= 5 + n //= 5 + + # compute nth root of xc*10**xe + if n > 1: + # if 1 < xc < 2**n then xc isn't an nth power + if xc_bits <= n: + return None + + xe, rem = divmod(xe, n) + if rem != 0: + return None + + # compute nth root of xc using Newton's method + a = 1 << -(-_nbits(xc)//n) # initial estimate + while True: + q, r = divmod(xc, a**(n-1)) + if a <= q: + break + else: + a = (a*(n-1) + q)//n + if not (a == q and r == 0): + return None + xc = a + + # now xc*10**xe is the nth root of the original xc*10**xe + # compute mth power of xc*10**xe + + # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m > + # 10**p and the result is not representable. + if xc > 1 and m > p*100//_log10_lb(xc): + return None + xc = xc**m + xe *= m + # An exact power of 10 is representable, but can convert to a string + # of any length. But an exact power of 10 shouldn't be possible at + # this point. + assert xc > 1, self + assert xc % 10 != 0, self + str_xc = str(xc) + if len(str_xc) > p: + return None + + # by this point the result *is* exactly representable + # adjust the exponent to get as close as possible to the ideal + # exponent, if necessary + if other._isinteger() and other._sign == 0: + ideal_exponent = self._exp*int(other) + zeros = min(xe-ideal_exponent, p-len(str_xc)) + else: + zeros = 0 + return _dec_from_triple(0, str_xc+'0'*zeros, xe-zeros) + + def __pow__(self, other, modulo=None, context=None): + """Return self ** other [ % modulo]. + + With two arguments, compute self**other. + + With three arguments, compute (self**other) % modulo. For the + three argument form, the following restrictions on the + arguments hold: + + - all three arguments must be integral + - other must be nonnegative + - either self or other (or both) must be nonzero + - modulo must be nonzero and must have at most p digits, + where p is the context precision. + + If any of these restrictions is violated the InvalidOperation + flag is raised. + + The result of pow(self, other, modulo) is identical to the + result that would be obtained by computing (self**other) % + modulo with unbounded precision, but is computed more + efficiently. It is always exact. + """ + + if modulo is not None: + return self._power_modulo(other, modulo, context) + + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + # either argument is a NaN => result is NaN + ans = self._check_nans(other, context) + if ans: + return ans + + # 0**0 = NaN (!), x**0 = 1 for nonzero x (including +/-Infinity) + if not other: + if not self: + return context._raise_error(InvalidOperation, '0 ** 0') + else: + return _One + + # result has sign 1 iff self._sign is 1 and other is an odd integer + result_sign = 0 + if self._sign == 1: + if other._isinteger(): + if not other._iseven(): + result_sign = 1 + else: + # -ve**noninteger = NaN + # (-0)**noninteger = 0**noninteger + if self: + return context._raise_error(InvalidOperation, + 'x ** y with x negative and y not an integer') + # negate self, without doing any unwanted rounding + self = self.copy_negate() + + # 0**(+ve or Inf)= 0; 0**(-ve or -Inf) = Infinity + if not self: + if other._sign == 0: + return _dec_from_triple(result_sign, '0', 0) + else: + return _SignedInfinity[result_sign] + + # Inf**(+ve or Inf) = Inf; Inf**(-ve or -Inf) = 0 + if self._isinfinity(): + if other._sign == 0: + return _SignedInfinity[result_sign] + else: + return _dec_from_triple(result_sign, '0', 0) + + # 1**other = 1, but the choice of exponent and the flags + # depend on the exponent of self, and on whether other is a + # positive integer, a negative integer, or neither + if self == _One: + if other._isinteger(): + # exp = max(self._exp*max(int(other), 0), + # 1-context.prec) but evaluating int(other) directly + # is dangerous until we know other is small (other + # could be 1e999999999) + if other._sign == 1: + multiplier = 0 + elif other > context.prec: + multiplier = context.prec + else: + multiplier = int(other) + + exp = self._exp * multiplier + if exp < 1-context.prec: + exp = 1-context.prec + context._raise_error(Rounded) + else: + context._raise_error(Inexact) + context._raise_error(Rounded) + exp = 1-context.prec + + return _dec_from_triple(result_sign, '1'+'0'*-exp, exp) + + # compute adjusted exponent of self + self_adj = self.adjusted() + + # self ** infinity is infinity if self > 1, 0 if self < 1 + # self ** -infinity is infinity if self < 1, 0 if self > 1 + if other._isinfinity(): + if (other._sign == 0) == (self_adj < 0): + return _dec_from_triple(result_sign, '0', 0) + else: + return _SignedInfinity[result_sign] + + # from here on, the result always goes through the call + # to _fix at the end of this function. + ans = None + exact = False + + # crude test to catch cases of extreme overflow/underflow. If + # log10(self)*other >= 10**bound and bound >= len(str(Emax)) + # then 10**bound >= 10**len(str(Emax)) >= Emax+1 and hence + # self**other >= 10**(Emax+1), so overflow occurs. The test + # for underflow is similar. + bound = self._log10_exp_bound() + other.adjusted() + if (self_adj >= 0) == (other._sign == 0): + # self > 1 and other +ve, or self < 1 and other -ve + # possibility of overflow + if bound >= len(str(context.Emax)): + ans = _dec_from_triple(result_sign, '1', context.Emax+1) + else: + # self > 1 and other -ve, or self < 1 and other +ve + # possibility of underflow to 0 + Etiny = context.Etiny() + if bound >= len(str(-Etiny)): + ans = _dec_from_triple(result_sign, '1', Etiny-1) + + # try for an exact result with precision +1 + if ans is None: + ans = self._power_exact(other, context.prec + 1) + if ans is not None: + if result_sign == 1: + ans = _dec_from_triple(1, ans._int, ans._exp) + exact = True + + # usual case: inexact result, x**y computed directly as exp(y*log(x)) + if ans is None: + p = context.prec + x = _WorkRep(self) + xc, xe = x.int, x.exp + y = _WorkRep(other) + yc, ye = y.int, y.exp + if y.sign == 1: + yc = -yc + + # compute correctly rounded result: start with precision +3, + # then increase precision until result is unambiguously roundable + extra = 3 + while True: + coeff, exp = _dpower(xc, xe, yc, ye, p+extra) + if coeff % (5*10**(len(str(coeff))-p-1)): + break + extra += 3 + + ans = _dec_from_triple(result_sign, str(coeff), exp) + + # unlike exp, ln and log10, the power function respects the + # rounding mode; no need to switch to ROUND_HALF_EVEN here + + # There's a difficulty here when 'other' is not an integer and + # the result is exact. In this case, the specification + # requires that the Inexact flag be raised (in spite of + # exactness), but since the result is exact _fix won't do this + # for us. (Correspondingly, the Underflow signal should also + # be raised for subnormal results.) We can't directly raise + # these signals either before or after calling _fix, since + # that would violate the precedence for signals. So we wrap + # the ._fix call in a temporary context, and reraise + # afterwards. + if exact and not other._isinteger(): + # pad with zeros up to length context.prec+1 if necessary; this + # ensures that the Rounded signal will be raised. + if len(ans._int) <= context.prec: + expdiff = context.prec + 1 - len(ans._int) + ans = _dec_from_triple(ans._sign, ans._int+'0'*expdiff, + ans._exp-expdiff) + + # create a copy of the current context, with cleared flags/traps + newcontext = context.copy() + newcontext.clear_flags() + for exception in _signals: + newcontext.traps[exception] = 0 + + # round in the new context + ans = ans._fix(newcontext) + + # raise Inexact, and if necessary, Underflow + newcontext._raise_error(Inexact) + if newcontext.flags[Subnormal]: + newcontext._raise_error(Underflow) + + # propagate signals to the original context; _fix could + # have raised any of Overflow, Underflow, Subnormal, + # Inexact, Rounded, Clamped. Overflow needs the correct + # arguments. Note that the order of the exceptions is + # important here. + if newcontext.flags[Overflow]: + context._raise_error(Overflow, 'above Emax', ans._sign) + for exception in Underflow, Subnormal, Inexact, Rounded, Clamped: + if newcontext.flags[exception]: + context._raise_error(exception) + + else: + ans = ans._fix(context) + + return ans + + def __rpow__(self, other, context=None): + """Swaps self/other and returns __pow__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__pow__(self, context=context) + + def normalize(self, context=None): + """Normalize- strip trailing 0s, change anything equal to 0 to 0e0""" + + if context is None: + context = getcontext() + + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + dup = self._fix(context) + if dup._isinfinity(): + return dup + + if not dup: + return _dec_from_triple(dup._sign, '0', 0) + exp_max = [context.Emax, context.Etop()][context.clamp] + end = len(dup._int) + exp = dup._exp + while dup._int[end-1] == '0' and exp < exp_max: + exp += 1 + end -= 1 + return _dec_from_triple(dup._sign, dup._int[:end], exp) + + def quantize(self, exp, rounding=None, context=None): + """Quantize self so its exponent is the same as that of exp. + + Similar to self._rescale(exp._exp) but with error checking. + """ + exp = _convert_other(exp, raiseit=True) + + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + + if self._is_special or exp._is_special: + ans = self._check_nans(exp, context) + if ans: + return ans + + if exp._isinfinity() or self._isinfinity(): + if exp._isinfinity() and self._isinfinity(): + return Decimal(self) # if both are inf, it is OK + return context._raise_error(InvalidOperation, + 'quantize with one INF') + + # exp._exp should be between Etiny and Emax + if not (context.Etiny() <= exp._exp <= context.Emax): + return context._raise_error(InvalidOperation, + 'target exponent out of bounds in quantize') + + if not self: + ans = _dec_from_triple(self._sign, '0', exp._exp) + return ans._fix(context) + + self_adjusted = self.adjusted() + if self_adjusted > context.Emax: + return context._raise_error(InvalidOperation, + 'exponent of quantize result too large for current context') + if self_adjusted - exp._exp + 1 > context.prec: + return context._raise_error(InvalidOperation, + 'quantize result has too many digits for current context') + + ans = self._rescale(exp._exp, rounding) + if ans.adjusted() > context.Emax: + return context._raise_error(InvalidOperation, + 'exponent of quantize result too large for current context') + if len(ans._int) > context.prec: + return context._raise_error(InvalidOperation, + 'quantize result has too many digits for current context') + + # raise appropriate flags + if ans and ans.adjusted() < context.Emin: + context._raise_error(Subnormal) + if ans._exp > self._exp: + if ans != self: + context._raise_error(Inexact) + context._raise_error(Rounded) + + # call to fix takes care of any necessary folddown, and + # signals Clamped if necessary + ans = ans._fix(context) + return ans + + def same_quantum(self, other, context=None): + """Return True if self and other have the same exponent; otherwise + return False. + + If either operand is a special value, the following rules are used: + * return True if both operands are infinities + * return True if both operands are NaNs + * otherwise, return False. + """ + other = _convert_other(other, raiseit=True) + if self._is_special or other._is_special: + return (self.is_nan() and other.is_nan() or + self.is_infinite() and other.is_infinite()) + return self._exp == other._exp + + def _rescale(self, exp, rounding): + """Rescale self so that the exponent is exp, either by padding with zeros + or by truncating digits, using the given rounding mode. + + Specials are returned without change. This operation is + quiet: it raises no flags, and uses no information from the + context. + + exp = exp to scale to (an integer) + rounding = rounding mode + """ + if self._is_special: + return Decimal(self) + if not self: + return _dec_from_triple(self._sign, '0', exp) + + if self._exp >= exp: + # pad answer with zeros if necessary + return _dec_from_triple(self._sign, + self._int + '0'*(self._exp - exp), exp) + + # too many digits; round and lose data. If self.adjusted() < + # exp-1, replace self by 10**(exp-1) before rounding + digits = len(self._int) + self._exp - exp + if digits < 0: + self = _dec_from_triple(self._sign, '1', exp-1) + digits = 0 + this_function = self._pick_rounding_function[rounding] + changed = this_function(self, digits) + coeff = self._int[:digits] or '0' + if changed == 1: + coeff = str(int(coeff)+1) + return _dec_from_triple(self._sign, coeff, exp) + + def _round(self, places, rounding): + """Round a nonzero, nonspecial Decimal to a fixed number of + significant figures, using the given rounding mode. + + Infinities, NaNs and zeros are returned unaltered. + + This operation is quiet: it raises no flags, and uses no + information from the context. + + """ + if places <= 0: + raise ValueError("argument should be at least 1 in _round") + if self._is_special or not self: + return Decimal(self) + ans = self._rescale(self.adjusted()+1-places, rounding) + # it can happen that the rescale alters the adjusted exponent; + # for example when rounding 99.97 to 3 significant figures. + # When this happens we end up with an extra 0 at the end of + # the number; a second rescale fixes this. + if ans.adjusted() != self.adjusted(): + ans = ans._rescale(ans.adjusted()+1-places, rounding) + return ans + + def to_integral_exact(self, rounding=None, context=None): + """Rounds to a nearby integer. + + If no rounding mode is specified, take the rounding mode from + the context. This method raises the Rounded and Inexact flags + when appropriate. + + See also: to_integral_value, which does exactly the same as + this method except that it doesn't raise Inexact or Rounded. + """ + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + return Decimal(self) + if self._exp >= 0: + return Decimal(self) + if not self: + return _dec_from_triple(self._sign, '0', 0) + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + ans = self._rescale(0, rounding) + if ans != self: + context._raise_error(Inexact) + context._raise_error(Rounded) + return ans + + def to_integral_value(self, rounding=None, context=None): + """Rounds to the nearest integer, without raising inexact, rounded.""" + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + return Decimal(self) + if self._exp >= 0: + return Decimal(self) + else: + return self._rescale(0, rounding) + + # the method name changed, but we provide also the old one, for compatibility + to_integral = to_integral_value + + def sqrt(self, context=None): + """Return the square root of self.""" + if context is None: + context = getcontext() + + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if self._isinfinity() and self._sign == 0: + return Decimal(self) + + if not self: + # exponent = self._exp // 2. sqrt(-0) = -0 + ans = _dec_from_triple(self._sign, '0', self._exp // 2) + return ans._fix(context) + + if self._sign == 1: + return context._raise_error(InvalidOperation, 'sqrt(-x), x > 0') + + # At this point self represents a positive number. Let p be + # the desired precision and express self in the form c*100**e + # with c a positive real number and e an integer, c and e + # being chosen so that 100**(p-1) <= c < 100**p. Then the + # (exact) square root of self is sqrt(c)*10**e, and 10**(p-1) + # <= sqrt(c) < 10**p, so the closest representable Decimal at + # precision p is n*10**e where n = round_half_even(sqrt(c)), + # the closest integer to sqrt(c) with the even integer chosen + # in the case of a tie. + # + # To ensure correct rounding in all cases, we use the + # following trick: we compute the square root to an extra + # place (precision p+1 instead of precision p), rounding down. + # Then, if the result is inexact and its last digit is 0 or 5, + # we increase the last digit to 1 or 6 respectively; if it's + # exact we leave the last digit alone. Now the final round to + # p places (or fewer in the case of underflow) will round + # correctly and raise the appropriate flags. + + # use an extra digit of precision + prec = context.prec+1 + + # write argument in the form c*100**e where e = self._exp//2 + # is the 'ideal' exponent, to be used if the square root is + # exactly representable. l is the number of 'digits' of c in + # base 100, so that 100**(l-1) <= c < 100**l. + op = _WorkRep(self) + e = op.exp >> 1 + if op.exp & 1: + c = op.int * 10 + l = (len(self._int) >> 1) + 1 + else: + c = op.int + l = len(self._int)+1 >> 1 + + # rescale so that c has exactly prec base 100 'digits' + shift = prec-l + if shift >= 0: + c *= 100**shift + exact = True + else: + c, remainder = divmod(c, 100**-shift) + exact = not remainder + e -= shift + + # find n = floor(sqrt(c)) using Newton's method + n = 10**prec + while True: + q = c//n + if n <= q: + break + else: + n = n + q >> 1 + exact = exact and n*n == c + + if exact: + # result is exact; rescale to use ideal exponent e + if shift >= 0: + # assert n % 10**shift == 0 + n //= 10**shift + else: + n *= 10**-shift + e += shift + else: + # result is not exact; fix last digit as described above + if n % 5 == 0: + n += 1 + + ans = _dec_from_triple(0, str(n), e) + + # round, and fit to current context + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + + return ans + + def max(self, other, context=None): + """Returns the larger value. + + Like max(self, other) except if one is not a number, returns + NaN (and signals if one is sNaN). Also rounds. + """ + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self._cmp(other) + if c == 0: + # If both operands are finite and equal in numerical value + # then an ordering is applied: + # + # If the signs differ then max returns the operand with the + # positive sign and min returns the operand with the negative sign + # + # If the signs are the same then the exponent is used to select + # the result. This is exactly the ordering used in compare_total. + c = self.compare_total(other) + + if c == -1: + ans = other + else: + ans = self + + return ans._fix(context) + + def min(self, other, context=None): + """Returns the smaller value. + + Like min(self, other) except if one is not a number, returns + NaN (and signals if one is sNaN). Also rounds. + """ + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self._cmp(other) + if c == 0: + c = self.compare_total(other) + + if c == -1: + ans = self + else: + ans = other + + return ans._fix(context) + + def _isinteger(self): + """Returns whether self is an integer""" + if self._is_special: + return False + if self._exp >= 0: + return True + rest = self._int[self._exp:] + return rest == '0'*len(rest) + + def _iseven(self): + """Returns True if self is even. Assumes self is an integer.""" + if not self or self._exp > 0: + return True + return self._int[-1+self._exp] in '02468' + + def adjusted(self): + """Return the adjusted exponent of self""" + try: + return self._exp + len(self._int) - 1 + # If NaN or Infinity, self._exp is string + except TypeError: + return 0 + + def canonical(self): + """Returns the same Decimal object. + + As we do not have different encodings for the same number, the + received object already is in its canonical form. + """ + return self + + def compare_signal(self, other, context=None): + """Compares self to the other operand numerically. + + It's pretty much like compare(), but all NaNs signal, with signaling + NaNs taking precedence over quiet NaNs. + """ + other = _convert_other(other, raiseit = True) + ans = self._compare_check_nans(other, context) + if ans: + return ans + return self.compare(other, context=context) + + def compare_total(self, other, context=None): + """Compares self to other using the abstract representations. + + This is not like the standard compare, which use their numerical + value. Note that a total ordering is defined for all possible abstract + representations. + """ + other = _convert_other(other, raiseit=True) + + # if one is negative and the other is positive, it's easy + if self._sign and not other._sign: + return _NegativeOne + if not self._sign and other._sign: + return _One + sign = self._sign + + # let's handle both NaN types + self_nan = self._isnan() + other_nan = other._isnan() + if self_nan or other_nan: + if self_nan == other_nan: + # compare payloads as though they're integers + self_key = len(self._int), self._int + other_key = len(other._int), other._int + if self_key < other_key: + if sign: + return _One + else: + return _NegativeOne + if self_key > other_key: + if sign: + return _NegativeOne + else: + return _One + return _Zero + + if sign: + if self_nan == 1: + return _NegativeOne + if other_nan == 1: + return _One + if self_nan == 2: + return _NegativeOne + if other_nan == 2: + return _One + else: + if self_nan == 1: + return _One + if other_nan == 1: + return _NegativeOne + if self_nan == 2: + return _One + if other_nan == 2: + return _NegativeOne + + if self < other: + return _NegativeOne + if self > other: + return _One + + if self._exp < other._exp: + if sign: + return _One + else: + return _NegativeOne + if self._exp > other._exp: + if sign: + return _NegativeOne + else: + return _One + return _Zero + + + def compare_total_mag(self, other, context=None): + """Compares self to other using abstract repr., ignoring sign. + + Like compare_total, but with operand's sign ignored and assumed to be 0. + """ + other = _convert_other(other, raiseit=True) + + s = self.copy_abs() + o = other.copy_abs() + return s.compare_total(o) + + def copy_abs(self): + """Returns a copy with the sign set to 0. """ + return _dec_from_triple(0, self._int, self._exp, self._is_special) + + def copy_negate(self): + """Returns a copy with the sign inverted.""" + if self._sign: + return _dec_from_triple(0, self._int, self._exp, self._is_special) + else: + return _dec_from_triple(1, self._int, self._exp, self._is_special) + + def copy_sign(self, other, context=None): + """Returns self with the sign of other.""" + other = _convert_other(other, raiseit=True) + return _dec_from_triple(other._sign, self._int, + self._exp, self._is_special) + + def exp(self, context=None): + """Returns e ** self.""" + + if context is None: + context = getcontext() + + # exp(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + # exp(-Infinity) = 0 + if self._isinfinity() == -1: + return _Zero + + # exp(0) = 1 + if not self: + return _One + + # exp(Infinity) = Infinity + if self._isinfinity() == 1: + return Decimal(self) + + # the result is now guaranteed to be inexact (the true + # mathematical result is transcendental). There's no need to + # raise Rounded and Inexact here---they'll always be raised as + # a result of the call to _fix. + p = context.prec + adj = self.adjusted() + + # we only need to do any computation for quite a small range + # of adjusted exponents---for example, -29 <= adj <= 10 for + # the default context. For smaller exponent the result is + # indistinguishable from 1 at the given precision, while for + # larger exponent the result either overflows or underflows. + if self._sign == 0 and adj > len(str((context.Emax+1)*3)): + # overflow + ans = _dec_from_triple(0, '1', context.Emax+1) + elif self._sign == 1 and adj > len(str((-context.Etiny()+1)*3)): + # underflow to 0 + ans = _dec_from_triple(0, '1', context.Etiny()-1) + elif self._sign == 0 and adj < -p: + # p+1 digits; final round will raise correct flags + ans = _dec_from_triple(0, '1' + '0'*(p-1) + '1', -p) + elif self._sign == 1 and adj < -p-1: + # p+1 digits; final round will raise correct flags + ans = _dec_from_triple(0, '9'*(p+1), -p-1) + # general case + else: + op = _WorkRep(self) + c, e = op.int, op.exp + if op.sign == 1: + c = -c + + # compute correctly rounded result: increase precision by + # 3 digits at a time until we get an unambiguously + # roundable result + extra = 3 + while True: + coeff, exp = _dexp(c, e, p+extra) + if coeff % (5*10**(len(str(coeff))-p-1)): + break + extra += 3 + + ans = _dec_from_triple(0, str(coeff), exp) + + # at this stage, ans should round correctly with *any* + # rounding mode, not just with ROUND_HALF_EVEN + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + + return ans + + def is_canonical(self): + """Return True if self is canonical; otherwise return False. + + Currently, the encoding of a Decimal instance is always + canonical, so this method returns True for any Decimal. + """ + return True + + def is_finite(self): + """Return True if self is finite; otherwise return False. + + A Decimal instance is considered finite if it is neither + infinite nor a NaN. + """ + return not self._is_special + + def is_infinite(self): + """Return True if self is infinite; otherwise return False.""" + return self._exp == 'F' + + def is_nan(self): + """Return True if self is a qNaN or sNaN; otherwise return False.""" + return self._exp in ('n', 'N') + + def is_normal(self, context=None): + """Return True if self is a normal number; otherwise return False.""" + if self._is_special or not self: + return False + if context is None: + context = getcontext() + return context.Emin <= self.adjusted() + + def is_qnan(self): + """Return True if self is a quiet NaN; otherwise return False.""" + return self._exp == 'n' + + def is_signed(self): + """Return True if self is negative; otherwise return False.""" + return self._sign == 1 + + def is_snan(self): + """Return True if self is a signaling NaN; otherwise return False.""" + return self._exp == 'N' + + def is_subnormal(self, context=None): + """Return True if self is subnormal; otherwise return False.""" + if self._is_special or not self: + return False + if context is None: + context = getcontext() + return self.adjusted() < context.Emin + + def is_zero(self): + """Return True if self is a zero; otherwise return False.""" + return not self._is_special and self._int == '0' + + def _ln_exp_bound(self): + """Compute a lower bound for the adjusted exponent of self.ln(). + In other words, compute r such that self.ln() >= 10**r. Assumes + that self is finite and positive and that self != 1. + """ + + # for 0.1 <= x <= 10 we use the inequalities 1-1/x <= ln(x) <= x-1 + adj = self._exp + len(self._int) - 1 + if adj >= 1: + # argument >= 10; we use 23/10 = 2.3 as a lower bound for ln(10) + return len(str(adj*23//10)) - 1 + if adj <= -2: + # argument <= 0.1 + return len(str((-1-adj)*23//10)) - 1 + op = _WorkRep(self) + c, e = op.int, op.exp + if adj == 0: + # 1 < self < 10 + num = str(c-10**-e) + den = str(c) + return len(num) - len(den) - (num < den) + # adj == -1, 0.1 <= self < 1 + return e + len(str(10**-e - c)) - 1 + + + def ln(self, context=None): + """Returns the natural (base e) logarithm of self.""" + + if context is None: + context = getcontext() + + # ln(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + # ln(0.0) == -Infinity + if not self: + return _NegativeInfinity + + # ln(Infinity) = Infinity + if self._isinfinity() == 1: + return _Infinity + + # ln(1.0) == 0.0 + if self == _One: + return _Zero + + # ln(negative) raises InvalidOperation + if self._sign == 1: + return context._raise_error(InvalidOperation, + 'ln of a negative value') + + # result is irrational, so necessarily inexact + op = _WorkRep(self) + c, e = op.int, op.exp + p = context.prec + + # correctly rounded result: repeatedly increase precision by 3 + # until we get an unambiguously roundable result + places = p - self._ln_exp_bound() + 2 # at least p+3 places + while True: + coeff = _dlog(c, e, places) + # assert len(str(abs(coeff)))-p >= 1 + if coeff % (5*10**(len(str(abs(coeff)))-p-1)): + break + places += 3 + ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) + + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + return ans + + def _log10_exp_bound(self): + """Compute a lower bound for the adjusted exponent of self.log10(). + In other words, find r such that self.log10() >= 10**r. + Assumes that self is finite and positive and that self != 1. + """ + + # For x >= 10 or x < 0.1 we only need a bound on the integer + # part of log10(self), and this comes directly from the + # exponent of x. For 0.1 <= x <= 10 we use the inequalities + # 1-1/x <= log(x) <= x-1. If x > 1 we have |log10(x)| > + # (1-1/x)/2.31 > 0. If x < 1 then |log10(x)| > (1-x)/2.31 > 0 + + adj = self._exp + len(self._int) - 1 + if adj >= 1: + # self >= 10 + return len(str(adj))-1 + if adj <= -2: + # self < 0.1 + return len(str(-1-adj))-1 + op = _WorkRep(self) + c, e = op.int, op.exp + if adj == 0: + # 1 < self < 10 + num = str(c-10**-e) + den = str(231*c) + return len(num) - len(den) - (num < den) + 2 + # adj == -1, 0.1 <= self < 1 + num = str(10**-e-c) + return len(num) + e - (num < "231") - 1 + + def log10(self, context=None): + """Returns the base 10 logarithm of self.""" + + if context is None: + context = getcontext() + + # log10(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + # log10(0.0) == -Infinity + if not self: + return _NegativeInfinity + + # log10(Infinity) = Infinity + if self._isinfinity() == 1: + return _Infinity + + # log10(negative or -Infinity) raises InvalidOperation + if self._sign == 1: + return context._raise_error(InvalidOperation, + 'log10 of a negative value') + + # log10(10**n) = n + if self._int[0] == '1' and self._int[1:] == '0'*(len(self._int) - 1): + # answer may need rounding + ans = Decimal(self._exp + len(self._int) - 1) + else: + # result is irrational, so necessarily inexact + op = _WorkRep(self) + c, e = op.int, op.exp + p = context.prec + + # correctly rounded result: repeatedly increase precision + # until result is unambiguously roundable + places = p-self._log10_exp_bound()+2 + while True: + coeff = _dlog10(c, e, places) + # assert len(str(abs(coeff)))-p >= 1 + if coeff % (5*10**(len(str(abs(coeff)))-p-1)): + break + places += 3 + ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) + + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + return ans + + def logb(self, context=None): + """ Returns the exponent of the magnitude of self's MSD. + + The result is the integer which is the exponent of the magnitude + of the most significant digit of self (as though it were truncated + to a single digit while maintaining the value of that digit and + without limiting the resulting exponent). + """ + # logb(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + if context is None: + context = getcontext() + + # logb(+/-Inf) = +Inf + if self._isinfinity(): + return _Infinity + + # logb(0) = -Inf, DivisionByZero + if not self: + return context._raise_error(DivisionByZero, 'logb(0)', 1) + + # otherwise, simply return the adjusted exponent of self, as a + # Decimal. Note that no attempt is made to fit the result + # into the current context. + ans = Decimal(self.adjusted()) + return ans._fix(context) + + def _islogical(self): + """Return True if self is a logical operand. + + For being logical, it must be a finite number with a sign of 0, + an exponent of 0, and a coefficient whose digits must all be + either 0 or 1. + """ + if self._sign != 0 or self._exp != 0: + return False + for dig in self._int: + if dig not in '01': + return False + return True + + def _fill_logical(self, context, opa, opb): + dif = context.prec - len(opa) + if dif > 0: + opa = '0'*dif + opa + elif dif < 0: + opa = opa[-context.prec:] + dif = context.prec - len(opb) + if dif > 0: + opb = '0'*dif + opb + elif dif < 0: + opb = opb[-context.prec:] + return opa, opb + + def logical_and(self, other, context=None): + """Applies an 'and' operation between self and other's digits. + + Both self and other must be logical numbers. + """ + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + if not self._islogical() or not other._islogical(): + return context._raise_error(InvalidOperation) + + # fill to context.prec + (opa, opb) = self._fill_logical(context, self._int, other._int) + + # make the operation, and clean starting zeroes + result = "".join([str(int(a)&int(b)) for a,b in zip(opa,opb)]) + return _dec_from_triple(0, result.lstrip('0') or '0', 0) + + def logical_invert(self, context=None): + """Invert all its digits. + + The self must be logical number. + """ + if context is None: + context = getcontext() + return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), + context) + + def logical_or(self, other, context=None): + """Applies an 'or' operation between self and other's digits. + + Both self and other must be logical numbers. + """ + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + if not self._islogical() or not other._islogical(): + return context._raise_error(InvalidOperation) + + # fill to context.prec + (opa, opb) = self._fill_logical(context, self._int, other._int) + + # make the operation, and clean starting zeroes + result = "".join([str(int(a)|int(b)) for a,b in zip(opa,opb)]) + return _dec_from_triple(0, result.lstrip('0') or '0', 0) + + def logical_xor(self, other, context=None): + """Applies an 'xor' operation between self and other's digits. + + Both self and other must be logical numbers. + """ + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + if not self._islogical() or not other._islogical(): + return context._raise_error(InvalidOperation) + + # fill to context.prec + (opa, opb) = self._fill_logical(context, self._int, other._int) + + # make the operation, and clean starting zeroes + result = "".join([str(int(a)^int(b)) for a,b in zip(opa,opb)]) + return _dec_from_triple(0, result.lstrip('0') or '0', 0) + + def max_mag(self, other, context=None): + """Compares the values numerically with their sign ignored.""" + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self.copy_abs()._cmp(other.copy_abs()) + if c == 0: + c = self.compare_total(other) + + if c == -1: + ans = other + else: + ans = self + + return ans._fix(context) + + def min_mag(self, other, context=None): + """Compares the values numerically with their sign ignored.""" + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self.copy_abs()._cmp(other.copy_abs()) + if c == 0: + c = self.compare_total(other) + + if c == -1: + ans = self + else: + ans = other + + return ans._fix(context) + + def next_minus(self, context=None): + """Returns the largest representable number smaller than itself.""" + if context is None: + context = getcontext() + + ans = self._check_nans(context=context) + if ans: + return ans + + if self._isinfinity() == -1: + return _NegativeInfinity + if self._isinfinity() == 1: + return _dec_from_triple(0, '9'*context.prec, context.Etop()) + + context = context.copy() + context._set_rounding(ROUND_FLOOR) + context._ignore_all_flags() + new_self = self._fix(context) + if new_self != self: + return new_self + return self.__sub__(_dec_from_triple(0, '1', context.Etiny()-1), + context) + + def next_plus(self, context=None): + """Returns the smallest representable number larger than itself.""" + if context is None: + context = getcontext() + + ans = self._check_nans(context=context) + if ans: + return ans + + if self._isinfinity() == 1: + return _Infinity + if self._isinfinity() == -1: + return _dec_from_triple(1, '9'*context.prec, context.Etop()) + + context = context.copy() + context._set_rounding(ROUND_CEILING) + context._ignore_all_flags() + new_self = self._fix(context) + if new_self != self: + return new_self + return self.__add__(_dec_from_triple(0, '1', context.Etiny()-1), + context) + + def next_toward(self, other, context=None): + """Returns the number closest to self, in the direction towards other. + + The result is the closest representable number to self + (excluding self) that is in the direction towards other, + unless both have the same value. If the two operands are + numerically equal, then the result is a copy of self with the + sign set to be the same as the sign of other. + """ + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return ans + + comparison = self._cmp(other) + if comparison == 0: + return self.copy_sign(other) + + if comparison == -1: + ans = self.next_plus(context) + else: # comparison == 1 + ans = self.next_minus(context) + + # decide which flags to raise using value of ans + if ans._isinfinity(): + context._raise_error(Overflow, + 'Infinite result from next_toward', + ans._sign) + context._raise_error(Inexact) + context._raise_error(Rounded) + elif ans.adjusted() < context.Emin: + context._raise_error(Underflow) + context._raise_error(Subnormal) + context._raise_error(Inexact) + context._raise_error(Rounded) + # if precision == 1 then we don't raise Clamped for a + # result 0E-Etiny. + if not ans: + context._raise_error(Clamped) + + return ans + + def number_class(self, context=None): + """Returns an indication of the class of self. + + The class is one of the following strings: + sNaN + NaN + -Infinity + -Normal + -Subnormal + -Zero + +Zero + +Subnormal + +Normal + +Infinity + """ + if self.is_snan(): + return "sNaN" + if self.is_qnan(): + return "NaN" + inf = self._isinfinity() + if inf == 1: + return "+Infinity" + if inf == -1: + return "-Infinity" + if self.is_zero(): + if self._sign: + return "-Zero" + else: + return "+Zero" + if context is None: + context = getcontext() + if self.is_subnormal(context=context): + if self._sign: + return "-Subnormal" + else: + return "+Subnormal" + # just a normal, regular, boring number, :) + if self._sign: + return "-Normal" + else: + return "+Normal" + + def radix(self): + """Just returns 10, as this is Decimal, :)""" + return Decimal(10) + + def rotate(self, other, context=None): + """Returns a rotated copy of self, value-of-other times.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + if other._exp != 0: + return context._raise_error(InvalidOperation) + if not (-context.prec <= int(other) <= context.prec): + return context._raise_error(InvalidOperation) + + if self._isinfinity(): + return Decimal(self) + + # get values, pad if necessary + torot = int(other) + rotdig = self._int + topad = context.prec - len(rotdig) + if topad > 0: + rotdig = '0'*topad + rotdig + elif topad < 0: + rotdig = rotdig[-topad:] + + # let's rotate! + rotated = rotdig[torot:] + rotdig[:torot] + return _dec_from_triple(self._sign, + rotated.lstrip('0') or '0', self._exp) + + def scaleb(self, other, context=None): + """Returns self operand after adding the second value to its exp.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + if other._exp != 0: + return context._raise_error(InvalidOperation) + liminf = -2 * (context.Emax + context.prec) + limsup = 2 * (context.Emax + context.prec) + if not (liminf <= int(other) <= limsup): + return context._raise_error(InvalidOperation) + + if self._isinfinity(): + return Decimal(self) + + d = _dec_from_triple(self._sign, self._int, self._exp + int(other)) + d = d._fix(context) + return d + + def shift(self, other, context=None): + """Returns a shifted copy of self, value-of-other times.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + if other._exp != 0: + return context._raise_error(InvalidOperation) + if not (-context.prec <= int(other) <= context.prec): + return context._raise_error(InvalidOperation) + + if self._isinfinity(): + return Decimal(self) + + # get values, pad if necessary + torot = int(other) + rotdig = self._int + topad = context.prec - len(rotdig) + if topad > 0: + rotdig = '0'*topad + rotdig + elif topad < 0: + rotdig = rotdig[-topad:] + + # let's shift! + if torot < 0: + shifted = rotdig[:torot] + else: + shifted = rotdig + '0'*torot + shifted = shifted[-context.prec:] + + return _dec_from_triple(self._sign, + shifted.lstrip('0') or '0', self._exp) + + # Support for pickling, copy, and deepcopy + def __reduce__(self): + return (self.__class__, (str(self),)) + + def __copy__(self): + if type(self) is Decimal: + return self # I'm immutable; therefore I am my own clone + return self.__class__(str(self)) + + def __deepcopy__(self, memo): + if type(self) is Decimal: + return self # My components are also immutable + return self.__class__(str(self)) + + # PEP 3101 support. the _localeconv keyword argument should be + # considered private: it's provided for ease of testing only. + def __format__(self, specifier, context=None, _localeconv=None): + """Format a Decimal instance according to the given specifier. + + The specifier should be a standard format specifier, with the + form described in PEP 3101. Formatting types 'e', 'E', 'f', + 'F', 'g', 'G', 'n' and '%' are supported. If the formatting + type is omitted it defaults to 'g' or 'G', depending on the + value of context.capitals. + """ + + # Note: PEP 3101 says that if the type is not present then + # there should be at least one digit after the decimal point. + # We take the liberty of ignoring this requirement for + # Decimal---it's presumably there to make sure that + # format(float, '') behaves similarly to str(float). + if context is None: + context = getcontext() + + spec = _parse_format_specifier(specifier, _localeconv=_localeconv) + + # special values don't care about the type or precision + if self._is_special: + sign = _format_sign(self._sign, spec) + body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' + return _format_align(sign, body, spec) + + # a type of None defaults to 'g' or 'G', depending on context + if spec['type'] is None: + spec['type'] = ['g', 'G'][context.capitals] + + # if type is '%', adjust exponent of self accordingly + if spec['type'] == '%': + self = _dec_from_triple(self._sign, self._int, self._exp+2) + + # round if necessary, taking rounding mode from the context + rounding = context.rounding + precision = spec['precision'] + if precision is not None: + if spec['type'] in 'eE': + self = self._round(precision+1, rounding) + elif spec['type'] in 'fF%': + self = self._rescale(-precision, rounding) + elif spec['type'] in 'gG' and len(self._int) > precision: + self = self._round(precision, rounding) + # special case: zeros with a positive exponent can't be + # represented in fixed point; rescale them to 0e0. + if not self and self._exp > 0 and spec['type'] in 'fF%': + self = self._rescale(0, rounding) + if not self and spec['no_neg_0'] and self._sign: + adjusted_sign = 0 + else: + adjusted_sign = self._sign + + # figure out placement of the decimal point + leftdigits = self._exp + len(self._int) + if spec['type'] in 'eE': + if not self and precision is not None: + dotplace = 1 - precision + else: + dotplace = 1 + elif spec['type'] in 'fF%': + dotplace = leftdigits + elif spec['type'] in 'gG': + if self._exp <= 0 and leftdigits > -6: + dotplace = leftdigits + else: + dotplace = 1 + + # find digits before and after decimal point, and get exponent + if dotplace < 0: + intpart = '0' + fracpart = '0'*(-dotplace) + self._int + elif dotplace > len(self._int): + intpart = self._int + '0'*(dotplace-len(self._int)) + fracpart = '' + else: + intpart = self._int[:dotplace] or '0' + fracpart = self._int[dotplace:] + exp = leftdigits-dotplace + + # done with the decimal-specific stuff; hand over the rest + # of the formatting to the _format_number function + return _format_number(adjusted_sign, intpart, fracpart, exp, spec) + +def _dec_from_triple(sign, coefficient, exponent, special=False): + """Create a decimal instance directly, without any validation, + normalization (e.g. removal of leading zeros) or argument + conversion. + + This function is for *internal use only*. + """ + + self = object.__new__(Decimal) + self._sign = sign + self._int = coefficient + self._exp = exponent + self._is_special = special + + return self + +# Register Decimal as a kind of Number (an abstract base class). +# However, do not register it as Real (because Decimals are not +# interoperable with floats). +_numbers.Number.register(Decimal) + + +##### Context class ####################################################### + +class _ContextManager(object): + """Context manager class to support localcontext(). + + Sets a copy of the supplied context in __enter__() and restores + the previous decimal context in __exit__() + """ + def __init__(self, new_context): + self.new_context = new_context.copy() + def __enter__(self): + self.saved_context = getcontext() + setcontext(self.new_context) + return self.new_context + def __exit__(self, t, v, tb): + setcontext(self.saved_context) + +class Context(object): + """Contains the context for a Decimal instance. + + Contains: + prec - precision (for use in rounding, division, square roots..) + rounding - rounding type (how you round) + traps - If traps[exception] = 1, then the exception is + raised when it is caused. Otherwise, a value is + substituted in. + flags - When an exception is caused, flags[exception] is set. + (Whether or not the trap_enabler is set) + Should be reset by user of Decimal instance. + Emin - Minimum exponent + Emax - Maximum exponent + capitals - If 1, 1*10^1 is printed as 1E+1. + If 0, printed as 1e1 + clamp - If 1, change exponents if too high (Default 0) + """ + + def __init__(self, prec=None, rounding=None, Emin=None, Emax=None, + capitals=None, clamp=None, flags=None, traps=None, + _ignored_flags=None): + # Set defaults; for everything except flags and _ignored_flags, + # inherit from DefaultContext. + try: + dc = DefaultContext + except NameError: + pass + + self.prec = prec if prec is not None else dc.prec + self.rounding = rounding if rounding is not None else dc.rounding + self.Emin = Emin if Emin is not None else dc.Emin + self.Emax = Emax if Emax is not None else dc.Emax + self.capitals = capitals if capitals is not None else dc.capitals + self.clamp = clamp if clamp is not None else dc.clamp + + if _ignored_flags is None: + self._ignored_flags = [] + else: + self._ignored_flags = _ignored_flags + + if traps is None: + self.traps = dc.traps.copy() + elif not isinstance(traps, dict): + self.traps = dict((s, int(s in traps)) for s in _signals + traps) + else: + self.traps = traps + + if flags is None: + self.flags = dict.fromkeys(_signals, 0) + elif not isinstance(flags, dict): + self.flags = dict((s, int(s in flags)) for s in _signals + flags) + else: + self.flags = flags + + def _set_integer_check(self, name, value, vmin, vmax): + if not isinstance(value, int): + raise TypeError("%s must be an integer" % name) + if vmin == '-inf': + if value > vmax: + raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value)) + elif vmax == 'inf': + if value < vmin: + raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value)) + else: + if value < vmin or value > vmax: + raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value)) + return object.__setattr__(self, name, value) + + def _set_signal_dict(self, name, d): + if not isinstance(d, dict): + raise TypeError("%s must be a signal dict" % d) + for key in d: + if not key in _signals: + raise KeyError("%s is not a valid signal dict" % d) + for key in _signals: + if not key in d: + raise KeyError("%s is not a valid signal dict" % d) + return object.__setattr__(self, name, d) + + def __setattr__(self, name, value): + if name == 'prec': + return self._set_integer_check(name, value, 1, 'inf') + elif name == 'Emin': + return self._set_integer_check(name, value, '-inf', 0) + elif name == 'Emax': + return self._set_integer_check(name, value, 0, 'inf') + elif name == 'capitals': + return self._set_integer_check(name, value, 0, 1) + elif name == 'clamp': + return self._set_integer_check(name, value, 0, 1) + elif name == 'rounding': + if not value in _rounding_modes: + # raise TypeError even for strings to have consistency + # among various implementations. + raise TypeError("%s: invalid rounding mode" % value) + return object.__setattr__(self, name, value) + elif name == 'flags' or name == 'traps': + return self._set_signal_dict(name, value) + elif name == '_ignored_flags': + return object.__setattr__(self, name, value) + else: + raise AttributeError( + "'decimal.Context' object has no attribute '%s'" % name) + + def __delattr__(self, name): + raise AttributeError("%s cannot be deleted" % name) + + # Support for pickling, copy, and deepcopy + def __reduce__(self): + flags = [sig for sig, v in self.flags.items() if v] + traps = [sig for sig, v in self.traps.items() if v] + return (self.__class__, + (self.prec, self.rounding, self.Emin, self.Emax, + self.capitals, self.clamp, flags, traps)) + + def __repr__(self): + """Show the current context.""" + s = [] + s.append('Context(prec=%(prec)d, rounding=%(rounding)s, ' + 'Emin=%(Emin)d, Emax=%(Emax)d, capitals=%(capitals)d, ' + 'clamp=%(clamp)d' + % vars(self)) + names = [f.__name__ for f, v in self.flags.items() if v] + s.append('flags=[' + ', '.join(names) + ']') + names = [t.__name__ for t, v in self.traps.items() if v] + s.append('traps=[' + ', '.join(names) + ']') + return ', '.join(s) + ')' + + def clear_flags(self): + """Reset all flags to zero""" + for flag in self.flags: + self.flags[flag] = 0 + + def clear_traps(self): + """Reset all traps to zero""" + for flag in self.traps: + self.traps[flag] = 0 + + def _shallow_copy(self): + """Returns a shallow copy from self.""" + nc = Context(self.prec, self.rounding, self.Emin, self.Emax, + self.capitals, self.clamp, self.flags, self.traps, + self._ignored_flags) + return nc + + def copy(self): + """Returns a deep copy from self.""" + nc = Context(self.prec, self.rounding, self.Emin, self.Emax, + self.capitals, self.clamp, + self.flags.copy(), self.traps.copy(), + self._ignored_flags) + return nc + __copy__ = copy + + def _raise_error(self, condition, explanation = None, *args): + """Handles an error + + If the flag is in _ignored_flags, returns the default response. + Otherwise, it sets the flag, then, if the corresponding + trap_enabler is set, it reraises the exception. Otherwise, it returns + the default value after setting the flag. + """ + error = _condition_map.get(condition, condition) + if error in self._ignored_flags: + # Don't touch the flag + return error().handle(self, *args) + + self.flags[error] = 1 + if not self.traps[error]: + # The errors define how to handle themselves. + return condition().handle(self, *args) + + # Errors should only be risked on copies of the context + # self._ignored_flags = [] + raise error(explanation) + + def _ignore_all_flags(self): + """Ignore all flags, if they are raised""" + return self._ignore_flags(*_signals) + + def _ignore_flags(self, *flags): + """Ignore the flags, if they are raised""" + # Do not mutate-- This way, copies of a context leave the original + # alone. + self._ignored_flags = (self._ignored_flags + list(flags)) + return list(flags) + + def _regard_flags(self, *flags): + """Stop ignoring the flags, if they are raised""" + if flags and isinstance(flags[0], (tuple,list)): + flags = flags[0] + for flag in flags: + self._ignored_flags.remove(flag) + + # We inherit object.__hash__, so we must deny this explicitly + __hash__ = None + + def Etiny(self): + """Returns Etiny (= Emin - prec + 1)""" + return int(self.Emin - self.prec + 1) + + def Etop(self): + """Returns maximum exponent (= Emax - prec + 1)""" + return int(self.Emax - self.prec + 1) + + def _set_rounding(self, type): + """Sets the rounding type. + + Sets the rounding type, and returns the current (previous) + rounding type. Often used like: + + context = context.copy() + # so you don't change the calling context + # if an error occurs in the middle. + rounding = context._set_rounding(ROUND_UP) + val = self.__sub__(other, context=context) + context._set_rounding(rounding) + + This will make it round up for that operation. + """ + rounding = self.rounding + self.rounding = type + return rounding + + def create_decimal(self, num='0'): + """Creates a new Decimal instance but using self as context. + + This method implements the to-number operation of the + IBM Decimal specification.""" + + if isinstance(num, str) and (num != num.strip() or '_' in num): + return self._raise_error(ConversionSyntax, + "trailing or leading whitespace and " + "underscores are not permitted.") + + d = Decimal(num, context=self) + if d._isnan() and len(d._int) > self.prec - self.clamp: + return self._raise_error(ConversionSyntax, + "diagnostic info too long in NaN") + return d._fix(self) + + def create_decimal_from_float(self, f): + """Creates a new Decimal instance from a float but rounding using self + as the context. + + >>> context = Context(prec=5, rounding=ROUND_DOWN) + >>> context.create_decimal_from_float(3.1415926535897932) + Decimal('3.1415') + >>> context = Context(prec=5, traps=[Inexact]) + >>> context.create_decimal_from_float(3.1415926535897932) + Traceback (most recent call last): + ... + decimal.Inexact: None + + """ + d = Decimal.from_float(f) # An exact conversion + return d._fix(self) # Apply the context rounding + + # Methods + def abs(self, a): + """Returns the absolute value of the operand. + + If the operand is negative, the result is the same as using the minus + operation on the operand. Otherwise, the result is the same as using + the plus operation on the operand. + + >>> ExtendedContext.abs(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.abs(Decimal('-100')) + Decimal('100') + >>> ExtendedContext.abs(Decimal('101.5')) + Decimal('101.5') + >>> ExtendedContext.abs(Decimal('-101.5')) + Decimal('101.5') + >>> ExtendedContext.abs(-1) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.__abs__(context=self) + + def add(self, a, b): + """Return the sum of the two operands. + + >>> ExtendedContext.add(Decimal('12'), Decimal('7.00')) + Decimal('19.00') + >>> ExtendedContext.add(Decimal('1E+2'), Decimal('1.01E+4')) + Decimal('1.02E+4') + >>> ExtendedContext.add(1, Decimal(2)) + Decimal('3') + >>> ExtendedContext.add(Decimal(8), 5) + Decimal('13') + >>> ExtendedContext.add(5, 5) + Decimal('10') + """ + a = _convert_other(a, raiseit=True) + r = a.__add__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def _apply(self, a): + return str(a._fix(self)) + + def canonical(self, a): + """Returns the same Decimal object. + + As we do not have different encodings for the same number, the + received object already is in its canonical form. + + >>> ExtendedContext.canonical(Decimal('2.50')) + Decimal('2.50') + """ + if not isinstance(a, Decimal): + raise TypeError("canonical requires a Decimal as an argument.") + return a.canonical() + + def compare(self, a, b): + """Compares values numerically. + + If the signs of the operands differ, a value representing each operand + ('-1' if the operand is less than zero, '0' if the operand is zero or + negative zero, or '1' if the operand is greater than zero) is used in + place of that operand for the comparison instead of the actual + operand. + + The comparison is then effected by subtracting the second operand from + the first and then returning a value according to the result of the + subtraction: '-1' if the result is less than zero, '0' if the result is + zero or negative zero, or '1' if the result is greater than zero. + + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) + Decimal('-1') + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.1')) + Decimal('0') + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.10')) + Decimal('0') + >>> ExtendedContext.compare(Decimal('3'), Decimal('2.1')) + Decimal('1') + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('-3')) + Decimal('1') + >>> ExtendedContext.compare(Decimal('-3'), Decimal('2.1')) + Decimal('-1') + >>> ExtendedContext.compare(1, 2) + Decimal('-1') + >>> ExtendedContext.compare(Decimal(1), 2) + Decimal('-1') + >>> ExtendedContext.compare(1, Decimal(2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.compare(b, context=self) + + def compare_signal(self, a, b): + """Compares the values of the two operands numerically. + + It's pretty much like compare(), but all NaNs signal, with signaling + NaNs taking precedence over quiet NaNs. + + >>> c = ExtendedContext + >>> c.compare_signal(Decimal('2.1'), Decimal('3')) + Decimal('-1') + >>> c.compare_signal(Decimal('2.1'), Decimal('2.1')) + Decimal('0') + >>> c.flags[InvalidOperation] = 0 + >>> print(c.flags[InvalidOperation]) + 0 + >>> c.compare_signal(Decimal('NaN'), Decimal('2.1')) + Decimal('NaN') + >>> print(c.flags[InvalidOperation]) + 1 + >>> c.flags[InvalidOperation] = 0 + >>> print(c.flags[InvalidOperation]) + 0 + >>> c.compare_signal(Decimal('sNaN'), Decimal('2.1')) + Decimal('NaN') + >>> print(c.flags[InvalidOperation]) + 1 + >>> c.compare_signal(-1, 2) + Decimal('-1') + >>> c.compare_signal(Decimal(-1), 2) + Decimal('-1') + >>> c.compare_signal(-1, Decimal(2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.compare_signal(b, context=self) + + def compare_total(self, a, b): + """Compares two operands using their abstract representation. + + This is not like the standard compare, which use their numerical + value. Note that a total ordering is defined for all possible abstract + representations. + + >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal('-127'), Decimal('12')) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.3')) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.30')) + Decimal('0') + >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('12.300')) + Decimal('1') + >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('NaN')) + Decimal('-1') + >>> ExtendedContext.compare_total(1, 2) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal(1), 2) + Decimal('-1') + >>> ExtendedContext.compare_total(1, Decimal(2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.compare_total(b) + + def compare_total_mag(self, a, b): + """Compares two operands using their abstract representation ignoring sign. + + Like compare_total, but with operand's sign ignored and assumed to be 0. + """ + a = _convert_other(a, raiseit=True) + return a.compare_total_mag(b) + + def copy_abs(self, a): + """Returns a copy of the operand with the sign set to 0. + + >>> ExtendedContext.copy_abs(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.copy_abs(Decimal('-100')) + Decimal('100') + >>> ExtendedContext.copy_abs(-1) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.copy_abs() + + def copy_decimal(self, a): + """Returns a copy of the decimal object. + + >>> ExtendedContext.copy_decimal(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.copy_decimal(Decimal('-1.00')) + Decimal('-1.00') + >>> ExtendedContext.copy_decimal(1) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return Decimal(a) + + def copy_negate(self, a): + """Returns a copy of the operand with the sign inverted. + + >>> ExtendedContext.copy_negate(Decimal('101.5')) + Decimal('-101.5') + >>> ExtendedContext.copy_negate(Decimal('-101.5')) + Decimal('101.5') + >>> ExtendedContext.copy_negate(1) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.copy_negate() + + def copy_sign(self, a, b): + """Copies the second operand's sign to the first one. + + In detail, it returns a copy of the first operand with the sign + equal to the sign of the second operand. + + >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('7.33')) + Decimal('1.50') + >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('7.33')) + Decimal('1.50') + >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('-7.33')) + Decimal('-1.50') + >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('-7.33')) + Decimal('-1.50') + >>> ExtendedContext.copy_sign(1, -2) + Decimal('-1') + >>> ExtendedContext.copy_sign(Decimal(1), -2) + Decimal('-1') + >>> ExtendedContext.copy_sign(1, Decimal(-2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.copy_sign(b) + + def divide(self, a, b): + """Decimal division in a specified context. + + >>> ExtendedContext.divide(Decimal('1'), Decimal('3')) + Decimal('0.333333333') + >>> ExtendedContext.divide(Decimal('2'), Decimal('3')) + Decimal('0.666666667') + >>> ExtendedContext.divide(Decimal('5'), Decimal('2')) + Decimal('2.5') + >>> ExtendedContext.divide(Decimal('1'), Decimal('10')) + Decimal('0.1') + >>> ExtendedContext.divide(Decimal('12'), Decimal('12')) + Decimal('1') + >>> ExtendedContext.divide(Decimal('8.00'), Decimal('2')) + Decimal('4.00') + >>> ExtendedContext.divide(Decimal('2.400'), Decimal('2.0')) + Decimal('1.20') + >>> ExtendedContext.divide(Decimal('1000'), Decimal('100')) + Decimal('10') + >>> ExtendedContext.divide(Decimal('1000'), Decimal('1')) + Decimal('1000') + >>> ExtendedContext.divide(Decimal('2.40E+6'), Decimal('2')) + Decimal('1.20E+6') + >>> ExtendedContext.divide(5, 5) + Decimal('1') + >>> ExtendedContext.divide(Decimal(5), 5) + Decimal('1') + >>> ExtendedContext.divide(5, Decimal(5)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + r = a.__truediv__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def divide_int(self, a, b): + """Divides two numbers and returns the integer part of the result. + + >>> ExtendedContext.divide_int(Decimal('2'), Decimal('3')) + Decimal('0') + >>> ExtendedContext.divide_int(Decimal('10'), Decimal('3')) + Decimal('3') + >>> ExtendedContext.divide_int(Decimal('1'), Decimal('0.3')) + Decimal('3') + >>> ExtendedContext.divide_int(10, 3) + Decimal('3') + >>> ExtendedContext.divide_int(Decimal(10), 3) + Decimal('3') + >>> ExtendedContext.divide_int(10, Decimal(3)) + Decimal('3') + """ + a = _convert_other(a, raiseit=True) + r = a.__floordiv__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def divmod(self, a, b): + """Return (a // b, a % b). + + >>> ExtendedContext.divmod(Decimal(8), Decimal(3)) + (Decimal('2'), Decimal('2')) + >>> ExtendedContext.divmod(Decimal(8), Decimal(4)) + (Decimal('2'), Decimal('0')) + >>> ExtendedContext.divmod(8, 4) + (Decimal('2'), Decimal('0')) + >>> ExtendedContext.divmod(Decimal(8), 4) + (Decimal('2'), Decimal('0')) + >>> ExtendedContext.divmod(8, Decimal(4)) + (Decimal('2'), Decimal('0')) + """ + a = _convert_other(a, raiseit=True) + r = a.__divmod__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def exp(self, a): + """Returns e ** a. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.exp(Decimal('-Infinity')) + Decimal('0') + >>> c.exp(Decimal('-1')) + Decimal('0.367879441') + >>> c.exp(Decimal('0')) + Decimal('1') + >>> c.exp(Decimal('1')) + Decimal('2.71828183') + >>> c.exp(Decimal('0.693147181')) + Decimal('2.00000000') + >>> c.exp(Decimal('+Infinity')) + Decimal('Infinity') + >>> c.exp(10) + Decimal('22026.4658') + """ + a =_convert_other(a, raiseit=True) + return a.exp(context=self) + + def fma(self, a, b, c): + """Returns a multiplied by b, plus c. + + The first two operands are multiplied together, using multiply, + the third operand is then added to the result of that + multiplication, using add, all with only one final rounding. + + >>> ExtendedContext.fma(Decimal('3'), Decimal('5'), Decimal('7')) + Decimal('22') + >>> ExtendedContext.fma(Decimal('3'), Decimal('-5'), Decimal('7')) + Decimal('-8') + >>> ExtendedContext.fma(Decimal('888565290'), Decimal('1557.96930'), Decimal('-86087.7578')) + Decimal('1.38435736E+12') + >>> ExtendedContext.fma(1, 3, 4) + Decimal('7') + >>> ExtendedContext.fma(1, Decimal(3), 4) + Decimal('7') + >>> ExtendedContext.fma(1, 3, Decimal(4)) + Decimal('7') + """ + a = _convert_other(a, raiseit=True) + return a.fma(b, c, context=self) + + def is_canonical(self, a): + """Return True if the operand is canonical; otherwise return False. + + Currently, the encoding of a Decimal instance is always + canonical, so this method returns True for any Decimal. + + >>> ExtendedContext.is_canonical(Decimal('2.50')) + True + """ + if not isinstance(a, Decimal): + raise TypeError("is_canonical requires a Decimal as an argument.") + return a.is_canonical() + + def is_finite(self, a): + """Return True if the operand is finite; otherwise return False. + + A Decimal instance is considered finite if it is neither + infinite nor a NaN. + + >>> ExtendedContext.is_finite(Decimal('2.50')) + True + >>> ExtendedContext.is_finite(Decimal('-0.3')) + True + >>> ExtendedContext.is_finite(Decimal('0')) + True + >>> ExtendedContext.is_finite(Decimal('Inf')) + False + >>> ExtendedContext.is_finite(Decimal('NaN')) + False + >>> ExtendedContext.is_finite(1) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_finite() + + def is_infinite(self, a): + """Return True if the operand is infinite; otherwise return False. + + >>> ExtendedContext.is_infinite(Decimal('2.50')) + False + >>> ExtendedContext.is_infinite(Decimal('-Inf')) + True + >>> ExtendedContext.is_infinite(Decimal('NaN')) + False + >>> ExtendedContext.is_infinite(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_infinite() + + def is_nan(self, a): + """Return True if the operand is a qNaN or sNaN; + otherwise return False. + + >>> ExtendedContext.is_nan(Decimal('2.50')) + False + >>> ExtendedContext.is_nan(Decimal('NaN')) + True + >>> ExtendedContext.is_nan(Decimal('-sNaN')) + True + >>> ExtendedContext.is_nan(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_nan() + + def is_normal(self, a): + """Return True if the operand is a normal number; + otherwise return False. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.is_normal(Decimal('2.50')) + True + >>> c.is_normal(Decimal('0.1E-999')) + False + >>> c.is_normal(Decimal('0.00')) + False + >>> c.is_normal(Decimal('-Inf')) + False + >>> c.is_normal(Decimal('NaN')) + False + >>> c.is_normal(1) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_normal(context=self) + + def is_qnan(self, a): + """Return True if the operand is a quiet NaN; otherwise return False. + + >>> ExtendedContext.is_qnan(Decimal('2.50')) + False + >>> ExtendedContext.is_qnan(Decimal('NaN')) + True + >>> ExtendedContext.is_qnan(Decimal('sNaN')) + False + >>> ExtendedContext.is_qnan(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_qnan() + + def is_signed(self, a): + """Return True if the operand is negative; otherwise return False. + + >>> ExtendedContext.is_signed(Decimal('2.50')) + False + >>> ExtendedContext.is_signed(Decimal('-12')) + True + >>> ExtendedContext.is_signed(Decimal('-0')) + True + >>> ExtendedContext.is_signed(8) + False + >>> ExtendedContext.is_signed(-8) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_signed() + + def is_snan(self, a): + """Return True if the operand is a signaling NaN; + otherwise return False. + + >>> ExtendedContext.is_snan(Decimal('2.50')) + False + >>> ExtendedContext.is_snan(Decimal('NaN')) + False + >>> ExtendedContext.is_snan(Decimal('sNaN')) + True + >>> ExtendedContext.is_snan(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_snan() + + def is_subnormal(self, a): + """Return True if the operand is subnormal; otherwise return False. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.is_subnormal(Decimal('2.50')) + False + >>> c.is_subnormal(Decimal('0.1E-999')) + True + >>> c.is_subnormal(Decimal('0.00')) + False + >>> c.is_subnormal(Decimal('-Inf')) + False + >>> c.is_subnormal(Decimal('NaN')) + False + >>> c.is_subnormal(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_subnormal(context=self) + + def is_zero(self, a): + """Return True if the operand is a zero; otherwise return False. + + >>> ExtendedContext.is_zero(Decimal('0')) + True + >>> ExtendedContext.is_zero(Decimal('2.50')) + False + >>> ExtendedContext.is_zero(Decimal('-0E+2')) + True + >>> ExtendedContext.is_zero(1) + False + >>> ExtendedContext.is_zero(0) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_zero() + + def ln(self, a): + """Returns the natural (base e) logarithm of the operand. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.ln(Decimal('0')) + Decimal('-Infinity') + >>> c.ln(Decimal('1.000')) + Decimal('0') + >>> c.ln(Decimal('2.71828183')) + Decimal('1.00000000') + >>> c.ln(Decimal('10')) + Decimal('2.30258509') + >>> c.ln(Decimal('+Infinity')) + Decimal('Infinity') + >>> c.ln(1) + Decimal('0') + """ + a = _convert_other(a, raiseit=True) + return a.ln(context=self) + + def log10(self, a): + """Returns the base 10 logarithm of the operand. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.log10(Decimal('0')) + Decimal('-Infinity') + >>> c.log10(Decimal('0.001')) + Decimal('-3') + >>> c.log10(Decimal('1.000')) + Decimal('0') + >>> c.log10(Decimal('2')) + Decimal('0.301029996') + >>> c.log10(Decimal('10')) + Decimal('1') + >>> c.log10(Decimal('70')) + Decimal('1.84509804') + >>> c.log10(Decimal('+Infinity')) + Decimal('Infinity') + >>> c.log10(0) + Decimal('-Infinity') + >>> c.log10(1) + Decimal('0') + """ + a = _convert_other(a, raiseit=True) + return a.log10(context=self) + + def logb(self, a): + """ Returns the exponent of the magnitude of the operand's MSD. + + The result is the integer which is the exponent of the magnitude + of the most significant digit of the operand (as though the + operand were truncated to a single digit while maintaining the + value of that digit and without limiting the resulting exponent). + + >>> ExtendedContext.logb(Decimal('250')) + Decimal('2') + >>> ExtendedContext.logb(Decimal('2.50')) + Decimal('0') + >>> ExtendedContext.logb(Decimal('0.03')) + Decimal('-2') + >>> ExtendedContext.logb(Decimal('0')) + Decimal('-Infinity') + >>> ExtendedContext.logb(1) + Decimal('0') + >>> ExtendedContext.logb(10) + Decimal('1') + >>> ExtendedContext.logb(100) + Decimal('2') + """ + a = _convert_other(a, raiseit=True) + return a.logb(context=self) + + def logical_and(self, a, b): + """Applies the logical operation 'and' between each operand's digits. + + The operands must be both logical numbers. + + >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010')) + Decimal('1000') + >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10')) + Decimal('10') + >>> ExtendedContext.logical_and(110, 1101) + Decimal('100') + >>> ExtendedContext.logical_and(Decimal(110), 1101) + Decimal('100') + >>> ExtendedContext.logical_and(110, Decimal(1101)) + Decimal('100') + """ + a = _convert_other(a, raiseit=True) + return a.logical_and(b, context=self) + + def logical_invert(self, a): + """Invert all the digits in the operand. + + The operand must be a logical number. + + >>> ExtendedContext.logical_invert(Decimal('0')) + Decimal('111111111') + >>> ExtendedContext.logical_invert(Decimal('1')) + Decimal('111111110') + >>> ExtendedContext.logical_invert(Decimal('111111111')) + Decimal('0') + >>> ExtendedContext.logical_invert(Decimal('101010101')) + Decimal('10101010') + >>> ExtendedContext.logical_invert(1101) + Decimal('111110010') + """ + a = _convert_other(a, raiseit=True) + return a.logical_invert(context=self) + + def logical_or(self, a, b): + """Applies the logical operation 'or' between each operand's digits. + + The operands must be both logical numbers. + + >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010')) + Decimal('1110') + >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10')) + Decimal('1110') + >>> ExtendedContext.logical_or(110, 1101) + Decimal('1111') + >>> ExtendedContext.logical_or(Decimal(110), 1101) + Decimal('1111') + >>> ExtendedContext.logical_or(110, Decimal(1101)) + Decimal('1111') + """ + a = _convert_other(a, raiseit=True) + return a.logical_or(b, context=self) + + def logical_xor(self, a, b): + """Applies the logical operation 'xor' between each operand's digits. + + The operands must be both logical numbers. + + >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0')) + Decimal('1') + >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1')) + Decimal('0') + >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010')) + Decimal('110') + >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10')) + Decimal('1101') + >>> ExtendedContext.logical_xor(110, 1101) + Decimal('1011') + >>> ExtendedContext.logical_xor(Decimal(110), 1101) + Decimal('1011') + >>> ExtendedContext.logical_xor(110, Decimal(1101)) + Decimal('1011') + """ + a = _convert_other(a, raiseit=True) + return a.logical_xor(b, context=self) + + def max(self, a, b): + """max compares two values numerically and returns the maximum. + + If either operand is a NaN then the general rules apply. + Otherwise, the operands are compared as though by the compare + operation. If they are numerically equal then the left-hand operand + is chosen as the result. Otherwise the maximum (closer to positive + infinity) of the two operands is chosen as the result. + + >>> ExtendedContext.max(Decimal('3'), Decimal('2')) + Decimal('3') + >>> ExtendedContext.max(Decimal('-10'), Decimal('3')) + Decimal('3') + >>> ExtendedContext.max(Decimal('1.0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.max(Decimal('7'), Decimal('NaN')) + Decimal('7') + >>> ExtendedContext.max(1, 2) + Decimal('2') + >>> ExtendedContext.max(Decimal(1), 2) + Decimal('2') + >>> ExtendedContext.max(1, Decimal(2)) + Decimal('2') + """ + a = _convert_other(a, raiseit=True) + return a.max(b, context=self) + + def max_mag(self, a, b): + """Compares the values numerically with their sign ignored. + + >>> ExtendedContext.max_mag(Decimal('7'), Decimal('NaN')) + Decimal('7') + >>> ExtendedContext.max_mag(Decimal('7'), Decimal('-10')) + Decimal('-10') + >>> ExtendedContext.max_mag(1, -2) + Decimal('-2') + >>> ExtendedContext.max_mag(Decimal(1), -2) + Decimal('-2') + >>> ExtendedContext.max_mag(1, Decimal(-2)) + Decimal('-2') + """ + a = _convert_other(a, raiseit=True) + return a.max_mag(b, context=self) + + def min(self, a, b): + """min compares two values numerically and returns the minimum. + + If either operand is a NaN then the general rules apply. + Otherwise, the operands are compared as though by the compare + operation. If they are numerically equal then the left-hand operand + is chosen as the result. Otherwise the minimum (closer to negative + infinity) of the two operands is chosen as the result. + + >>> ExtendedContext.min(Decimal('3'), Decimal('2')) + Decimal('2') + >>> ExtendedContext.min(Decimal('-10'), Decimal('3')) + Decimal('-10') + >>> ExtendedContext.min(Decimal('1.0'), Decimal('1')) + Decimal('1.0') + >>> ExtendedContext.min(Decimal('7'), Decimal('NaN')) + Decimal('7') + >>> ExtendedContext.min(1, 2) + Decimal('1') + >>> ExtendedContext.min(Decimal(1), 2) + Decimal('1') + >>> ExtendedContext.min(1, Decimal(29)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.min(b, context=self) + + def min_mag(self, a, b): + """Compares the values numerically with their sign ignored. + + >>> ExtendedContext.min_mag(Decimal('3'), Decimal('-2')) + Decimal('-2') + >>> ExtendedContext.min_mag(Decimal('-3'), Decimal('NaN')) + Decimal('-3') + >>> ExtendedContext.min_mag(1, -2) + Decimal('1') + >>> ExtendedContext.min_mag(Decimal(1), -2) + Decimal('1') + >>> ExtendedContext.min_mag(1, Decimal(-2)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.min_mag(b, context=self) + + def minus(self, a): + """Minus corresponds to unary prefix minus in Python. + + The operation is evaluated using the same rules as subtract; the + operation minus(a) is calculated as subtract('0', a) where the '0' + has the same exponent as the operand. + + >>> ExtendedContext.minus(Decimal('1.3')) + Decimal('-1.3') + >>> ExtendedContext.minus(Decimal('-1.3')) + Decimal('1.3') + >>> ExtendedContext.minus(1) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.__neg__(context=self) + + def multiply(self, a, b): + """multiply multiplies two operands. + + If either operand is a special value then the general rules apply. + Otherwise, the operands are multiplied together + ('long multiplication'), resulting in a number which may be as long as + the sum of the lengths of the two operands. + + >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) + Decimal('3.60') + >>> ExtendedContext.multiply(Decimal('7'), Decimal('3')) + Decimal('21') + >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('0.8')) + Decimal('0.72') + >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('-0')) + Decimal('-0.0') + >>> ExtendedContext.multiply(Decimal('654321'), Decimal('654321')) + Decimal('4.28135971E+11') + >>> ExtendedContext.multiply(7, 7) + Decimal('49') + >>> ExtendedContext.multiply(Decimal(7), 7) + Decimal('49') + >>> ExtendedContext.multiply(7, Decimal(7)) + Decimal('49') + """ + a = _convert_other(a, raiseit=True) + r = a.__mul__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def next_minus(self, a): + """Returns the largest representable number smaller than a. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> ExtendedContext.next_minus(Decimal('1')) + Decimal('0.999999999') + >>> c.next_minus(Decimal('1E-1007')) + Decimal('0E-1007') + >>> ExtendedContext.next_minus(Decimal('-1.00000003')) + Decimal('-1.00000004') + >>> c.next_minus(Decimal('Infinity')) + Decimal('9.99999999E+999') + >>> c.next_minus(1) + Decimal('0.999999999') + """ + a = _convert_other(a, raiseit=True) + return a.next_minus(context=self) + + def next_plus(self, a): + """Returns the smallest representable number larger than a. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> ExtendedContext.next_plus(Decimal('1')) + Decimal('1.00000001') + >>> c.next_plus(Decimal('-1E-1007')) + Decimal('-0E-1007') + >>> ExtendedContext.next_plus(Decimal('-1.00000003')) + Decimal('-1.00000002') + >>> c.next_plus(Decimal('-Infinity')) + Decimal('-9.99999999E+999') + >>> c.next_plus(1) + Decimal('1.00000001') + """ + a = _convert_other(a, raiseit=True) + return a.next_plus(context=self) + + def next_toward(self, a, b): + """Returns the number closest to a, in direction towards b. + + The result is the closest representable number from the first + operand (but not the first operand) that is in the direction + towards the second operand, unless the operands have the same + value. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.next_toward(Decimal('1'), Decimal('2')) + Decimal('1.00000001') + >>> c.next_toward(Decimal('-1E-1007'), Decimal('1')) + Decimal('-0E-1007') + >>> c.next_toward(Decimal('-1.00000003'), Decimal('0')) + Decimal('-1.00000002') + >>> c.next_toward(Decimal('1'), Decimal('0')) + Decimal('0.999999999') + >>> c.next_toward(Decimal('1E-1007'), Decimal('-100')) + Decimal('0E-1007') + >>> c.next_toward(Decimal('-1.00000003'), Decimal('-10')) + Decimal('-1.00000004') + >>> c.next_toward(Decimal('0.00'), Decimal('-0.0000')) + Decimal('-0.00') + >>> c.next_toward(0, 1) + Decimal('1E-1007') + >>> c.next_toward(Decimal(0), 1) + Decimal('1E-1007') + >>> c.next_toward(0, Decimal(1)) + Decimal('1E-1007') + """ + a = _convert_other(a, raiseit=True) + return a.next_toward(b, context=self) + + def normalize(self, a): + """normalize reduces an operand to its simplest form. + + Essentially a plus operation with all trailing zeros removed from the + result. + + >>> ExtendedContext.normalize(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.normalize(Decimal('-2.0')) + Decimal('-2') + >>> ExtendedContext.normalize(Decimal('1.200')) + Decimal('1.2') + >>> ExtendedContext.normalize(Decimal('-120')) + Decimal('-1.2E+2') + >>> ExtendedContext.normalize(Decimal('120.00')) + Decimal('1.2E+2') + >>> ExtendedContext.normalize(Decimal('0.00')) + Decimal('0') + >>> ExtendedContext.normalize(6) + Decimal('6') + """ + a = _convert_other(a, raiseit=True) + return a.normalize(context=self) + + def number_class(self, a): + """Returns an indication of the class of the operand. + + The class is one of the following strings: + -sNaN + -NaN + -Infinity + -Normal + -Subnormal + -Zero + +Zero + +Subnormal + +Normal + +Infinity + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.number_class(Decimal('Infinity')) + '+Infinity' + >>> c.number_class(Decimal('1E-10')) + '+Normal' + >>> c.number_class(Decimal('2.50')) + '+Normal' + >>> c.number_class(Decimal('0.1E-999')) + '+Subnormal' + >>> c.number_class(Decimal('0')) + '+Zero' + >>> c.number_class(Decimal('-0')) + '-Zero' + >>> c.number_class(Decimal('-0.1E-999')) + '-Subnormal' + >>> c.number_class(Decimal('-1E-10')) + '-Normal' + >>> c.number_class(Decimal('-2.50')) + '-Normal' + >>> c.number_class(Decimal('-Infinity')) + '-Infinity' + >>> c.number_class(Decimal('NaN')) + 'NaN' + >>> c.number_class(Decimal('-NaN')) + 'NaN' + >>> c.number_class(Decimal('sNaN')) + 'sNaN' + >>> c.number_class(123) + '+Normal' + """ + a = _convert_other(a, raiseit=True) + return a.number_class(context=self) + + def plus(self, a): + """Plus corresponds to unary prefix plus in Python. + + The operation is evaluated using the same rules as add; the + operation plus(a) is calculated as add('0', a) where the '0' + has the same exponent as the operand. + + >>> ExtendedContext.plus(Decimal('1.3')) + Decimal('1.3') + >>> ExtendedContext.plus(Decimal('-1.3')) + Decimal('-1.3') + >>> ExtendedContext.plus(-1) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.__pos__(context=self) + + def power(self, a, b, modulo=None): + """Raises a to the power of b, to modulo if given. + + With two arguments, compute a**b. If a is negative then b + must be integral. The result will be inexact unless b is + integral and the result is finite and can be expressed exactly + in 'precision' digits. + + With three arguments, compute (a**b) % modulo. For the + three argument form, the following restrictions on the + arguments hold: + + - all three arguments must be integral + - b must be nonnegative + - at least one of a or b must be nonzero + - modulo must be nonzero and have at most 'precision' digits + + The result of pow(a, b, modulo) is identical to the result + that would be obtained by computing (a**b) % modulo with + unbounded precision, but is computed more efficiently. It is + always exact. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.power(Decimal('2'), Decimal('3')) + Decimal('8') + >>> c.power(Decimal('-2'), Decimal('3')) + Decimal('-8') + >>> c.power(Decimal('2'), Decimal('-3')) + Decimal('0.125') + >>> c.power(Decimal('1.7'), Decimal('8')) + Decimal('69.7575744') + >>> c.power(Decimal('10'), Decimal('0.301029996')) + Decimal('2.00000000') + >>> c.power(Decimal('Infinity'), Decimal('-1')) + Decimal('0') + >>> c.power(Decimal('Infinity'), Decimal('0')) + Decimal('1') + >>> c.power(Decimal('Infinity'), Decimal('1')) + Decimal('Infinity') + >>> c.power(Decimal('-Infinity'), Decimal('-1')) + Decimal('-0') + >>> c.power(Decimal('-Infinity'), Decimal('0')) + Decimal('1') + >>> c.power(Decimal('-Infinity'), Decimal('1')) + Decimal('-Infinity') + >>> c.power(Decimal('-Infinity'), Decimal('2')) + Decimal('Infinity') + >>> c.power(Decimal('0'), Decimal('0')) + Decimal('NaN') + + >>> c.power(Decimal('3'), Decimal('7'), Decimal('16')) + Decimal('11') + >>> c.power(Decimal('-3'), Decimal('7'), Decimal('16')) + Decimal('-11') + >>> c.power(Decimal('-3'), Decimal('8'), Decimal('16')) + Decimal('1') + >>> c.power(Decimal('3'), Decimal('7'), Decimal('-16')) + Decimal('11') + >>> c.power(Decimal('23E12345'), Decimal('67E189'), Decimal('123456789')) + Decimal('11729830') + >>> c.power(Decimal('-0'), Decimal('17'), Decimal('1729')) + Decimal('-0') + >>> c.power(Decimal('-23'), Decimal('0'), Decimal('65537')) + Decimal('1') + >>> ExtendedContext.power(7, 7) + Decimal('823543') + >>> ExtendedContext.power(Decimal(7), 7) + Decimal('823543') + >>> ExtendedContext.power(7, Decimal(7), 2) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + r = a.__pow__(b, modulo, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def quantize(self, a, b): + """Returns a value equal to 'a' (rounded), having the exponent of 'b'. + + The coefficient of the result is derived from that of the left-hand + operand. It may be rounded using the current rounding setting (if the + exponent is being increased), multiplied by a positive power of ten (if + the exponent is being decreased), or is unchanged (if the exponent is + already equal to that of the right-hand operand). + + Unlike other operations, if the length of the coefficient after the + quantize operation would be greater than precision then an Invalid + operation condition is raised. This guarantees that, unless there is + an error condition, the exponent of the result of a quantize is always + equal to that of the right-hand operand. + + Also unlike other operations, quantize will never raise Underflow, even + if the result is subnormal and inexact. + + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) + Decimal('2.170') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.01')) + Decimal('2.17') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.1')) + Decimal('2.2') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+0')) + Decimal('2') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+1')) + Decimal('0E+1') + >>> ExtendedContext.quantize(Decimal('-Inf'), Decimal('Infinity')) + Decimal('-Infinity') + >>> ExtendedContext.quantize(Decimal('2'), Decimal('Infinity')) + Decimal('NaN') + >>> ExtendedContext.quantize(Decimal('-0.1'), Decimal('1')) + Decimal('-0') + >>> ExtendedContext.quantize(Decimal('-0'), Decimal('1e+5')) + Decimal('-0E+5') + >>> ExtendedContext.quantize(Decimal('+35236450.6'), Decimal('1e-2')) + Decimal('NaN') + >>> ExtendedContext.quantize(Decimal('-35236450.6'), Decimal('1e-2')) + Decimal('NaN') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-1')) + Decimal('217.0') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-0')) + Decimal('217') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+1')) + Decimal('2.2E+2') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+2')) + Decimal('2E+2') + >>> ExtendedContext.quantize(1, 2) + Decimal('1') + >>> ExtendedContext.quantize(Decimal(1), 2) + Decimal('1') + >>> ExtendedContext.quantize(1, Decimal(2)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.quantize(b, context=self) + + def radix(self): + """Just returns 10, as this is Decimal, :) + + >>> ExtendedContext.radix() + Decimal('10') + """ + return Decimal(10) + + def remainder(self, a, b): + """Returns the remainder from integer division. + + The result is the residue of the dividend after the operation of + calculating integer division as described for divide-integer, rounded + to precision digits if necessary. The sign of the result, if + non-zero, is the same as that of the original dividend. + + This operation will fail under the same conditions as integer division + (that is, if integer division on the same two operands would fail, the + remainder cannot be calculated). + + >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) + Decimal('2.1') + >>> ExtendedContext.remainder(Decimal('10'), Decimal('3')) + Decimal('1') + >>> ExtendedContext.remainder(Decimal('-10'), Decimal('3')) + Decimal('-1') + >>> ExtendedContext.remainder(Decimal('10.2'), Decimal('1')) + Decimal('0.2') + >>> ExtendedContext.remainder(Decimal('10'), Decimal('0.3')) + Decimal('0.1') + >>> ExtendedContext.remainder(Decimal('3.6'), Decimal('1.3')) + Decimal('1.0') + >>> ExtendedContext.remainder(22, 6) + Decimal('4') + >>> ExtendedContext.remainder(Decimal(22), 6) + Decimal('4') + >>> ExtendedContext.remainder(22, Decimal(6)) + Decimal('4') + """ + a = _convert_other(a, raiseit=True) + r = a.__mod__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def remainder_near(self, a, b): + """Returns to be "a - b * n", where n is the integer nearest the exact + value of "x / b" (if two integers are equally near then the even one + is chosen). If the result is equal to 0 then its sign will be the + sign of a. + + This operation will fail under the same conditions as integer division + (that is, if integer division on the same two operands would fail, the + remainder cannot be calculated). + + >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) + Decimal('-0.9') + >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('6')) + Decimal('-2') + >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('3')) + Decimal('1') + >>> ExtendedContext.remainder_near(Decimal('-10'), Decimal('3')) + Decimal('-1') + >>> ExtendedContext.remainder_near(Decimal('10.2'), Decimal('1')) + Decimal('0.2') + >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('0.3')) + Decimal('0.1') + >>> ExtendedContext.remainder_near(Decimal('3.6'), Decimal('1.3')) + Decimal('-0.3') + >>> ExtendedContext.remainder_near(3, 11) + Decimal('3') + >>> ExtendedContext.remainder_near(Decimal(3), 11) + Decimal('3') + >>> ExtendedContext.remainder_near(3, Decimal(11)) + Decimal('3') + """ + a = _convert_other(a, raiseit=True) + return a.remainder_near(b, context=self) + + def rotate(self, a, b): + """Returns a rotated copy of a, b times. + + The coefficient of the result is a rotated copy of the digits in + the coefficient of the first operand. The number of places of + rotation is taken from the absolute value of the second operand, + with the rotation being to the left if the second operand is + positive or to the right otherwise. + + >>> ExtendedContext.rotate(Decimal('34'), Decimal('8')) + Decimal('400000003') + >>> ExtendedContext.rotate(Decimal('12'), Decimal('9')) + Decimal('12') + >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('-2')) + Decimal('891234567') + >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('0')) + Decimal('123456789') + >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('+2')) + Decimal('345678912') + >>> ExtendedContext.rotate(1333333, 1) + Decimal('13333330') + >>> ExtendedContext.rotate(Decimal(1333333), 1) + Decimal('13333330') + >>> ExtendedContext.rotate(1333333, Decimal(1)) + Decimal('13333330') + """ + a = _convert_other(a, raiseit=True) + return a.rotate(b, context=self) + + def same_quantum(self, a, b): + """Returns True if the two operands have the same exponent. + + The result is never affected by either the sign or the coefficient of + either operand. + + >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) + False + >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.01')) + True + >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('1')) + False + >>> ExtendedContext.same_quantum(Decimal('Inf'), Decimal('-Inf')) + True + >>> ExtendedContext.same_quantum(10000, -1) + True + >>> ExtendedContext.same_quantum(Decimal(10000), -1) + True + >>> ExtendedContext.same_quantum(10000, Decimal(-1)) + True + """ + a = _convert_other(a, raiseit=True) + return a.same_quantum(b) + + def scaleb (self, a, b): + """Returns the first operand after adding the second value its exp. + + >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('-2')) + Decimal('0.0750') + >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('0')) + Decimal('7.50') + >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('3')) + Decimal('7.50E+3') + >>> ExtendedContext.scaleb(1, 4) + Decimal('1E+4') + >>> ExtendedContext.scaleb(Decimal(1), 4) + Decimal('1E+4') + >>> ExtendedContext.scaleb(1, Decimal(4)) + Decimal('1E+4') + """ + a = _convert_other(a, raiseit=True) + return a.scaleb(b, context=self) + + def shift(self, a, b): + """Returns a shifted copy of a, b times. + + The coefficient of the result is a shifted copy of the digits + in the coefficient of the first operand. The number of places + to shift is taken from the absolute value of the second operand, + with the shift being to the left if the second operand is + positive or to the right otherwise. Digits shifted into the + coefficient are zeros. + + >>> ExtendedContext.shift(Decimal('34'), Decimal('8')) + Decimal('400000000') + >>> ExtendedContext.shift(Decimal('12'), Decimal('9')) + Decimal('0') + >>> ExtendedContext.shift(Decimal('123456789'), Decimal('-2')) + Decimal('1234567') + >>> ExtendedContext.shift(Decimal('123456789'), Decimal('0')) + Decimal('123456789') + >>> ExtendedContext.shift(Decimal('123456789'), Decimal('+2')) + Decimal('345678900') + >>> ExtendedContext.shift(88888888, 2) + Decimal('888888800') + >>> ExtendedContext.shift(Decimal(88888888), 2) + Decimal('888888800') + >>> ExtendedContext.shift(88888888, Decimal(2)) + Decimal('888888800') + """ + a = _convert_other(a, raiseit=True) + return a.shift(b, context=self) + + def sqrt(self, a): + """Square root of a non-negative number to context precision. + + If the result must be inexact, it is rounded using the round-half-even + algorithm. + + >>> ExtendedContext.sqrt(Decimal('0')) + Decimal('0') + >>> ExtendedContext.sqrt(Decimal('-0')) + Decimal('-0') + >>> ExtendedContext.sqrt(Decimal('0.39')) + Decimal('0.624499800') + >>> ExtendedContext.sqrt(Decimal('100')) + Decimal('10') + >>> ExtendedContext.sqrt(Decimal('1')) + Decimal('1') + >>> ExtendedContext.sqrt(Decimal('1.0')) + Decimal('1.0') + >>> ExtendedContext.sqrt(Decimal('1.00')) + Decimal('1.0') + >>> ExtendedContext.sqrt(Decimal('7')) + Decimal('2.64575131') + >>> ExtendedContext.sqrt(Decimal('10')) + Decimal('3.16227766') + >>> ExtendedContext.sqrt(2) + Decimal('1.41421356') + >>> ExtendedContext.prec + 9 + """ + a = _convert_other(a, raiseit=True) + return a.sqrt(context=self) + + def subtract(self, a, b): + """Return the difference between the two operands. + + >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.07')) + Decimal('0.23') + >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.30')) + Decimal('0.00') + >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('2.07')) + Decimal('-0.77') + >>> ExtendedContext.subtract(8, 5) + Decimal('3') + >>> ExtendedContext.subtract(Decimal(8), 5) + Decimal('3') + >>> ExtendedContext.subtract(8, Decimal(5)) + Decimal('3') + """ + a = _convert_other(a, raiseit=True) + r = a.__sub__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def to_eng_string(self, a): + """Convert to a string, using engineering notation if an exponent is needed. + + Engineering notation has an exponent which is a multiple of 3. This + can leave up to 3 digits to the left of the decimal place and may + require the addition of either one or two trailing zeros. + + The operation is not affected by the context. + + >>> ExtendedContext.to_eng_string(Decimal('123E+1')) + '1.23E+3' + >>> ExtendedContext.to_eng_string(Decimal('123E+3')) + '123E+3' + >>> ExtendedContext.to_eng_string(Decimal('123E-10')) + '12.3E-9' + >>> ExtendedContext.to_eng_string(Decimal('-123E-12')) + '-123E-12' + >>> ExtendedContext.to_eng_string(Decimal('7E-7')) + '700E-9' + >>> ExtendedContext.to_eng_string(Decimal('7E+1')) + '70' + >>> ExtendedContext.to_eng_string(Decimal('0E+1')) + '0.00E+3' + + """ + a = _convert_other(a, raiseit=True) + return a.to_eng_string(context=self) + + def to_sci_string(self, a): + """Converts a number to a string, using scientific notation. + + The operation is not affected by the context. + """ + a = _convert_other(a, raiseit=True) + return a.__str__(context=self) + + def to_integral_exact(self, a): + """Rounds to an integer. + + When the operand has a negative exponent, the result is the same + as using the quantize() operation using the given operand as the + left-hand-operand, 1E+0 as the right-hand-operand, and the precision + of the operand as the precision setting; Inexact and Rounded flags + are allowed in this operation. The rounding mode is taken from the + context. + + >>> ExtendedContext.to_integral_exact(Decimal('2.1')) + Decimal('2') + >>> ExtendedContext.to_integral_exact(Decimal('100')) + Decimal('100') + >>> ExtendedContext.to_integral_exact(Decimal('100.0')) + Decimal('100') + >>> ExtendedContext.to_integral_exact(Decimal('101.5')) + Decimal('102') + >>> ExtendedContext.to_integral_exact(Decimal('-101.5')) + Decimal('-102') + >>> ExtendedContext.to_integral_exact(Decimal('10E+5')) + Decimal('1.0E+6') + >>> ExtendedContext.to_integral_exact(Decimal('7.89E+77')) + Decimal('7.89E+77') + >>> ExtendedContext.to_integral_exact(Decimal('-Inf')) + Decimal('-Infinity') + """ + a = _convert_other(a, raiseit=True) + return a.to_integral_exact(context=self) + + def to_integral_value(self, a): + """Rounds to an integer. + + When the operand has a negative exponent, the result is the same + as using the quantize() operation using the given operand as the + left-hand-operand, 1E+0 as the right-hand-operand, and the precision + of the operand as the precision setting, except that no flags will + be set. The rounding mode is taken from the context. + + >>> ExtendedContext.to_integral_value(Decimal('2.1')) + Decimal('2') + >>> ExtendedContext.to_integral_value(Decimal('100')) + Decimal('100') + >>> ExtendedContext.to_integral_value(Decimal('100.0')) + Decimal('100') + >>> ExtendedContext.to_integral_value(Decimal('101.5')) + Decimal('102') + >>> ExtendedContext.to_integral_value(Decimal('-101.5')) + Decimal('-102') + >>> ExtendedContext.to_integral_value(Decimal('10E+5')) + Decimal('1.0E+6') + >>> ExtendedContext.to_integral_value(Decimal('7.89E+77')) + Decimal('7.89E+77') + >>> ExtendedContext.to_integral_value(Decimal('-Inf')) + Decimal('-Infinity') + """ + a = _convert_other(a, raiseit=True) + return a.to_integral_value(context=self) + + # the method name changed, but we provide also the old one, for compatibility + to_integral = to_integral_value + +class _WorkRep(object): + __slots__ = ('sign','int','exp') + # sign: 0 or 1 + # int: int + # exp: None, int, or string + + def __init__(self, value=None): + if value is None: + self.sign = None + self.int = 0 + self.exp = None + elif isinstance(value, Decimal): + self.sign = value._sign + self.int = int(value._int) + self.exp = value._exp + else: + # assert isinstance(value, tuple) + self.sign = value[0] + self.int = value[1] + self.exp = value[2] + + def __repr__(self): + return "(%r, %r, %r)" % (self.sign, self.int, self.exp) + + + +def _normalize(op1, op2, prec = 0): + """Normalizes op1, op2 to have the same exp and length of coefficient. + + Done during addition. + """ + if op1.exp < op2.exp: + tmp = op2 + other = op1 + else: + tmp = op1 + other = op2 + + # Let exp = min(tmp.exp - 1, tmp.adjusted() - precision - 1). + # Then adding 10**exp to tmp has the same effect (after rounding) + # as adding any positive quantity smaller than 10**exp; similarly + # for subtraction. So if other is smaller than 10**exp we replace + # it with 10**exp. This avoids tmp.exp - other.exp getting too large. + tmp_len = len(str(tmp.int)) + other_len = len(str(other.int)) + exp = tmp.exp + min(-1, tmp_len - prec - 2) + if other_len + other.exp - 1 < exp: + other.int = 1 + other.exp = exp + + tmp.int *= 10 ** (tmp.exp - other.exp) + tmp.exp = other.exp + return op1, op2 + +##### Integer arithmetic functions used by ln, log10, exp and __pow__ ##### + +_nbits = int.bit_length + +def _decimal_lshift_exact(n, e): + """ Given integers n and e, return n * 10**e if it's an integer, else None. + + The computation is designed to avoid computing large powers of 10 + unnecessarily. + + >>> _decimal_lshift_exact(3, 4) + 30000 + >>> _decimal_lshift_exact(300, -999999999) # returns None + + """ + if n == 0: + return 0 + elif e >= 0: + return n * 10**e + else: + # val_n = largest power of 10 dividing n. + str_n = str(abs(n)) + val_n = len(str_n) - len(str_n.rstrip('0')) + return None if val_n < -e else n // 10**-e + +def _sqrt_nearest(n, a): + """Closest integer to the square root of the positive integer n. a is + an initial approximation to the square root. Any positive integer + will do for a, but the closer a is to the square root of n the + faster convergence will be. + + """ + if n <= 0 or a <= 0: + raise ValueError("Both arguments to _sqrt_nearest should be positive.") + + b=0 + while a != b: + b, a = a, a--n//a>>1 + return a + +def _rshift_nearest(x, shift): + """Given an integer x and a nonnegative integer shift, return closest + integer to x / 2**shift; use round-to-even in case of a tie. + + """ + b, q = 1 << shift, x >> shift + return q + (2*(x & (b-1)) + (q&1) > b) + +def _div_nearest(a, b): + """Closest integer to a/b, a and b positive integers; rounds to even + in the case of a tie. + + """ + q, r = divmod(a, b) + return q + (2*r + (q&1) > b) + +def _ilog(x, M, L = 8): + """Integer approximation to M*log(x/M), with absolute error boundable + in terms only of x/M. + + Given positive integers x and M, return an integer approximation to + M * log(x/M). For L = 8 and 0.1 <= x/M <= 10 the difference + between the approximation and the exact result is at most 22. For + L = 8 and 1.0 <= x/M <= 10.0 the difference is at most 15. In + both cases these are upper bounds on the error; it will usually be + much smaller.""" + + # The basic algorithm is the following: let log1p be the function + # log1p(x) = log(1+x). Then log(x/M) = log1p((x-M)/M). We use + # the reduction + # + # log1p(y) = 2*log1p(y/(1+sqrt(1+y))) + # + # repeatedly until the argument to log1p is small (< 2**-L in + # absolute value). For small y we can use the Taylor series + # expansion + # + # log1p(y) ~ y - y**2/2 + y**3/3 - ... - (-y)**T/T + # + # truncating at T such that y**T is small enough. The whole + # computation is carried out in a form of fixed-point arithmetic, + # with a real number z being represented by an integer + # approximation to z*M. To avoid loss of precision, the y below + # is actually an integer approximation to 2**R*y*M, where R is the + # number of reductions performed so far. + + y = x-M + # argument reduction; R = number of reductions performed + R = 0 + while (R <= L and abs(y) << L-R >= M or + R > L and abs(y) >> R-L >= M): + y = _div_nearest((M*y) << 1, + M + _sqrt_nearest(M*(M+_rshift_nearest(y, R)), M)) + R += 1 + + # Taylor series with T terms + T = -int(-10*len(str(M))//(3*L)) + yshift = _rshift_nearest(y, R) + w = _div_nearest(M, T) + for k in range(T-1, 0, -1): + w = _div_nearest(M, k) - _div_nearest(yshift*w, M) + + return _div_nearest(w*y, M) + +def _dlog10(c, e, p): + """Given integers c, e and p with c > 0, p >= 0, compute an integer + approximation to 10**p * log10(c*10**e), with an absolute error of + at most 1. Assumes that c*10**e is not exactly 1.""" + + # increase precision by 2; compensate for this by dividing + # final result by 100 + p += 2 + + # write c*10**e as d*10**f with either: + # f >= 0 and 1 <= d <= 10, or + # f <= 0 and 0.1 <= d <= 1. + # Thus for c*10**e close to 1, f = 0 + l = len(str(c)) + f = e+l - (e+l >= 1) + + if p > 0: + M = 10**p + k = e+p-f + if k >= 0: + c *= 10**k + else: + c = _div_nearest(c, 10**-k) + + log_d = _ilog(c, M) # error < 5 + 22 = 27 + log_10 = _log10_digits(p) # error < 1 + log_d = _div_nearest(log_d*M, log_10) + log_tenpower = f*M # exact + else: + log_d = 0 # error < 2.31 + log_tenpower = _div_nearest(f, 10**-p) # error < 0.5 + + return _div_nearest(log_tenpower+log_d, 100) + +def _dlog(c, e, p): + """Given integers c, e and p with c > 0, compute an integer + approximation to 10**p * log(c*10**e), with an absolute error of + at most 1. Assumes that c*10**e is not exactly 1.""" + + # Increase precision by 2. The precision increase is compensated + # for at the end with a division by 100. + p += 2 + + # rewrite c*10**e as d*10**f with either f >= 0 and 1 <= d <= 10, + # or f <= 0 and 0.1 <= d <= 1. Then we can compute 10**p * log(c*10**e) + # as 10**p * log(d) + 10**p*f * log(10). + l = len(str(c)) + f = e+l - (e+l >= 1) + + # compute approximation to 10**p*log(d), with error < 27 + if p > 0: + k = e+p-f + if k >= 0: + c *= 10**k + else: + c = _div_nearest(c, 10**-k) # error of <= 0.5 in c + + # _ilog magnifies existing error in c by a factor of at most 10 + log_d = _ilog(c, 10**p) # error < 5 + 22 = 27 + else: + # p <= 0: just approximate the whole thing by 0; error < 2.31 + log_d = 0 + + # compute approximation to f*10**p*log(10), with error < 11. + if f: + extra = len(str(abs(f)))-1 + if p + extra >= 0: + # error in f * _log10_digits(p+extra) < |f| * 1 = |f| + # after division, error < |f|/10**extra + 0.5 < 10 + 0.5 < 11 + f_log_ten = _div_nearest(f*_log10_digits(p+extra), 10**extra) + else: + f_log_ten = 0 + else: + f_log_ten = 0 + + # error in sum < 11+27 = 38; error after division < 0.38 + 0.5 < 1 + return _div_nearest(f_log_ten + log_d, 100) + +class _Log10Memoize(object): + """Class to compute, store, and allow retrieval of, digits of the + constant log(10) = 2.302585.... This constant is needed by + Decimal.ln, Decimal.log10, Decimal.exp and Decimal.__pow__.""" + def __init__(self): + self.digits = "23025850929940456840179914546843642076011014886" + + def getdigits(self, p): + """Given an integer p >= 0, return floor(10**p)*log(10). + + For example, self.getdigits(3) returns 2302. + """ + # digits are stored as a string, for quick conversion to + # integer in the case that we've already computed enough + # digits; the stored digits should always be correct + # (truncated, not rounded to nearest). + if p < 0: + raise ValueError("p should be nonnegative") + + if p >= len(self.digits): + # compute p+3, p+6, p+9, ... digits; continue until at + # least one of the extra digits is nonzero + extra = 3 + while True: + # compute p+extra digits, correct to within 1ulp + M = 10**(p+extra+2) + digits = str(_div_nearest(_ilog(10*M, M), 100)) + if digits[-extra:] != '0'*extra: + break + extra += 3 + # keep all reliable digits so far; remove trailing zeros + # and next nonzero digit + self.digits = digits.rstrip('0')[:-1] + return int(self.digits[:p+1]) + +_log10_digits = _Log10Memoize().getdigits + +def _iexp(x, M, L=8): + """Given integers x and M, M > 0, such that x/M is small in absolute + value, compute an integer approximation to M*exp(x/M). For 0 <= + x/M <= 2.4, the absolute error in the result is bounded by 60 (and + is usually much smaller).""" + + # Algorithm: to compute exp(z) for a real number z, first divide z + # by a suitable power R of 2 so that |z/2**R| < 2**-L. Then + # compute expm1(z/2**R) = exp(z/2**R) - 1 using the usual Taylor + # series + # + # expm1(x) = x + x**2/2! + x**3/3! + ... + # + # Now use the identity + # + # expm1(2x) = expm1(x)*(expm1(x)+2) + # + # R times to compute the sequence expm1(z/2**R), + # expm1(z/2**(R-1)), ... , exp(z/2), exp(z). + + # Find R such that x/2**R/M <= 2**-L + R = _nbits((x< M + T = -int(-10*len(str(M))//(3*L)) + y = _div_nearest(x, T) + Mshift = M<= 0: + cshift = c*10**shift + else: + cshift = c//10**-shift + quot, rem = divmod(cshift, _log10_digits(q)) + + # reduce remainder back to original precision + rem = _div_nearest(rem, 10**extra) + + # error in result of _iexp < 120; error after division < 0.62 + return _div_nearest(_iexp(rem, 10**p), 1000), quot - p + 3 + +def _dpower(xc, xe, yc, ye, p): + """Given integers xc, xe, yc and ye representing Decimals x = xc*10**xe and + y = yc*10**ye, compute x**y. Returns a pair of integers (c, e) such that: + + 10**(p-1) <= c <= 10**p, and + (c-1)*10**e < x**y < (c+1)*10**e + + in other words, c*10**e is an approximation to x**y with p digits + of precision, and with an error in c of at most 1. (This is + almost, but not quite, the same as the error being < 1ulp: when c + == 10**(p-1) we can only guarantee error < 10ulp.) + + We assume that: x is positive and not equal to 1, and y is nonzero. + """ + + # Find b such that 10**(b-1) <= |y| <= 10**b + b = len(str(abs(yc))) + ye + + # log(x) = lxc*10**(-p-b-1), to p+b+1 places after the decimal point + lxc = _dlog(xc, xe, p+b+1) + + # compute product y*log(x) = yc*lxc*10**(-p-b-1+ye) = pc*10**(-p-1) + shift = ye-b + if shift >= 0: + pc = lxc*yc*10**shift + else: + pc = _div_nearest(lxc*yc, 10**-shift) + + if pc == 0: + # we prefer a result that isn't exactly 1; this makes it + # easier to compute a correctly rounded result in __pow__ + if ((len(str(xc)) + xe >= 1) == (yc > 0)): # if x**y > 1: + coeff, exp = 10**(p-1)+1, 1-p + else: + coeff, exp = 10**p-1, -p + else: + coeff, exp = _dexp(pc, -(p+1), p+1) + coeff = _div_nearest(coeff, 10) + exp += 1 + + return coeff, exp + +def _log10_lb(c, correction = { + '1': 100, '2': 70, '3': 53, '4': 40, '5': 31, + '6': 23, '7': 16, '8': 10, '9': 5}): + """Compute a lower bound for 100*log10(c) for a positive integer c.""" + if c <= 0: + raise ValueError("The argument to _log10_lb should be nonnegative.") + str_c = str(c) + return 100*len(str_c) - correction[str_c[0]] + +##### Helper Functions #################################################### + +def _convert_other(other, raiseit=False, allow_float=False): + """Convert other to Decimal. + + Verifies that it's ok to use in an implicit construction. + If allow_float is true, allow conversion from float; this + is used in the comparison methods (__eq__ and friends). + + """ + if isinstance(other, Decimal): + return other + if isinstance(other, int): + return Decimal(other) + if allow_float and isinstance(other, float): + return Decimal.from_float(other) + + if raiseit: + raise TypeError("Unable to convert %s to Decimal" % other) + return NotImplemented + +def _convert_for_comparison(self, other, equality_op=False): + """Given a Decimal instance self and a Python object other, return + a pair (s, o) of Decimal instances such that "s op o" is + equivalent to "self op other" for any of the 6 comparison + operators "op". + + """ + if isinstance(other, Decimal): + return self, other + + # Comparison with a Rational instance (also includes integers): + # self op n/d <=> self*d op n (for n and d integers, d positive). + # A NaN or infinity can be left unchanged without affecting the + # comparison result. + if isinstance(other, _numbers.Rational): + if not self._is_special: + self = _dec_from_triple(self._sign, + str(int(self._int) * other.denominator), + self._exp) + return self, Decimal(other.numerator) + + # Comparisons with float and complex types. == and != comparisons + # with complex numbers should succeed, returning either True or False + # as appropriate. Other comparisons return NotImplemented. + if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0: + other = other.real + if isinstance(other, float): + context = getcontext() + if equality_op: + context.flags[FloatOperation] = 1 + else: + context._raise_error(FloatOperation, + "strict semantics for mixing floats and Decimals are enabled") + return self, Decimal.from_float(other) + return NotImplemented, NotImplemented + + +##### Setup Specific Contexts ############################################ + +# The default context prototype used by Context() +# Is mutable, so that new contexts can have different default values + +DefaultContext = Context( + prec=28, rounding=ROUND_HALF_EVEN, + traps=[DivisionByZero, Overflow, InvalidOperation], + flags=[], + Emax=999999, + Emin=-999999, + capitals=1, + clamp=0 +) + +# Pre-made alternate contexts offered by the specification +# Don't change these; the user should be able to select these +# contexts and be able to reproduce results from other implementations +# of the spec. + +BasicContext = Context( + prec=9, rounding=ROUND_HALF_UP, + traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow], + flags=[], +) + +ExtendedContext = Context( + prec=9, rounding=ROUND_HALF_EVEN, + traps=[], + flags=[], +) + + +##### crud for parsing strings ############################################# +# +# Regular expression used for parsing numeric strings. Additional +# comments: +# +# 1. Uncomment the two '\s*' lines to allow leading and/or trailing +# whitespace. But note that the specification disallows whitespace in +# a numeric string. +# +# 2. For finite numbers (not infinities and NaNs) the body of the +# number between the optional sign and the optional exponent must have +# at least one decimal digit, possibly after the decimal point. The +# lookahead expression '(?=\d|\.\d)' checks this. + +import re +_parser = re.compile(r""" # A numeric string consists of: +# \s* + (?P[-+])? # an optional sign, followed by either... + ( + (?=\d|\.\d) # ...a number (with at least one digit) + (?P\d*) # having a (possibly empty) integer part + (\.(?P\d*))? # followed by an optional fractional part + (E(?P[-+]?\d+))? # followed by an optional exponent, or... + | + Inf(inity)? # ...an infinity, or... + | + (?Ps)? # ...an (optionally signaling) + NaN # NaN + (?P\d*) # with (possibly empty) diagnostic info. + ) +# \s* + \Z +""", re.VERBOSE | re.IGNORECASE).match + +_all_zeros = re.compile('0*$').match +_exact_half = re.compile('50*$').match + +##### PEP3101 support functions ############################################## +# The functions in this section have little to do with the Decimal +# class, and could potentially be reused or adapted for other pure +# Python numeric classes that want to implement __format__ +# +# A format specifier for Decimal looks like: +# +# [[fill]align][sign][z][#][0][minimumwidth][,][.precision][type] + +_parse_format_specifier_regex = re.compile(r"""\A +(?: + (?P.)? + (?P[<>=^]) +)? +(?P[-+ ])? +(?Pz)? +(?P\#)? +(?P0)? +(?P(?!0)\d+)? +(?P[,_])? +(?:\.(?P0|(?!0)\d+))? +(?P[eEfFgGn%])? +\Z +""", re.VERBOSE|re.DOTALL) + +del re + +# The locale module is only needed for the 'n' format specifier. The +# rest of the PEP 3101 code functions quite happily without it, so we +# don't care too much if locale isn't present. +try: + import locale as _locale +except ImportError: + pass + +def _parse_format_specifier(format_spec, _localeconv=None): + """Parse and validate a format specifier. + + Turns a standard numeric format specifier into a dict, with the + following entries: + + fill: fill character to pad field to minimum width + align: alignment type, either '<', '>', '=' or '^' + sign: either '+', '-' or ' ' + minimumwidth: nonnegative integer giving minimum width + zeropad: boolean, indicating whether to pad with zeros + thousands_sep: string to use as thousands separator, or '' + grouping: grouping for thousands separators, in format + used by localeconv + decimal_point: string to use for decimal point + precision: nonnegative integer giving precision, or None + type: one of the characters 'eEfFgG%', or None + + """ + m = _parse_format_specifier_regex.match(format_spec) + if m is None: + raise ValueError("Invalid format specifier: " + format_spec) + + # get the dictionary + format_dict = m.groupdict() + + # zeropad; defaults for fill and alignment. If zero padding + # is requested, the fill and align fields should be absent. + fill = format_dict['fill'] + align = format_dict['align'] + format_dict['zeropad'] = (format_dict['zeropad'] is not None) + if format_dict['zeropad']: + if fill is not None: + raise ValueError("Fill character conflicts with '0'" + " in format specifier: " + format_spec) + if align is not None: + raise ValueError("Alignment conflicts with '0' in " + "format specifier: " + format_spec) + format_dict['fill'] = fill or ' ' + # PEP 3101 originally specified that the default alignment should + # be left; it was later agreed that right-aligned makes more sense + # for numeric types. See http://bugs.python.org/issue6857. + format_dict['align'] = align or '>' + + # default sign handling: '-' for negative, '' for positive + if format_dict['sign'] is None: + format_dict['sign'] = '-' + + # minimumwidth defaults to 0; precision remains None if not given + format_dict['minimumwidth'] = int(format_dict['minimumwidth'] or '0') + if format_dict['precision'] is not None: + format_dict['precision'] = int(format_dict['precision']) + + # if format type is 'g' or 'G' then a precision of 0 makes little + # sense; convert it to 1. Same if format type is unspecified. + if format_dict['precision'] == 0: + if format_dict['type'] is None or format_dict['type'] in 'gGn': + format_dict['precision'] = 1 + + # determine thousands separator, grouping, and decimal separator, and + # add appropriate entries to format_dict + if format_dict['type'] == 'n': + # apart from separators, 'n' behaves just like 'g' + format_dict['type'] = 'g' + if _localeconv is None: + _localeconv = _locale.localeconv() + if format_dict['thousands_sep'] is not None: + raise ValueError("Explicit thousands separator conflicts with " + "'n' type in format specifier: " + format_spec) + format_dict['thousands_sep'] = _localeconv['thousands_sep'] + format_dict['grouping'] = _localeconv['grouping'] + format_dict['decimal_point'] = _localeconv['decimal_point'] + else: + if format_dict['thousands_sep'] is None: + format_dict['thousands_sep'] = '' + format_dict['grouping'] = [3, 0] + format_dict['decimal_point'] = '.' + + return format_dict + +def _format_align(sign, body, spec): + """Given an unpadded, non-aligned numeric string 'body' and sign + string 'sign', add padding and alignment conforming to the given + format specifier dictionary 'spec' (as produced by + parse_format_specifier). + + """ + # how much extra space do we have to play with? + minimumwidth = spec['minimumwidth'] + fill = spec['fill'] + padding = fill*(minimumwidth - len(sign) - len(body)) + + align = spec['align'] + if align == '<': + result = sign + body + padding + elif align == '>': + result = padding + sign + body + elif align == '=': + result = sign + padding + body + elif align == '^': + half = len(padding)//2 + result = padding[:half] + sign + body + padding[half:] + else: + raise ValueError('Unrecognised alignment field') + + return result + +def _group_lengths(grouping): + """Convert a localeconv-style grouping into a (possibly infinite) + iterable of integers representing group lengths. + + """ + # The result from localeconv()['grouping'], and the input to this + # function, should be a list of integers in one of the + # following three forms: + # + # (1) an empty list, or + # (2) nonempty list of positive integers + [0] + # (3) list of positive integers + [locale.CHAR_MAX], or + + from itertools import chain, repeat + if not grouping: + return [] + elif grouping[-1] == 0 and len(grouping) >= 2: + return chain(grouping[:-1], repeat(grouping[-2])) + elif grouping[-1] == _locale.CHAR_MAX: + return grouping[:-1] + else: + raise ValueError('unrecognised format for grouping') + +def _insert_thousands_sep(digits, spec, min_width=1): + """Insert thousands separators into a digit string. + + spec is a dictionary whose keys should include 'thousands_sep' and + 'grouping'; typically it's the result of parsing the format + specifier using _parse_format_specifier. + + The min_width keyword argument gives the minimum length of the + result, which will be padded on the left with zeros if necessary. + + If necessary, the zero padding adds an extra '0' on the left to + avoid a leading thousands separator. For example, inserting + commas every three digits in '123456', with min_width=8, gives + '0,123,456', even though that has length 9. + + """ + + sep = spec['thousands_sep'] + grouping = spec['grouping'] + + groups = [] + for l in _group_lengths(grouping): + if l <= 0: + raise ValueError("group length should be positive") + # max(..., 1) forces at least 1 digit to the left of a separator + l = min(max(len(digits), min_width, 1), l) + groups.append('0'*(l - len(digits)) + digits[-l:]) + digits = digits[:-l] + min_width -= l + if not digits and min_width <= 0: + break + min_width -= len(sep) + else: + l = max(len(digits), min_width, 1) + groups.append('0'*(l - len(digits)) + digits[-l:]) + return sep.join(reversed(groups)) + +def _format_sign(is_negative, spec): + """Determine sign character.""" + + if is_negative: + return '-' + elif spec['sign'] in ' +': + return spec['sign'] + else: + return '' + +def _format_number(is_negative, intpart, fracpart, exp, spec): + """Format a number, given the following data: + + is_negative: true if the number is negative, else false + intpart: string of digits that must appear before the decimal point + fracpart: string of digits that must come after the point + exp: exponent, as an integer + spec: dictionary resulting from parsing the format specifier + + This function uses the information in spec to: + insert separators (decimal separator and thousands separators) + format the sign + format the exponent + add trailing '%' for the '%' type + zero-pad if necessary + fill and align if necessary + """ + + sign = _format_sign(is_negative, spec) + + if fracpart or spec['alt']: + fracpart = spec['decimal_point'] + fracpart + + if exp != 0 or spec['type'] in 'eE': + echar = {'E': 'E', 'e': 'e', 'G': 'E', 'g': 'e'}[spec['type']] + fracpart += "{0}{1:+}".format(echar, exp) + if spec['type'] == '%': + fracpart += '%' + + if spec['zeropad']: + min_width = spec['minimumwidth'] - len(fracpart) - len(sign) + else: + min_width = 0 + intpart = _insert_thousands_sep(intpart, spec, min_width) + + return _format_align(sign, intpart+fracpart, spec) + + +##### Useful Constants (internal use only) ################################ + +# Reusable defaults +_Infinity = Decimal('Inf') +_NegativeInfinity = Decimal('-Inf') +_NaN = Decimal('NaN') +_Zero = Decimal(0) +_One = Decimal(1) +_NegativeOne = Decimal(-1) + +# _SignedInfinity[sign] is infinity w/ that sign +_SignedInfinity = (_Infinity, _NegativeInfinity) + +# Constants related to the hash implementation; hash(x) is based +# on the reduction of x modulo _PyHASH_MODULUS +_PyHASH_MODULUS = sys.hash_info.modulus +# hash values to use for positive and negative infinities, and nans +_PyHASH_INF = sys.hash_info.inf +_PyHASH_NAN = sys.hash_info.nan + +# _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS +_PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) +del sys diff --git a/crates/weavepy-vm/src/stdlib/python/_weakrefset.py b/crates/weavepy-vm/src/stdlib/python/_weakrefset.py new file mode 100644 index 0000000..489eec7 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_weakrefset.py @@ -0,0 +1,205 @@ +# Access WeakSet through the weakref module. +# This code is separated-out because it is needed +# by abc.py to load everything else at startup. + +from _weakref import ref +from types import GenericAlias + +__all__ = ['WeakSet'] + + +class _IterationGuard: + # This context manager registers itself in the current iterators of the + # weak container, such as to delay all removals until the context manager + # exits. + # This technique should be relatively thread-safe (since sets are). + + def __init__(self, weakcontainer): + # Don't create cycles + self.weakcontainer = ref(weakcontainer) + + def __enter__(self): + w = self.weakcontainer() + if w is not None: + w._iterating.add(self) + return self + + def __exit__(self, e, t, b): + w = self.weakcontainer() + if w is not None: + s = w._iterating + s.remove(self) + if not s: + w._commit_removals() + + +class WeakSet: + def __init__(self, data=None): + self.data = set() + def _remove(item, selfref=ref(self)): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(item) + else: + self.data.discard(item) + self._remove = _remove + # A list of keys to be removed + self._pending_removals = [] + self._iterating = set() + if data is not None: + self.update(data) + + def _commit_removals(self): + pop = self._pending_removals.pop + discard = self.data.discard + while True: + try: + item = pop() + except IndexError: + return + discard(item) + + def __iter__(self): + with _IterationGuard(self): + for itemref in self.data: + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item + + def __len__(self): + return len(self.data) - len(self._pending_removals) + + def __contains__(self, item): + try: + wr = ref(item) + except TypeError: + return False + return wr in self.data + + def __reduce__(self): + return self.__class__, (list(self),), self.__getstate__() + + def add(self, item): + if self._pending_removals: + self._commit_removals() + self.data.add(ref(item, self._remove)) + + def clear(self): + if self._pending_removals: + self._commit_removals() + self.data.clear() + + def copy(self): + return self.__class__(self) + + def pop(self): + if self._pending_removals: + self._commit_removals() + while True: + try: + itemref = self.data.pop() + except KeyError: + raise KeyError('pop from empty WeakSet') from None + item = itemref() + if item is not None: + return item + + def remove(self, item): + if self._pending_removals: + self._commit_removals() + self.data.remove(ref(item)) + + def discard(self, item): + if self._pending_removals: + self._commit_removals() + self.data.discard(ref(item)) + + def update(self, other): + if self._pending_removals: + self._commit_removals() + for element in other: + self.add(element) + + def __ior__(self, other): + self.update(other) + return self + + def difference(self, other): + newset = self.copy() + newset.difference_update(other) + return newset + __sub__ = difference + + def difference_update(self, other): + self.__isub__(other) + def __isub__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.difference_update(ref(item) for item in other) + return self + + def intersection(self, other): + return self.__class__(item for item in other if item in self) + __and__ = intersection + + def intersection_update(self, other): + self.__iand__(other) + def __iand__(self, other): + if self._pending_removals: + self._commit_removals() + self.data.intersection_update(ref(item) for item in other) + return self + + def issubset(self, other): + return self.data.issubset(ref(item) for item in other) + __le__ = issubset + + def __lt__(self, other): + return self.data < set(map(ref, other)) + + def issuperset(self, other): + return self.data.issuperset(ref(item) for item in other) + __ge__ = issuperset + + def __gt__(self, other): + return self.data > set(map(ref, other)) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.data == set(map(ref, other)) + + def symmetric_difference(self, other): + newset = self.copy() + newset.symmetric_difference_update(other) + return newset + __xor__ = symmetric_difference + + def symmetric_difference_update(self, other): + self.__ixor__(other) + def __ixor__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.symmetric_difference_update(ref(item, self._remove) for item in other) + return self + + def union(self, other): + return self.__class__(e for s in (self, other) for e in s) + __or__ = union + + def isdisjoint(self, other): + return len(self.intersection(other)) == 0 + + def __repr__(self): + return repr(self.data) + + __class_getitem__ = classmethod(GenericAlias) diff --git a/crates/weavepy-vm/src/stdlib/python/abc.py b/crates/weavepy-vm/src/stdlib/python/abc.py index 989fd76..81cecff 100644 --- a/crates/weavepy-vm/src/stdlib/python/abc.py +++ b/crates/weavepy-vm/src/stdlib/python/abc.py @@ -1,175 +1,136 @@ -"""Abstract Base Classes (PEP 3119), simplified. +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. -This is a WeavePy-compatible re-implementation of the surface of -CPython's :mod:`abc`. It supports: +"""Abstract Base Classes (ABCs) according to PEP 3119.""" -- ``ABCMeta``: a metaclass that records abstract methods and - permits virtual-subclass registration via ``register()``. -- ``ABC``: a convenience base whose metaclass is :class:`ABCMeta`. -- ``abstractmethod``: marks a method as abstract. -- ``abstractproperty`` / ``abstractclassmethod`` / ``abstractstaticmethod`` - for backward compatibility — the documented modern form is - ``@property @abstractmethod`` (etc.), but the old decorators are - still in widespread use. -What is intentionally omitted: +def abstractmethod(funcobj): + """A decorator indicating abstract methods. -- ``ABCMeta.__subclasshook__`` slow-path that walks the registered - subclass cache invalidating it as classes get added — we use a - simple set, since our object graph isn't watched for invalidation. -- The ``_py_abc`` C-accelerated fast path; everything here is pure - Python. -""" + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract methods are overridden. + The abstract methods can be called using any of the normal + 'super' call mechanisms. abstractmethod() may be used to declare + abstract methods for properties and descriptors. + Usage: -def abstractmethod(funcobj): - """Mark *funcobj* as abstract. The decorated callable still works - when invoked through ``super()`` from a concrete subclass; what - changes is that ``ABCMeta`` refuses to instantiate any class - that still carries an abstract method by the time its class - statement finishes. + class C(metaclass=ABCMeta): + @abstractmethod + def my_abstract_method(self, arg1, arg2, argN): + ... """ funcobj.__isabstractmethod__ = True return funcobj -class ABCMeta(type): - """Metaclass for defining Abstract Base Classes (ABCs). +class abstractclassmethod(classmethod): + """A decorator indicating abstract classmethods. + + Deprecated, use 'classmethod' with 'abstractmethod' instead: + + class C(ABC): + @classmethod + @abstractmethod + def my_abstract_classmethod(cls, ...): + ... - Use this metaclass to create an ABC. An ABC can be subclassed - directly, and then acts as a mix-in class. You can also register - unrelated concrete classes (even built-in classes) and unrelated - ABCs as 'virtual subclasses' — these and their descendants will - be considered subclasses of the registering ABC by the built-in - ``issubclass()`` function, but the registering ABC won't show up - in their MRO, nor will method implementations defined by the - registering ABC be callable (not even via ``super()``). """ - def __init__(cls, name, bases, namespace, **kwargs): - super().__init__(name, bases, namespace, **kwargs) - # Collect abstract methods declared on the class plus any - # inherited ones that are not overridden by a concrete impl. - abstracts = set() - for attr_name, value in namespace.items(): - if getattr(value, "__isabstractmethod__", False): - abstracts.add(attr_name) - for base in bases: - for attr_name in getattr(base, "__abstractmethods__", set()): - value = namespace.get(attr_name, None) - if value is None: - value = getattr(cls, attr_name, None) - if getattr(value, "__isabstractmethod__", False): - abstracts.add(attr_name) - cls.__abstractmethods__ = frozenset(abstracts) - cls._abc_registry = set() - cls._abc_descendants = [] - # Register this class on every ABCMeta-typed ancestor so - # ``Real.__subclasscheck__(...)`` can find concrete types - # registered on a more specific ABC like ``Integral``. - for base in bases: - if isinstance(base, ABCMeta): - base._abc_descendants.append(cls) - for ancestor in getattr(base, "_abc_descendants_parents", ()): - ancestor._abc_descendants.append(cls) - parents = [] - for base in bases: - if isinstance(base, ABCMeta): - parents.append(base) - parents.extend(getattr(base, "_abc_descendants_parents", ())) - cls._abc_descendants_parents = tuple(parents) - - def __call__(cls, *args, **kwargs): - # Refuse to instantiate a class that still has unimplemented - # abstract methods. Mirrors CPython's `object_new` check, but - # implemented here so we don't need a special hook in the VM. - abstracts = getattr(cls, "__abstractmethods__", None) - if abstracts: - names = ", ".join(sorted(abstracts)) - raise TypeError( - f"Can't instantiate abstract class {cls.__name__} " - f"with abstract methods {names}" - ) - # Bypass ``super().__call__`` (which would round-trip through - # ``type.__call__`` — not yet a real callable in this VM) and - # invoke the standard ``__new__`` / ``__init__`` dance directly. - new = cls.__new__ - instance = new(cls, *args, **kwargs) - if isinstance(instance, cls): - instance.__init__(*args, **kwargs) - return instance - - def register(cls, subclass): - """Register *subclass* as a virtual subclass of this ABC.""" - if not isinstance(subclass, type): - raise TypeError("Can only register classes") - if issubclass(subclass, cls): - return subclass - cls._abc_registry.add(subclass) - return subclass - - def __instancecheck__(cls, instance): - return cls.__subclasscheck__(type(instance)) - - def __subclasscheck__(cls, subclass): - # Fast path: ordinary subclass relationship. - if cls is subclass: - return True - if type(subclass) is type or isinstance(subclass, type): - if cls in getattr(subclass, "__mro__", ()): - return True - # Registered virtual subclasses (directly or transitively). - registry = getattr(cls, "_abc_registry", ()) - for reg in registry: - if reg is subclass: - return True - if isinstance(subclass, type) and issubclass(subclass, reg): - return True - # If we are an ABC, also consult the registries of all - # known descendant ABCs (tracked at class-creation time — - # WeavePy does not yet expose ``type.__subclasses__``). - for sub in getattr(cls, "_abc_descendants", ()): - if sub is cls: - continue - sub_registry = getattr(sub, "_abc_registry", ()) - for reg in sub_registry: - if reg is subclass: - return True - if isinstance(subclass, type) and issubclass(subclass, reg): - return True - return False + __isabstractmethod__ = True + def __init__(self, callable): + callable.__isabstractmethod__ = True + super().__init__(callable) -class ABC(metaclass=ABCMeta): - """Helper class — direct inheritance avoids having to spell - ``metaclass=ABCMeta`` every time.""" - __slots__ = () +class abstractstaticmethod(staticmethod): + """A decorator indicating abstract staticmethods. + Deprecated, use 'staticmethod' with 'abstractmethod' instead: -def abstractproperty(funcobj): - funcobj = property(funcobj) - funcobj.__isabstractmethod__ = True - return funcobj + class C(ABC): + @staticmethod + @abstractmethod + def my_abstract_staticmethod(...): + ... + + """ + + __isabstractmethod__ = True + def __init__(self, callable): + callable.__isabstractmethod__ = True + super().__init__(callable) -def abstractclassmethod(funcobj): - cm = classmethod(funcobj) - cm.__isabstractmethod__ = True - return cm +class abstractproperty(property): + """A decorator indicating abstract properties. -def abstractstaticmethod(funcobj): - sm = staticmethod(funcobj) - sm.__isabstractmethod__ = True - return sm + Deprecated, use 'property' with 'abstractmethod' instead: + + class C(ABC): + @property + @abstractmethod + def my_abstract_property(self): + ... + + """ + __isabstractmethod__ = True -__all__ = [ - "ABCMeta", - "ABC", - "abstractmethod", - "abstractproperty", - "abstractclassmethod", - "abstractstaticmethod", -] + +# WeavePy ships a minimal `_abc` accelerator, so — like a CPython build +# compiled without the `_abc` C extension — we use the pure-Python +# `_py_abc` implementation as the source of truth. `_py_abc.ABCMeta` +# implements the full PEP 3119 protocol (virtual-subclass registry, +# `__subclasshook__`, negative-result caching keyed on the invalidation +# counter) on top of `type.__subclasses__()` and `_weakrefset.WeakSet`. +from _py_abc import ABCMeta, get_cache_token +ABCMeta.__module__ = 'abc' + + +def update_abstractmethods(cls): + """Recalculate the set of abstract methods of an abstract class. + + If a class has had one of its abstract methods implemented after the + class was created, the method will not be considered implemented until + this function is called. Alternatively, if a new abstract method has been + added to the class, it will only be considered an abstract method of the + class after this function is called. + + This function should be called before any use is made of the class, + usually in class decorators that add methods to the subject class. + + Returns cls, to allow usage as a class decorator. + + If cls is not an instance of ABCMeta, does nothing. + """ + if not hasattr(cls, '__abstractmethods__'): + # We check for __abstractmethods__ here because cls might by a C + # implementation or a python implementation (especially during + # testing), and we want to handle both cases. + return cls + + abstracts = set() + # Check the existing abstract methods of the parents, keep only the ones + # that are not implemented. + for scls in cls.__bases__: + for name in getattr(scls, '__abstractmethods__', ()): + value = getattr(cls, name, None) + if getattr(value, "__isabstractmethod__", False): + abstracts.add(name) + # Also add any other newly added abstract methods. + for name, value in cls.__dict__.items(): + if getattr(value, "__isabstractmethod__", False): + abstracts.add(name) + cls.__abstractmethods__ = frozenset(abstracts) + return cls + + +class ABC(metaclass=ABCMeta): + """Helper class that provides a standard way to create an ABC using + inheritance. + """ + __slots__ = () diff --git a/crates/weavepy-vm/src/stdlib/python/argparse.py b/crates/weavepy-vm/src/stdlib/python/argparse.py index d7678b8..a6880f8 100644 --- a/crates/weavepy-vm/src/stdlib/python/argparse.py +++ b/crates/weavepy-vm/src/stdlib/python/argparse.py @@ -68,6 +68,26 @@ def _flag_to_dest(flag): return flag.lstrip("-").replace("-", "_") +class _ArgumentGroup: + """Lightweight stand-in for ``argparse``'s argument groups: forwards + ``add_argument`` to the owning parser so grouped options participate in + normal parsing (we don't render the per-group help sections).""" + + def __init__(self, container, title=None, description=None): + self._container = container + self.title = title + self.description = description + + def add_argument(self, *flags, **kwargs): + return self._container.add_argument(*flags, **kwargs) + + def add_argument_group(self, *args, **kwargs): + return _ArgumentGroup(self._container) + + def add_mutually_exclusive_group(self, **kwargs): + return _ArgumentGroup(self._container) + + class ArgumentParser: def __init__( self, @@ -131,6 +151,15 @@ def add_argument(self, *flags, **kwargs): self._actions.append(a) return a + def add_argument_group(self, *args, **kwargs): + # Argument groups only affect help grouping in CPython; their + # arguments live in the parent parser's action list. A thin proxy + # that forwards `add_argument` is enough for parsing parity. + return _ArgumentGroup(self) + + def add_mutually_exclusive_group(self, **kwargs): + return _ArgumentGroup(self) + def _flag_action(self, token): for action in self._actions: for flag in action.flags: @@ -140,6 +169,21 @@ def _flag_action(self, token): return action return None + def _convert(self, action, value): + """Apply ``action.type`` to ``value``, converting a failed + conversion into argparse's ``usage:`` + ``error:`` exit (SystemExit), + exactly like CPython's ``_get_value``.""" + if action.type is None or value is None: + return value + try: + return action.type(value) + except (ValueError, TypeError): + name = action.flags[0] if action.flags else action.dest + type_name = getattr(action.type, "__name__", repr(action.type)) + self.error( + "argument %s: invalid %s value: %r" % (name, type_name, value) + ) + def _apply_action(self, action, value, namespace): if action.choices is not None and value not in action.choices: self.error( @@ -149,13 +193,13 @@ def _apply_action(self, action, value, namespace): + repr(value) ) if action.action == "store": - converted = action.type(value) if action.type and value is not None else value + converted = self._convert(action, value) setattr(namespace, action.dest, converted) elif action.action == "append": existing = getattr(namespace, action.dest, None) if existing is None: existing = [] - converted = action.type(value) if action.type else value + converted = self._convert(action, value) existing.append(converted) setattr(namespace, action.dest, existing) @@ -226,9 +270,7 @@ def parse_args(self, args=None, namespace=None): value = positional_values.pop(0) self._apply_action(action, value, namespace) elif nargs == "*": - values = [ - action.type(v) if action.type else v for v in positional_values - ] + values = [self._convert(action, v) for v in positional_values] positional_values = [] setattr(namespace, action.dest, values) elif nargs == "+": @@ -236,9 +278,7 @@ def parse_args(self, args=None, namespace=None): self.error( "the following arguments are required: " + action.dest ) - values = [ - action.type(v) if action.type else v for v in positional_values - ] + values = [self._convert(action, v) for v in positional_values] positional_values = [] setattr(namespace, action.dest, values) elif nargs == "?": @@ -252,7 +292,7 @@ def parse_args(self, args=None, namespace=None): self.error("expected " + str(nargs) + " arguments for " + action.dest) taken = positional_values[:nargs] positional_values = positional_values[nargs:] - values = [action.type(v) if action.type else v for v in taken] + values = [self._convert(action, v) for v in taken] setattr(namespace, action.dest, values) else: self.error("invalid nargs value for " + action.dest) @@ -347,15 +387,11 @@ def parse_known_args(self, args=None, namespace=None): else: setattr(namespace, action.dest, action.default) elif nargs == "*": - values = [ - action.type(v) if action.type else v for v in positional_values - ] + values = [self._convert(action, v) for v in positional_values] positional_values = [] setattr(namespace, action.dest, values) elif nargs == "+": - values = [ - action.type(v) if action.type else v for v in positional_values - ] + values = [self._convert(action, v) for v in positional_values] positional_values = [] setattr(namespace, action.dest, values) elif nargs == "?": @@ -367,7 +403,7 @@ def parse_known_args(self, args=None, namespace=None): elif isinstance(nargs, int): taken = positional_values[:nargs] positional_values = positional_values[nargs:] - values = [action.type(v) if action.type else v for v in taken] + values = [self._convert(action, v) for v in taken] setattr(namespace, action.dest, values) # Anything still in `args` that wasn't consumed is "unknown". @@ -390,6 +426,14 @@ def format_help(self): lines.append(self.epilog) return "\n".join(lines) + def format_usage(self): + return "usage: " + self.prog + " [options]\n" + + def print_usage(self, file=None): + if file is None: + file = sys.stdout + file.write(self.format_usage()) + def print_help(self, file=None): if file is None: file = sys.stdout @@ -397,5 +441,7 @@ def print_help(self, file=None): file.write("\n") def error(self, message): + # CPython prints the usage line before the error, then exits 2. + self.print_usage(sys.stderr) sys.stderr.write(self.prog + ": error: " + message + "\n") sys.exit(2) diff --git a/crates/weavepy-vm/src/stdlib/python/array_mod.py b/crates/weavepy-vm/src/stdlib/python/array_mod.py index f0961fa..46e1d26 100644 --- a/crates/weavepy-vm/src/stdlib/python/array_mod.py +++ b/crates/weavepy-vm/src/stdlib/python/array_mod.py @@ -103,6 +103,12 @@ def frombytes(self, blob): def tobytes(self): return b''.join(_struct.pack(self._fmt, v) for v in self._data) + def __buffer__(self, flags): + # PEP 688 buffer protocol: expose the packed bytes so buffer + # consumers (``float``/``int``/``bytes``/``memoryview``) can read the + # array's contents, mirroring CPython's C-level buffer export. + return memoryview(self.tobytes()) + def fromlist(self, seq): for v in seq: self._data.append(self._coerce(v)) diff --git a/crates/weavepy-vm/src/stdlib/python/bdb_mod.py b/crates/weavepy-vm/src/stdlib/python/bdb_mod.py index 41add2b..f256b56 100644 --- a/crates/weavepy-vm/src/stdlib/python/bdb_mod.py +++ b/crates/weavepy-vm/src/stdlib/python/bdb_mod.py @@ -1,204 +1,247 @@ -"""``bdb`` — generic Python debugger base class. +"""Debugger basics""" -Concrete debuggers (``pdb``) subclass :class:`Bdb` and override -``user_line``, ``user_call``, ``user_return``, and -``user_exception``. ``Bdb`` manages the per-file breakpoint table -and the trace-function hook. - -This implementation tracks CPython's ``Lib/bdb.py`` surface for the -methods most pdb commands reach for. The deep loop-control corners -(``runeval``, ``runcall``, multi-thread tracing) are approximate. -""" - -import os +import fnmatch import sys +import os +from contextlib import contextmanager +from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR -__all__ = ['Bdb', 'Breakpoint', 'BdbQuit', 'GENERATOR_AND_COROUTINE_FLAGS', - 'set_trace', 'effective', 'checkfuncname'] - +__all__ = ["BdbQuit", "Bdb", "Breakpoint"] -GENERATOR_AND_COROUTINE_FLAGS = 0x20 | 0x100 | 0x200 +GENERATOR_AND_COROUTINE_FLAGS = CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR class BdbQuit(Exception): - """Raised when the user quits the debugger.""" - - -class Breakpoint: - """Represent a single source-line breakpoint.""" + """Exception to give up completely.""" - next = 1 - bplist = {} # (file, line) -> [Breakpoint] - bpbynumber = [None] - def __init__(self, file, line, temporary=False, cond=None, - funcname=None): - self.file = file - self.line = line - self.temporary = temporary - self.cond = cond - self.funcname = funcname - self.enabled = True - self.ignore = 0 - self.hits = 0 - self.number = Breakpoint.next - Breakpoint.next += 1 - Breakpoint.bpbynumber.append(self) - Breakpoint.bplist.setdefault((file, line), []).append(self) - - def deleteMe(self): - Breakpoint.bplist[(self.file, self.line)].remove(self) - if not Breakpoint.bplist[(self.file, self.line)]: - del Breakpoint.bplist[(self.file, self.line)] - Breakpoint.bpbynumber[self.number] = None - - def enable(self): - self.enabled = True - - def disable(self): - self.enabled = False - - def bpprint(self, out=None): - out = out or sys.stdout - out.write('{} breakpoint keep {} at {}:{}\n'.format( - self.number, 'yes' if self.enabled else 'no', - self.file, self.line)) - if self.cond: - out.write('\tstop only if {}\n'.format(self.cond)) - if self.ignore: - out.write('\tignore next {} hits\n'.format(self.ignore)) - if self.hits: - out.write('\tbreakpoint already hit {} times\n'.format(self.hits)) - - def __str__(self): - return 'breakpoint {} at {}:{}'.format( - self.number, self.file, self.line) - - -def checkfuncname(b, frame): - if not b.funcname: - return True - if frame.f_code.co_name != b.funcname: - return False - return frame.f_code.co_firstlineno == b.line - - -def effective(file, line, frame): - """Identify the active breakpoint at ``(file, line)`` for ``frame``.""" - possibles = Breakpoint.bplist.get((file, line), []) - for b in possibles: - if not b.enabled: - continue - if not checkfuncname(b, frame): - continue - b.hits += 1 - if b.cond: - try: - ok = eval(b.cond, frame.f_globals, frame.f_locals) - except Exception: - return b, False - if not ok: - continue - if b.ignore > 0: - b.ignore -= 1 - continue - return b, True - return None, False +class Bdb: + """Generic Python debugger base class. + This class takes care of details of the trace facility; + a derived class should implement user interaction. + The standard debugger class (pdb.Pdb) is an example. -class Bdb: - """Generic Python debugger base.""" + The optional skip argument must be an iterable of glob-style + module name patterns. The debugger will not step into frames + that originate in a module that matches one of these patterns. + Whether a frame is considered to originate in a certain module + is determined by the __name__ in the frame globals. + """ def __init__(self, skip=None): self.skip = set(skip) if skip else None - self.breaks = {} # filename -> set of linenos + self.breaks = {} self.fncache = {} + self.frame_trace_lines_opcodes = {} self.frame_returning = None - self.botframe = None - self.stopframe = None - self.returnframe = None - self.quitting = False - self.stoplineno = 0 + self.trace_opcodes = False + self.enterframe = None + self.cmdframe = None + self.cmdlineno = None - # ---- canonicalisation ------------------------------------------------ + self._load_breaks() def canonic(self, filename): - if not filename: - return filename - if filename == '<' + filename[1:-1] + '>': + """Return canonical form of filename. + + For real filenames, the canonical form is a case-normalized (on + case insensitive filesystems) absolute path. 'Filenames' with + angle brackets, such as "", generated in interactive + mode, are returned unchanged. + """ + if filename == "<" + filename[1:-1] + ">": return filename canonic = self.fncache.get(filename) - if canonic is None: + if not canonic: canonic = os.path.abspath(filename) canonic = os.path.normcase(canonic) self.fncache[filename] = canonic return canonic - # ---- trace dispatch ------------------------------------------------- - def reset(self): + """Set values of attributes as ready to start debugging.""" import linecache linecache.checkcache() self.botframe = None - self.stopframe = None - self.returnframe = None - self.quitting = False + self._set_stopinfo(None, None) + + @contextmanager + def set_enterframe(self, frame): + self.enterframe = frame + yield + self.enterframe = None def trace_dispatch(self, frame, event, arg): - if self.quitting: - return None - if event == 'line': - return self.dispatch_line(frame) - if event == 'call': - return self.dispatch_call(frame, arg) - if event == 'return': - return self.dispatch_return(frame, arg) - if event == 'exception': - return self.dispatch_exception(frame, arg) - if event == 'opcode': + """Dispatch a trace function for debugged frames based on the event. + + This function is installed as the trace function for debugged + frames. Its return value is the new trace function, which is + usually itself. The default implementation decides how to + dispatch a frame, depending on the type of event (passed in as a + string) that is about to be executed. + + The event can be one of the following: + line: A new line of code is going to be executed. + call: A function is about to be called or another code block + is entered. + return: A function or other code block is about to return. + exception: An exception has occurred. + c_call: A C function is about to be called. + c_return: A C function has returned. + c_exception: A C function has raised an exception. + + For the Python events, specialized functions (see the dispatch_*() + methods) are called. For the C events, no action is taken. + + The arg parameter depends on the previous event. + """ + + with self.set_enterframe(frame): + if self.quitting: + return # None + if event == 'line': + return self.dispatch_line(frame) + if event == 'call': + return self.dispatch_call(frame, arg) + if event == 'return': + return self.dispatch_return(frame, arg) + if event == 'exception': + return self.dispatch_exception(frame, arg) + if event == 'c_call': + return self.trace_dispatch + if event == 'c_exception': + return self.trace_dispatch + if event == 'c_return': + return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) + print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch - return self.trace_dispatch def dispatch_line(self, frame): - if self.stop_here(frame) or self.break_here(frame): + """Invoke user function and return trace function for line event. + + If the debugger stops on the current line, invoke + self.user_line(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) - if self.quitting: - raise BdbQuit + if self.quitting: raise BdbQuit return self.trace_dispatch def dispatch_call(self, frame, arg): + """Invoke user function and return trace function for call event. + + If the debugger stops on this function call, invoke + self.user_call(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ + # XXX 'arg' is no longer used if self.botframe is None: - self.botframe = frame.f_back + # First call of dispatch since reset() + self.botframe = frame.f_back # (CT) Note that this may also be None! + return self.trace_dispatch + if not (self.stop_here(frame) or self.break_anywhere(frame)): + # No need to trace this function + return # None + # Ignore call events in generator except when stepping. + if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: return self.trace_dispatch - if not self.stop_here(frame) and not self.break_anywhere(frame): - return None self.user_call(frame, arg) - if self.quitting: - raise BdbQuit + if self.quitting: raise BdbQuit return self.trace_dispatch def dispatch_return(self, frame, arg): + """Invoke user function and return trace function for return event. + + If the debugger stops on this function return, invoke + self.user_return(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ if self.stop_here(frame) or frame == self.returnframe: + # Ignore return events in generator except when stepping. + if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: + return self.trace_dispatch try: self.frame_returning = frame self.user_return(frame, arg) finally: self.frame_returning = None - if self.quitting: - raise BdbQuit + if self.quitting: raise BdbQuit + # The user issued a 'next' or 'until' command. + if self.stopframe is frame and self.stoplineno != -1: + self._set_stopinfo(None, None) + # The previous frame might not have f_trace set, unless we are + # issuing a command that does not expect to stop, we should set + # f_trace + if self.stoplineno != -1: + self._set_caller_tracefunc(frame) return self.trace_dispatch def dispatch_exception(self, frame, arg): + """Invoke user function and return trace function for exception event. + + If the debugger stops on this exception, invoke + self.user_exception(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ if self.stop_here(frame): + # When stepping with next/until/return in a generator frame, skip + # the internal StopIteration exception (with no traceback) + # triggered by a subiterator run with the 'yield from' statement. + if not (frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS + and arg[0] is StopIteration and arg[2] is None): + self.user_exception(frame, arg) + if self.quitting: raise BdbQuit + # Stop at the StopIteration or GeneratorExit exception when the user + # has set stopframe in a generator by issuing a return command, or a + # next/until command at the last statement in the generator before the + # exception. + elif (self.stopframe and frame is not self.stopframe + and self.stopframe.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS + and arg[0] in (StopIteration, GeneratorExit)): self.user_exception(frame, arg) - if self.quitting: - raise BdbQuit + if self.quitting: raise BdbQuit + return self.trace_dispatch - # ---- stop checks ----------------------------------------------------- + def dispatch_opcode(self, frame, arg): + """Invoke user function and return trace function for opcode event. + If the debugger stops on the current opcode, invoke + self.user_opcode(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ + if self.stop_here(frame) or self.break_here(frame): + self.user_opcode(frame) + if self.quitting: raise BdbQuit + return self.trace_dispatch + + # Normally derived classes don't override the following + # methods, but they may if they want to redefine the + # definition of stopping and breakpoints. + + def is_skipped_module(self, module_name): + "Return True if module_name matches any skip pattern." + if module_name is None: # some modules do not have names + return False + for pattern in self.skip: + if fnmatch.fnmatch(module_name, pattern): + return True + return False def stop_here(self, frame): - if self.skip and self.is_skipped_module(frame.f_globals.get('__name__')): + "Return True if frame is below the starting frame in the stack." + # (CT) stopframe may now also be None, see dispatch_call. + # (CT) the former test for None is therefore removed from here. + if self.skip and \ + self.is_skipped_module(frame.f_globals.get('__name__')): return False if frame is self.stopframe: if self.stoplineno == -1: @@ -209,161 +252,722 @@ def stop_here(self, frame): return False def break_here(self, frame): + """Return True if there is an effective breakpoint for this line. + + Check for line or function breakpoint and if in effect. + Delete temporary breakpoints if effective() says to. + """ filename = self.canonic(frame.f_code.co_filename) if filename not in self.breaks: return False lineno = frame.f_lineno if lineno not in self.breaks[filename]: + # The line itself has no breakpoint, but maybe the line is the + # first line of a function with breakpoint set by function name. + lineno = frame.f_code.co_firstlineno + if lineno not in self.breaks[filename]: + return False + + # flag says ok to delete temp. bp + (bp, flag) = effective(filename, lineno, frame) + if bp: + self.currentbp = bp.number + if (flag and bp.temporary): + self.do_clear(str(bp.number)) + return True + else: return False - bp, found = effective(filename, lineno, frame) - if not found: - return False - self.currentbp = bp.number - if bp.temporary: - self.do_clear(str(bp.number)) - return True - def break_anywhere(self, frame): - filename = self.canonic(frame.f_code.co_filename) - return filename in self.breaks + def do_clear(self, arg): + """Remove temporary breakpoint. - def is_skipped_module(self, module_name): - return module_name in self.skip if self.skip and module_name else False + Must implement in derived classes or get NotImplementedError. + """ + raise NotImplementedError("subclass of bdb must implement do_clear()") + + def break_anywhere(self, frame): + """Return True if there is any breakpoint for frame's filename. + """ + return self.canonic(frame.f_code.co_filename) in self.breaks - # ---- user hooks (override in subclasses) ----------------------------- + # Derived classes should override the user_* methods + # to gain control. def user_call(self, frame, argument_list): + """Called if we might stop in a function.""" pass def user_line(self, frame): + """Called when we stop or break at a line.""" pass def user_return(self, frame, return_value): + """Called when a return trap is set here.""" pass def user_exception(self, frame, exc_info): + """Called when we stop on an exception.""" pass - # ---- step / continue ------------------------------------------------- + def user_opcode(self, frame): + """Called when we are about to execute an opcode.""" + pass - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): + def _set_trace_opcodes(self, trace_opcodes): + if trace_opcodes != self.trace_opcodes: + self.trace_opcodes = trace_opcodes + frame = self.enterframe + while frame is not None: + frame.f_trace_opcodes = trace_opcodes + if frame is self.botframe: + break + frame = frame.f_back + + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): + """Set the attributes for stopping. + + If stoplineno is greater than or equal to 0, then stop at line + greater than or equal to the stopline. If stoplineno is -1, then + don't stop at all. + """ self.stopframe = stopframe self.returnframe = returnframe self.quitting = False + # stoplineno >= 0 means: stop at line >= the stoplineno + # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno + self._set_trace_opcodes(opcode) + + def _set_caller_tracefunc(self, current_frame): + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame, unless + # the caller is the botframe. + caller_frame = current_frame.f_back + if caller_frame and not caller_frame.f_trace and caller_frame is not self.botframe: + caller_frame.f_trace = self.trace_dispatch + + # Derived classes and clients can call the following methods + # to affect the stepping state. def set_until(self, frame, lineno=None): + """Stop when the line with the lineno greater than the current one is + reached or when returning from current frame.""" + # the name "until" is borrowed from gdb if lineno is None: lineno = frame.f_lineno + 1 self._set_stopinfo(frame, frame, lineno) def set_step(self): - self._set_stopinfo(None, None) + """Stop after one line of code.""" + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) + + def set_stepinstr(self): + """Stop before the next instruction.""" + self._set_stopinfo(None, None, opcode=True) def set_next(self, frame): - self._set_stopinfo(frame, None) + """Stop on the next line in or below the given frame.""" + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): - self._set_stopinfo(frame.f_back, frame) + """Stop when returning from the given frame.""" + if frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: + self._set_stopinfo(frame, None, -1) + else: + self._set_stopinfo(frame.f_back, frame) def set_trace(self, frame=None): + """Start debugging from frame. + + If frame is not specified, debugging starts from caller's frame. + """ if frame is None: frame = sys._getframe().f_back self.reset() - while frame: - frame.f_trace = self.trace_dispatch - self.botframe = frame - frame = frame.f_back - self.set_step() + with self.set_enterframe(frame): + while frame: + frame.f_trace = self.trace_dispatch + self.botframe = frame + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work + frame.f_trace_lines = True + frame = frame.f_back + self.set_stepinstr() sys.settrace(self.trace_dispatch) def set_continue(self): + """Stop only at breakpoints or when finished. + + If there are no breakpoints, set the system trace function to None. + """ + # Don't stop except at breakpoints or when finished self._set_stopinfo(self.botframe, None, -1) + if not self.breaks: + # no breakpoints; run without debugger overhead + sys.settrace(None) + frame = sys._getframe().f_back + while frame and frame is not self.botframe: + del frame.f_trace + frame = frame.f_back + for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): + frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + self.frame_trace_lines_opcodes = {} def set_quit(self): + """Set quitting attribute to True. + + Raises BdbQuit exception in the next call to a dispatch_*() method. + """ self.stopframe = self.botframe self.returnframe = None self.quitting = True sys.settrace(None) - # ---- breakpoints ----------------------------------------------------- + # Derived classes and clients can call the following methods + # to manipulate breakpoints. These methods return an + # error message if something went wrong, None if all is well. + # Set_break prints out the breakpoint line and file:lineno. + # Call self.get_*break*() to see the breakpoints or better + # for bp in Breakpoint.bpbynumber: if bp: bp.bpprint(). + + def _add_to_breaks(self, filename, lineno): + """Add breakpoint to breaks, if not already there.""" + bp_linenos = self.breaks.setdefault(filename, []) + if lineno not in bp_linenos: + bp_linenos.append(lineno) def set_break(self, filename, lineno, temporary=False, cond=None, - funcname=None): + funcname=None): + """Set a new breakpoint for filename:lineno. + + If lineno doesn't exist for the filename, return an error message. + The filename should be in canonical form. + """ filename = self.canonic(filename) - self.breaks.setdefault(filename, set()).add(lineno) - Breakpoint(filename, lineno, temporary, cond, funcname) + import linecache # Import as late as possible + line = linecache.getline(filename, lineno) + if not line: + return 'Line %s:%d does not exist' % (filename, lineno) + self._add_to_breaks(filename, lineno) + bp = Breakpoint(filename, lineno, temporary, cond, funcname) + # After we set a new breakpoint, we need to search through all frames + # and set f_trace to trace_dispatch if there could be a breakpoint in + # that frame. + frame = self.enterframe + while frame: + if self.break_anywhere(frame): + frame.f_trace = self.trace_dispatch + frame = frame.f_back + return None + + def _load_breaks(self): + """Apply all breakpoints (set in other instances) to this one. + + Populates this instance's breaks list from the Breakpoint class's + list, which can have breakpoints set by another Bdb instance. This + is necessary for interactive sessions to keep the breakpoints + active across multiple calls to run(). + """ + for (filename, lineno) in Breakpoint.bplist.keys(): + self._add_to_breaks(filename, lineno) + + def _prune_breaks(self, filename, lineno): + """Prune breakpoints for filename:lineno. + + A list of breakpoints is maintained in the Bdb instance and in + the Breakpoint class. If a breakpoint in the Bdb instance no + longer exists in the Breakpoint class, then it's removed from the + Bdb instance. + """ + if (filename, lineno) not in Breakpoint.bplist: + self.breaks[filename].remove(lineno) + if not self.breaks[filename]: + del self.breaks[filename] def clear_break(self, filename, lineno): + """Delete breakpoints for filename:lineno. + + If no breakpoints were set, return an error message. + """ filename = self.canonic(filename) if filename not in self.breaks: - return 'no breakpoints at {}:{}'.format(filename, lineno) + return 'There are no breakpoints in %s' % filename if lineno not in self.breaks[filename]: - return 'no breakpoint at {}:{}'.format(filename, lineno) - for bp in Breakpoint.bplist.get((filename, lineno), [])[:]: + return 'There is no breakpoint at %s:%d' % (filename, lineno) + # If there's only one bp in the list for that file,line + # pair, then remove the breaks entry + for bp in Breakpoint.bplist[filename, lineno][:]: bp.deleteMe() - if not Breakpoint.bplist.get((filename, lineno)): - self.breaks[filename].discard(lineno) - if not self.breaks[filename]: - del self.breaks[filename] + self._prune_breaks(filename, lineno) + return None + + def clear_bpbynumber(self, arg): + """Delete a breakpoint by its index in Breakpoint.bpbynumber. + + If arg is invalid, return an error message. + """ + try: + bp = self.get_bpbynumber(arg) + except ValueError as err: + return str(err) + bp.deleteMe() + self._prune_breaks(bp.file, bp.line) + return None + + def clear_all_file_breaks(self, filename): + """Delete all breakpoints in filename. + + If none were set, return an error message. + """ + filename = self.canonic(filename) + if filename not in self.breaks: + return 'There are no breakpoints in %s' % filename + for line in self.breaks[filename]: + blist = Breakpoint.bplist[filename, line] + for bp in blist: + bp.deleteMe() + del self.breaks[filename] return None def clear_all_breaks(self): - for filename in list(self.breaks): - for lineno in list(self.breaks[filename]): - self.clear_break(filename, lineno) - self.breaks.clear() + """Delete all existing breakpoints. + + If none were set, return an error message. + """ + if not self.breaks: + return 'There are no breakpoints' + for bp in Breakpoint.bpbynumber: + if bp: + bp.deleteMe() + self.breaks = {} + return None + + def get_bpbynumber(self, arg): + """Return a breakpoint by its index in Breakpoint.bybpnumber. + + For invalid arg values or if the breakpoint doesn't exist, + raise a ValueError. + """ + if not arg: + raise ValueError('Breakpoint number expected') + try: + number = int(arg) + except ValueError: + raise ValueError('Non-numeric breakpoint number %s' % arg) from None + try: + bp = Breakpoint.bpbynumber[number] + except IndexError: + raise ValueError('Breakpoint number %d out of range' % number) from None + if bp is None: + raise ValueError('Breakpoint %d already deleted' % number) + return bp + + def get_break(self, filename, lineno): + """Return True if there is a breakpoint for filename:lineno.""" + filename = self.canonic(filename) + return filename in self.breaks and \ + lineno in self.breaks[filename] def get_breaks(self, filename, lineno): + """Return all breakpoints for filename:lineno. + + If no breakpoints are set, return an empty list. + """ filename = self.canonic(filename) - return Breakpoint.bplist.get((filename, lineno), []) + return filename in self.breaks and \ + lineno in self.breaks[filename] and \ + Breakpoint.bplist[filename, lineno] or [] def get_file_breaks(self, filename): - return list(self.breaks.get(self.canonic(filename), [])) + """Return all lines with breakpoints for filename. - def get_all_breaks(self): - return [(f, l) for f, ls in self.breaks.items() for l in ls] + If no breakpoints are set, return an empty list. + """ + filename = self.canonic(filename) + if filename in self.breaks: + return self.breaks[filename] + else: + return [] - def do_clear(self, arg): - """Default implementation — pdb overrides this.""" - try: - num = int(arg) - except ValueError: - return - if 0 < num < len(Breakpoint.bpbynumber): - bp = Breakpoint.bpbynumber[num] - if bp is not None: - bp.deleteMe() + def get_all_breaks(self): + """Return all breakpoints that are set.""" + return self.breaks + + # Derived classes and clients can call the following method + # to get a data structure representing a stack trace. + + def get_stack(self, f, t): + """Return a list of (frame, lineno) in a stack trace and a size. + + List starts with original calling frame, if there is one. + Size may be number of frames above or below f. + """ + stack = [] + if t and t.tb_frame is f: + t = t.tb_next + while f is not None: + stack.append((f, f.f_lineno)) + if f is self.botframe: + break + f = f.f_back + stack.reverse() + i = max(0, len(stack) - 1) + while t is not None: + stack.append((t.tb_frame, t.tb_lineno)) + t = t.tb_next + if f is None: + i = max(0, len(stack) - 1) + return stack, i + + def format_stack_entry(self, frame_lineno, lprefix=': '): + """Return a string with information about a stack entry. + + The stack entry frame_lineno is a (frame, lineno) tuple. The + return string contains the canonical filename, the function name + or '', the input arguments, the return value, and the + line of code (if it exists). + + """ + import linecache, reprlib + frame, lineno = frame_lineno + filename = self.canonic(frame.f_code.co_filename) + s = '%s(%r)' % (filename, lineno) + if frame.f_code.co_name: + s += frame.f_code.co_name + else: + s += "" + s += '()' + if '__return__' in frame.f_locals: + rv = frame.f_locals['__return__'] + s += '->' + s += reprlib.repr(rv) + if lineno is not None: + line = linecache.getline(filename, lineno, frame.f_globals) + if line: + s += lprefix + line.strip() + else: + s += f'{lprefix}Warning: lineno is None' + return s + + # The following methods can be called by clients to use + # a debugger to debug a statement or an expression. + # Both can be given as a string, or a code object. - # ---- run helpers ----------------------------------------------------- + def run(self, cmd, globals=None, locals=None): + """Debug a statement executed via the exec() function. - def runcall(self, func, *args, **kwds): + globals defaults to __main__.dict; locals defaults to globals. + """ + if globals is None: + import __main__ + globals = __main__.__dict__ + if locals is None: + locals = globals self.reset() + if isinstance(cmd, str): + cmd = compile(cmd, "", "exec") sys.settrace(self.trace_dispatch) try: - return func(*args, **kwds) + exec(cmd, globals, locals) + except BdbQuit: + pass finally: + self.quitting = True sys.settrace(None) - def run(self, cmd, globals=None, locals=None): + def runeval(self, expr, globals=None, locals=None): + """Debug an expression executed via the eval() function. + + globals defaults to __main__.dict; locals defaults to globals. + """ if globals is None: - globals = {'__name__': '__main__'} + import __main__ + globals = __main__.__dict__ if locals is None: locals = globals self.reset() - if isinstance(cmd, str): - cmd = compile(cmd, '', 'exec') sys.settrace(self.trace_dispatch) try: - exec(cmd, globals, locals) + return eval(expr, globals, locals) + except BdbQuit: + pass + finally: + self.quitting = True + sys.settrace(None) + + def runctx(self, cmd, globals, locals): + """For backwards-compatibility. Defers to run().""" + # B/W compatibility + self.run(cmd, globals, locals) + + # This method is more useful to debug a single function call. + + def runcall(self, func, /, *args, **kwds): + """Debug a single function call. + + Return the result of the function call. + """ + self.reset() + sys.settrace(self.trace_dispatch) + res = None + try: + res = func(*args, **kwds) except BdbQuit: pass finally: self.quitting = True sys.settrace(None) + return res def set_trace(): - """Standalone helper — drop into the default pdb.""" - import pdb - pdb.set_trace() + """Start debugging with a Bdb instance from the caller's frame.""" + Bdb().set_trace() + + +class Breakpoint: + """Breakpoint class. + + Implements temporary breakpoints, ignore counts, disabling and + (re)-enabling, and conditionals. + + Breakpoints are indexed by number through bpbynumber and by + the (file, line) tuple using bplist. The former points to a + single instance of class Breakpoint. The latter points to a + list of such instances since there may be more than one + breakpoint per line. + + When creating a breakpoint, its associated filename should be + in canonical form. If funcname is defined, a breakpoint hit will be + counted when the first line of that function is executed. A + conditional breakpoint always counts a hit. + """ + + # XXX Keeping state in the class is a mistake -- this means + # you cannot have more than one active Bdb instance. + + next = 1 # Next bp to be assigned + bplist = {} # indexed by (file, lineno) tuple + bpbynumber = [None] # Each entry is None or an instance of Bpt + # index 0 is unused, except for marking an + # effective break .... see effective() + + def __init__(self, file, line, temporary=False, cond=None, funcname=None): + self.funcname = funcname + # Needed if funcname is not None. + self.func_first_executable_line = None + self.file = file # This better be in canonical form! + self.line = line + self.temporary = temporary + self.cond = cond + self.enabled = True + self.ignore = 0 + self.hits = 0 + self.number = Breakpoint.next + Breakpoint.next += 1 + # Build the two lists + self.bpbynumber.append(self) + if (file, line) in self.bplist: + self.bplist[file, line].append(self) + else: + self.bplist[file, line] = [self] + + @staticmethod + def clearBreakpoints(): + Breakpoint.next = 1 + Breakpoint.bplist = {} + Breakpoint.bpbynumber = [None] + + def deleteMe(self): + """Delete the breakpoint from the list associated to a file:line. + + If it is the last breakpoint in that position, it also deletes + the entry for the file:line. + """ + + index = (self.file, self.line) + self.bpbynumber[self.number] = None # No longer in list + self.bplist[index].remove(self) + if not self.bplist[index]: + # No more bp for this f:l combo + del self.bplist[index] + + def enable(self): + """Mark the breakpoint as enabled.""" + self.enabled = True + + def disable(self): + """Mark the breakpoint as disabled.""" + self.enabled = False + + def bpprint(self, out=None): + """Print the output of bpformat(). + + The optional out argument directs where the output is sent + and defaults to standard output. + """ + if out is None: + out = sys.stdout + print(self.bpformat(), file=out) + + def bpformat(self): + """Return a string with information about the breakpoint. + + The information includes the breakpoint number, temporary + status, file:line position, break condition, number of times to + ignore, and number of times hit. + + """ + if self.temporary: + disp = 'del ' + else: + disp = 'keep ' + if self.enabled: + disp = disp + 'yes ' + else: + disp = disp + 'no ' + ret = '%-4dbreakpoint %s at %s:%d' % (self.number, disp, + self.file, self.line) + if self.cond: + ret += '\n\tstop only if %s' % (self.cond,) + if self.ignore: + ret += '\n\tignore next %d hits' % (self.ignore,) + if self.hits: + if self.hits > 1: + ss = 's' + else: + ss = '' + ret += '\n\tbreakpoint already hit %d time%s' % (self.hits, ss) + return ret + + def __str__(self): + "Return a condensed description of the breakpoint." + return 'breakpoint %s at %s:%s' % (self.number, self.file, self.line) + +# -----------end of Breakpoint class---------- + + +def checkfuncname(b, frame): + """Return True if break should happen here. + + Whether a break should happen depends on the way that b (the breakpoint) + was set. If it was set via line number, check if b.line is the same as + the one in the frame. If it was set via function name, check if this is + the right function and if it is on the first executable line. + """ + if not b.funcname: + # Breakpoint was set via line number. + if b.line != frame.f_lineno: + # Breakpoint was set at a line with a def statement and the function + # defined is called: don't break. + return False + return True + + # Breakpoint set via function name. + if frame.f_code.co_name != b.funcname: + # It's not a function call, but rather execution of def statement. + return False + + # We are in the right frame. + if not b.func_first_executable_line: + # The function is entered for the 1st time. + b.func_first_executable_line = frame.f_lineno + + if b.func_first_executable_line != frame.f_lineno: + # But we are not at the first line number: don't break. + return False + return True + + +def effective(file, line, frame): + """Return (active breakpoint, delete temporary flag) or (None, None) as + breakpoint to act upon. + + The "active breakpoint" is the first entry in bplist[line, file] (which + must exist) that is enabled, for which checkfuncname is True, and that + has neither a False condition nor a positive ignore count. The flag, + meaning that a temporary breakpoint should be deleted, is False only + when the condiion cannot be evaluated (in which case, ignore count is + ignored). + + If no such entry exists, then (None, None) is returned. + """ + possibles = Breakpoint.bplist[file, line] + for b in possibles: + if not b.enabled: + continue + if not checkfuncname(b, frame): + continue + # Count every hit when bp is enabled + b.hits += 1 + if not b.cond: + # If unconditional, and ignoring go on to next, else break + if b.ignore > 0: + b.ignore -= 1 + continue + else: + # breakpoint and marker that it's ok to delete if temporary + return (b, True) + else: + # Conditional bp. + # Ignore count applies only to those bpt hits where the + # condition evaluates to true. + try: + val = eval(b.cond, frame.f_globals, frame.f_locals) + if val: + if b.ignore > 0: + b.ignore -= 1 + # continue + else: + return (b, True) + # else: + # continue + except: + # if eval fails, most conservative thing is to stop on + # breakpoint regardless of ignore count. Don't delete + # temporary, as another hint to user. + return (b, False) + return (None, None) + + +# -------------------- testing -------------------- + +class Tdb(Bdb): + def user_call(self, frame, args): + name = frame.f_code.co_name + if not name: name = '???' + print('+++ call', name, args) + def user_line(self, frame): + import linecache + name = frame.f_code.co_name + if not name: name = '???' + fn = self.canonic(frame.f_code.co_filename) + line = linecache.getline(fn, frame.f_lineno, frame.f_globals) + print('+++', fn, frame.f_lineno, name, ':', line.strip()) + def user_return(self, frame, retval): + print('+++ return', retval) + def user_exception(self, frame, exc_stuff): + print('+++ exception', exc_stuff) + self.set_continue() + +def foo(n): + print('foo(', n, ')') + x = bar(n*10) + print('bar returned', x) + +def bar(a): + print('bar(', a, ')') + return a/2 + +def test(): + t = Tdb() + t.run('import bdb; bdb.foo(10)') diff --git a/crates/weavepy-vm/src/stdlib/python/calendar.py b/crates/weavepy-vm/src/stdlib/python/calendar.py new file mode 100644 index 0000000..8c1c646 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/calendar.py @@ -0,0 +1,813 @@ +"""Calendar printing functions + +Note when comparing these calendars to the ones printed by cal(1): By +default, these calendars have Monday as the first day of the week, and +Sunday as the last (the European convention). Use setfirstweekday() to +set the first day of the week (0=Monday, 6=Sunday).""" + +import sys +import datetime +from enum import IntEnum, global_enum +import locale as _locale +from itertools import repeat + +__all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", + "firstweekday", "isleap", "leapdays", "weekday", "monthrange", + "monthcalendar", "prmonth", "month", "prcal", "calendar", + "timegm", "month_name", "month_abbr", "day_name", "day_abbr", + "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", + "LocaleHTMLCalendar", "weekheader", + "Day", "Month", "JANUARY", "FEBRUARY", "MARCH", + "APRIL", "MAY", "JUNE", "JULY", + "AUGUST", "SEPTEMBER", "OCTOBER", "NOVEMBER", "DECEMBER", + "MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", + "SATURDAY", "SUNDAY"] + +# Exception raised for bad input (with string parameter for details) +error = ValueError + +# Exceptions raised for bad input +# This is trick for backward compatibility. Since 3.13, we will raise IllegalMonthError instead of +# IndexError for bad month number(out of 1-12). But we can't remove IndexError for backward compatibility. +class IllegalMonthError(ValueError, IndexError): + def __init__(self, month): + self.month = month + def __str__(self): + return "bad month number %r; must be 1-12" % self.month + + +class IllegalWeekdayError(ValueError): + def __init__(self, weekday): + self.weekday = weekday + def __str__(self): + return "bad weekday number %r; must be 0 (Monday) to 6 (Sunday)" % self.weekday + + +def __getattr__(name): + if name in ('January', 'February'): + import warnings + warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead", + DeprecationWarning, stacklevel=2) + if name == 'January': + return 1 + else: + return 2 + + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") + + +# Constants for months +@global_enum +class Month(IntEnum): + JANUARY = 1 + FEBRUARY = 2 + MARCH = 3 + APRIL = 4 + MAY = 5 + JUNE = 6 + JULY = 7 + AUGUST = 8 + SEPTEMBER = 9 + OCTOBER = 10 + NOVEMBER = 11 + DECEMBER = 12 + + +# Constants for days +@global_enum +class Day(IntEnum): + MONDAY = 0 + TUESDAY = 1 + WEDNESDAY = 2 + THURSDAY = 3 + FRIDAY = 4 + SATURDAY = 5 + SUNDAY = 6 + + +# Number of days per month (except for February in leap years) +mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + +# This module used to have hard-coded lists of day and month names, as +# English strings. The classes following emulate a read-only version of +# that, but supply localized names. Note that the values are computed +# fresh on each call, in case the user changes locale between calls. + +class _localized_month: + + _months = [datetime.date(2001, i+1, 1).strftime for i in range(12)] + _months.insert(0, lambda x: "") + + def __init__(self, format): + self.format = format + + def __getitem__(self, i): + funcs = self._months[i] + if isinstance(i, slice): + return [f(self.format) for f in funcs] + else: + return funcs(self.format) + + def __len__(self): + return 13 + + +class _localized_day: + + # January 1, 2001, was a Monday. + _days = [datetime.date(2001, 1, i+1).strftime for i in range(7)] + + def __init__(self, format): + self.format = format + + def __getitem__(self, i): + funcs = self._days[i] + if isinstance(i, slice): + return [f(self.format) for f in funcs] + else: + return funcs(self.format) + + def __len__(self): + return 7 + + +# Full and abbreviated names of weekdays +day_name = _localized_day('%A') +day_abbr = _localized_day('%a') + +# Full and abbreviated names of months (1-based arrays!!!) +month_name = _localized_month('%B') +month_abbr = _localized_month('%b') + + +def isleap(year): + """Return True for leap years, False for non-leap years.""" + return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0) + + +def leapdays(y1, y2): + """Return number of leap years in range [y1, y2). + Assume y1 <= y2.""" + y1 -= 1 + y2 -= 1 + return (y2//4 - y1//4) - (y2//100 - y1//100) + (y2//400 - y1//400) + + +def weekday(year, month, day): + """Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31).""" + if not datetime.MINYEAR <= year <= datetime.MAXYEAR: + year = 2000 + year % 400 + return Day(datetime.date(year, month, day).weekday()) + + +def _validate_month(month): + if not 1 <= month <= 12: + raise IllegalMonthError(month) + +def monthrange(year, month): + """Return weekday of first day of month (0-6 ~ Mon-Sun) + and number of days (28-31) for year, month.""" + _validate_month(month) + day1 = weekday(year, month, 1) + ndays = mdays[month] + (month == FEBRUARY and isleap(year)) + return day1, ndays + + +def _monthlen(year, month): + return mdays[month] + (month == FEBRUARY and isleap(year)) + + +def _prevmonth(year, month): + if month == 1: + return year-1, 12 + else: + return year, month-1 + + +def _nextmonth(year, month): + if month == 12: + return year+1, 1 + else: + return year, month+1 + + +class Calendar(object): + """ + Base calendar class. This class doesn't do any formatting. It simply + provides data to subclasses. + """ + + def __init__(self, firstweekday=0): + self.firstweekday = firstweekday # 0 = Monday, 6 = Sunday + + def getfirstweekday(self): + return self._firstweekday % 7 + + def setfirstweekday(self, firstweekday): + self._firstweekday = firstweekday + + firstweekday = property(getfirstweekday, setfirstweekday) + + def iterweekdays(self): + """ + Return an iterator for one week of weekday numbers starting with the + configured first one. + """ + for i in range(self.firstweekday, self.firstweekday + 7): + yield i%7 + + def itermonthdates(self, year, month): + """ + Return an iterator for one month. The iterator will yield datetime.date + values and will always iterate through complete weeks, so it will yield + dates outside the specified month. + """ + for y, m, d in self.itermonthdays3(year, month): + yield datetime.date(y, m, d) + + def itermonthdays(self, year, month): + """ + Like itermonthdates(), but will yield day numbers. For days outside + the specified month the day number is 0. + """ + day1, ndays = monthrange(year, month) + days_before = (day1 - self.firstweekday) % 7 + yield from repeat(0, days_before) + yield from range(1, ndays + 1) + days_after = (self.firstweekday - day1 - ndays) % 7 + yield from repeat(0, days_after) + + def itermonthdays2(self, year, month): + """ + Like itermonthdates(), but will yield (day number, weekday number) + tuples. For days outside the specified month the day number is 0. + """ + for i, d in enumerate(self.itermonthdays(year, month), self.firstweekday): + yield d, i % 7 + + def itermonthdays3(self, year, month): + """ + Like itermonthdates(), but will yield (year, month, day) tuples. Can be + used for dates outside of datetime.date range. + """ + day1, ndays = monthrange(year, month) + days_before = (day1 - self.firstweekday) % 7 + days_after = (self.firstweekday - day1 - ndays) % 7 + y, m = _prevmonth(year, month) + end = _monthlen(y, m) + 1 + for d in range(end-days_before, end): + yield y, m, d + for d in range(1, ndays + 1): + yield year, month, d + y, m = _nextmonth(year, month) + for d in range(1, days_after + 1): + yield y, m, d + + def itermonthdays4(self, year, month): + """ + Like itermonthdates(), but will yield (year, month, day, day_of_week) tuples. + Can be used for dates outside of datetime.date range. + """ + for i, (y, m, d) in enumerate(self.itermonthdays3(year, month)): + yield y, m, d, (self.firstweekday + i) % 7 + + def monthdatescalendar(self, year, month): + """ + Return a matrix (list of lists) representing a month's calendar. + Each row represents a week; week entries are datetime.date values. + """ + dates = list(self.itermonthdates(year, month)) + return [ dates[i:i+7] for i in range(0, len(dates), 7) ] + + def monthdays2calendar(self, year, month): + """ + Return a matrix representing a month's calendar. + Each row represents a week; week entries are + (day number, weekday number) tuples. Day numbers outside this month + are zero. + """ + days = list(self.itermonthdays2(year, month)) + return [ days[i:i+7] for i in range(0, len(days), 7) ] + + def monthdayscalendar(self, year, month): + """ + Return a matrix representing a month's calendar. + Each row represents a week; days outside this month are zero. + """ + days = list(self.itermonthdays(year, month)) + return [ days[i:i+7] for i in range(0, len(days), 7) ] + + def yeardatescalendar(self, year, width=3): + """ + Return the data for the specified year ready for formatting. The return + value is a list of month rows. Each month row contains up to width months. + Each month contains between 4 and 6 weeks and each week contains 1-7 + days. Days are datetime.date objects. + """ + months = [self.monthdatescalendar(year, m) for m in Month] + return [months[i:i+width] for i in range(0, len(months), width) ] + + def yeardays2calendar(self, year, width=3): + """ + Return the data for the specified year ready for formatting (similar to + yeardatescalendar()). Entries in the week lists are + (day number, weekday number) tuples. Day numbers outside this month are + zero. + """ + months = [self.monthdays2calendar(year, m) for m in Month] + return [months[i:i+width] for i in range(0, len(months), width) ] + + def yeardayscalendar(self, year, width=3): + """ + Return the data for the specified year ready for formatting (similar to + yeardatescalendar()). Entries in the week lists are day numbers. + Day numbers outside this month are zero. + """ + months = [self.monthdayscalendar(year, m) for m in Month] + return [months[i:i+width] for i in range(0, len(months), width) ] + + +class TextCalendar(Calendar): + """ + Subclass of Calendar that outputs a calendar as a simple plain text + similar to the UNIX program cal. + """ + + def prweek(self, theweek, width): + """ + Print a single week (no newline). + """ + print(self.formatweek(theweek, width), end='') + + def formatday(self, day, weekday, width): + """ + Returns a formatted day. + """ + if day == 0: + s = '' + else: + s = '%2i' % day # right-align single-digit days + return s.center(width) + + def formatweek(self, theweek, width): + """ + Returns a single week in a string (no newline). + """ + return ' '.join(self.formatday(d, wd, width) for (d, wd) in theweek) + + def formatweekday(self, day, width): + """ + Returns a formatted week day name. + """ + if width >= 9: + names = day_name + else: + names = day_abbr + return names[day][:width].center(width) + + def formatweekheader(self, width): + """ + Return a header for a week. + """ + return ' '.join(self.formatweekday(i, width) for i in self.iterweekdays()) + + def formatmonthname(self, theyear, themonth, width, withyear=True): + """ + Return a formatted month name. + """ + _validate_month(themonth) + + s = month_name[themonth] + if withyear: + s = "%s %r" % (s, theyear) + return s.center(width) + + def prmonth(self, theyear, themonth, w=0, l=0): + """ + Print a month's calendar. + """ + print(self.formatmonth(theyear, themonth, w, l), end='') + + def formatmonth(self, theyear, themonth, w=0, l=0): + """ + Return a month's calendar string (multi-line). + """ + w = max(2, w) + l = max(1, l) + s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1) + s = s.rstrip() + s += '\n' * l + s += self.formatweekheader(w).rstrip() + s += '\n' * l + for week in self.monthdays2calendar(theyear, themonth): + s += self.formatweek(week, w).rstrip() + s += '\n' * l + return s + + def formatyear(self, theyear, w=2, l=1, c=6, m=3): + """ + Returns a year's calendar as a multi-line string. + """ + w = max(2, w) + l = max(1, l) + c = max(2, c) + colwidth = (w + 1) * 7 - 1 + v = [] + a = v.append + a(repr(theyear).center(colwidth*m+c*(m-1)).rstrip()) + a('\n'*l) + header = self.formatweekheader(w) + for (i, row) in enumerate(self.yeardays2calendar(theyear, m)): + # months in this row + months = range(m*i+1, min(m*(i+1)+1, 13)) + a('\n'*l) + names = (self.formatmonthname(theyear, k, colwidth, False) + for k in months) + a(formatstring(names, colwidth, c).rstrip()) + a('\n'*l) + headers = (header for k in months) + a(formatstring(headers, colwidth, c).rstrip()) + a('\n'*l) + # max number of weeks for this row + height = max(len(cal) for cal in row) + for j in range(height): + weeks = [] + for cal in row: + if j >= len(cal): + weeks.append('') + else: + weeks.append(self.formatweek(cal[j], w)) + a(formatstring(weeks, colwidth, c).rstrip()) + a('\n' * l) + return ''.join(v) + + def pryear(self, theyear, w=0, l=0, c=6, m=3): + """Print a year's calendar.""" + print(self.formatyear(theyear, w, l, c, m), end='') + + +class HTMLCalendar(Calendar): + """ + This calendar returns complete HTML pages. + """ + + # CSS classes for the day s + cssclasses = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] + + # CSS classes for the day s + cssclasses_weekday_head = cssclasses + + # CSS class for the days before and after current month + cssclass_noday = "noday" + + # CSS class for the month's head + cssclass_month_head = "month" + + # CSS class for the month + cssclass_month = "month" + + # CSS class for the year's table head + cssclass_year_head = "year" + + # CSS class for the whole year table + cssclass_year = "year" + + def formatday(self, day, weekday): + """ + Return a day as a table cell. + """ + if day == 0: + # day outside month + return ' ' % self.cssclass_noday + else: + return '%d' % (self.cssclasses[weekday], day) + + def formatweek(self, theweek): + """ + Return a complete week as a table row. + """ + s = ''.join(self.formatday(d, wd) for (d, wd) in theweek) + return '%s' % s + + def formatweekday(self, day): + """ + Return a weekday name as a table header. + """ + return '%s' % ( + self.cssclasses_weekday_head[day], day_abbr[day]) + + def formatweekheader(self): + """ + Return a header for a week as a table row. + """ + s = ''.join(self.formatweekday(i) for i in self.iterweekdays()) + return '%s' % s + + def formatmonthname(self, theyear, themonth, withyear=True): + """ + Return a month name as a table row. + """ + _validate_month(themonth) + if withyear: + s = '%s %s' % (month_name[themonth], theyear) + else: + s = '%s' % month_name[themonth] + return '%s' % ( + self.cssclass_month_head, s) + + def formatmonth(self, theyear, themonth, withyear=True): + """ + Return a formatted month as a table. + """ + v = [] + a = v.append + a('' % ( + self.cssclass_month)) + a('\n') + a(self.formatmonthname(theyear, themonth, withyear=withyear)) + a('\n') + a(self.formatweekheader()) + a('\n') + for week in self.monthdays2calendar(theyear, themonth): + a(self.formatweek(week)) + a('\n') + a('
') + a('\n') + return ''.join(v) + + def formatyear(self, theyear, width=3): + """ + Return a formatted year as a table of tables. + """ + v = [] + a = v.append + width = max(width, 1) + a('' % + self.cssclass_year) + a('\n') + a('' % ( + width, self.cssclass_year_head, theyear)) + for i in range(JANUARY, JANUARY+12, width): + # months in this row + months = range(i, min(i+width, 13)) + a('') + for m in months: + a('') + a('') + a('
%s
') + a(self.formatmonth(theyear, m, withyear=False)) + a('
') + return ''.join(v) + + def formatyearpage(self, theyear, width=3, css='calendar.css', encoding=None): + """ + Return a formatted year as a complete HTML page. + """ + if encoding is None: + encoding = sys.getdefaultencoding() + v = [] + a = v.append + a('\n' % encoding) + a('\n') + a('\n') + a('\n') + a('\n' % encoding) + if css is not None: + a('\n' % css) + a('Calendar for %d\n' % theyear) + a('\n') + a('\n') + a(self.formatyear(theyear, width)) + a('\n') + a('\n') + return ''.join(v).encode(encoding, "xmlcharrefreplace") + + +class different_locale: + def __init__(self, locale): + self.locale = locale + self.oldlocale = None + + def __enter__(self): + self.oldlocale = _locale.setlocale(_locale.LC_TIME, None) + _locale.setlocale(_locale.LC_TIME, self.locale) + + def __exit__(self, *args): + _locale.setlocale(_locale.LC_TIME, self.oldlocale) + + +def _get_default_locale(): + locale = _locale.setlocale(_locale.LC_TIME, None) + if locale == "C": + with different_locale(""): + # The LC_TIME locale does not seem to be configured: + # get the user preferred locale. + locale = _locale.setlocale(_locale.LC_TIME, None) + return locale + + +class LocaleTextCalendar(TextCalendar): + """ + This class can be passed a locale name in the constructor and will return + month and weekday names in the specified locale. + """ + + def __init__(self, firstweekday=0, locale=None): + TextCalendar.__init__(self, firstweekday) + if locale is None: + locale = _get_default_locale() + self.locale = locale + + def formatweekday(self, day, width): + with different_locale(self.locale): + return super().formatweekday(day, width) + + def formatmonthname(self, theyear, themonth, width, withyear=True): + with different_locale(self.locale): + return super().formatmonthname(theyear, themonth, width, withyear) + + +class LocaleHTMLCalendar(HTMLCalendar): + """ + This class can be passed a locale name in the constructor and will return + month and weekday names in the specified locale. + """ + def __init__(self, firstweekday=0, locale=None): + HTMLCalendar.__init__(self, firstweekday) + if locale is None: + locale = _get_default_locale() + self.locale = locale + + def formatweekday(self, day): + with different_locale(self.locale): + return super().formatweekday(day) + + def formatmonthname(self, theyear, themonth, withyear=True): + with different_locale(self.locale): + return super().formatmonthname(theyear, themonth, withyear) + +# Support for old module level interface +c = TextCalendar() + +firstweekday = c.getfirstweekday + +def setfirstweekday(firstweekday): + if not MONDAY <= firstweekday <= SUNDAY: + raise IllegalWeekdayError(firstweekday) + c.firstweekday = firstweekday + +monthcalendar = c.monthdayscalendar +prweek = c.prweek +week = c.formatweek +weekheader = c.formatweekheader +prmonth = c.prmonth +month = c.formatmonth +calendar = c.formatyear +prcal = c.pryear + + +# Spacing of month columns for multi-column year calendar +_colwidth = 7*3 - 1 # Amount printed by prweek() +_spacing = 6 # Number of spaces between columns + + +def format(cols, colwidth=_colwidth, spacing=_spacing): + """Prints multi-column formatting for year calendars""" + print(formatstring(cols, colwidth, spacing)) + + +def formatstring(cols, colwidth=_colwidth, spacing=_spacing): + """Returns a string formatted from n strings, centered within n columns.""" + spacing *= ' ' + return spacing.join(c.center(colwidth) for c in cols) + + +EPOCH = 1970 +_EPOCH_ORD = datetime.date(EPOCH, 1, 1).toordinal() + + +def timegm(tuple): + """Unrelated but handy function to calculate Unix timestamp from GMT.""" + year, month, day, hour, minute, second = tuple[:6] + days = datetime.date(year, month, 1).toordinal() - _EPOCH_ORD + day - 1 + hours = days*24 + hour + minutes = hours*60 + minute + seconds = minutes*60 + second + return seconds + + +def main(args=None): + import argparse + parser = argparse.ArgumentParser() + textgroup = parser.add_argument_group('text only arguments') + htmlgroup = parser.add_argument_group('html only arguments') + textgroup.add_argument( + "-w", "--width", + type=int, default=2, + help="width of date column (default 2)" + ) + textgroup.add_argument( + "-l", "--lines", + type=int, default=1, + help="number of lines for each week (default 1)" + ) + textgroup.add_argument( + "-s", "--spacing", + type=int, default=6, + help="spacing between months (default 6)" + ) + textgroup.add_argument( + "-m", "--months", + type=int, default=3, + help="months per row (default 3)" + ) + htmlgroup.add_argument( + "-c", "--css", + default="calendar.css", + help="CSS to use for page" + ) + parser.add_argument( + "-L", "--locale", + default=None, + help="locale to use for month and weekday names" + ) + parser.add_argument( + "-e", "--encoding", + default=None, + help="encoding to use for output" + ) + parser.add_argument( + "-t", "--type", + default="text", + choices=("text", "html"), + help="output type (text or html)" + ) + parser.add_argument( + "-f", "--first-weekday", + type=int, default=0, + help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)" + ) + parser.add_argument( + "year", + nargs='?', type=int, + help="year number" + ) + parser.add_argument( + "month", + nargs='?', type=int, + help="month number (1-12, text only)" + ) + + options = parser.parse_args(args) + + if options.locale and not options.encoding: + parser.error("if --locale is specified --encoding is required") + sys.exit(1) + + locale = options.locale, options.encoding + + if options.type == "html": + if options.month: + parser.error("incorrect number of arguments") + sys.exit(1) + if options.locale: + cal = LocaleHTMLCalendar(locale=locale) + else: + cal = HTMLCalendar() + cal.setfirstweekday(options.first_weekday) + encoding = options.encoding + if encoding is None: + encoding = sys.getdefaultencoding() + optdict = dict(encoding=encoding, css=options.css) + write = sys.stdout.buffer.write + if options.year is None: + write(cal.formatyearpage(datetime.date.today().year, **optdict)) + else: + write(cal.formatyearpage(options.year, **optdict)) + else: + if options.locale: + cal = LocaleTextCalendar(locale=locale) + else: + cal = TextCalendar() + cal.setfirstweekday(options.first_weekday) + optdict = dict(w=options.width, l=options.lines) + if options.month is None: + optdict["c"] = options.spacing + optdict["m"] = options.months + if options.month is not None: + _validate_month(options.month) + if options.year is None: + result = cal.formatyear(datetime.date.today().year, **optdict) + elif options.month is None: + result = cal.formatyear(options.year, **optdict) + else: + result = cal.formatmonth(options.year, options.month, **optdict) + write = sys.stdout.write + if options.encoding: + result = result.encode(options.encoding) + write = sys.stdout.buffer.write + write(result) + + +if __name__ == "__main__": + main() diff --git a/crates/weavepy-vm/src/stdlib/python/cmath.py b/crates/weavepy-vm/src/stdlib/python/cmath.py new file mode 100644 index 0000000..a27145b --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/cmath.py @@ -0,0 +1,174 @@ +"""Faithful pure-Python ``cmath`` over the native ``math`` core. + +CPython ships ``cmath`` as a C module (``Modules/cmathmodule.c``); WeavePy +provides a Python implementation that computes the same principal-branch +values via real :mod:`math` primitives and complex arithmetic. The public +surface (constants + functions) matches CPython 3.13 (RFC 0037 WS8). +""" + +import math as _math + +pi = _math.pi +e = _math.e +tau = _math.tau +inf = _math.inf +nan = _math.nan +infj = complex(0.0, _math.inf) +nanj = complex(0.0, _math.nan) + +__all__ = [ + "pi", "e", "tau", "inf", "nan", "infj", "nanj", + "phase", "polar", "rect", + "exp", "log", "log10", "sqrt", + "acos", "asin", "atan", "cos", "sin", "tan", + "acosh", "asinh", "atanh", "cosh", "sinh", "tanh", + "isfinite", "isinf", "isnan", "isclose", +] + + +def _c(z): + """Coerce ``z`` to ``complex`` (accepting ints/floats and objects with + ``__complex__``/``__float__``/``__index__``), matching cmath's argument + handling.""" + if isinstance(z, complex): + return z + return complex(z) + + +def phase(z): + z = _c(z) + return _math.atan2(z.imag, z.real) + + +def polar(z): + z = _c(z) + return (abs(z), _math.atan2(z.imag, z.real)) + + +def rect(r, phi): + r = float(r) + phi = float(phi) + # Mirror CPython's special handling so rect(r, 0) keeps the sign of an + # infinite/zero r on the real axis with a clean zero imaginary part. + if phi == 0.0: + return complex(r, 0.0 * r) + return complex(r * _math.cos(phi), r * _math.sin(phi)) + + +def isfinite(z): + z = _c(z) + return _math.isfinite(z.real) and _math.isfinite(z.imag) + + +def isinf(z): + z = _c(z) + return _math.isinf(z.real) or _math.isinf(z.imag) + + +def isnan(z): + z = _c(z) + return _math.isnan(z.real) or _math.isnan(z.imag) + + +def isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0): + a = _c(a) + b = _c(b) + if rel_tol < 0.0 or abs_tol < 0.0: + raise ValueError("tolerances must be non-negative") + if a == b: + return True + if isinf(a) or isinf(b): + return False + diff = abs(a - b) + return (diff <= abs(rel_tol * b)) or (diff <= abs(rel_tol * a)) or (diff <= abs_tol) + + +def exp(z): + z = _c(z) + r = _math.exp(z.real) + return complex(r * _math.cos(z.imag), r * _math.sin(z.imag)) + + +def log(z, base=None): + z = _c(z) + if base is not None: + return log(z) / log(base) + return complex(_math.log(abs(z)), _math.atan2(z.imag, z.real)) + + +def log10(z): + return log(z) / _math.log(10.0) + + +def sqrt(z): + z = _c(z) + if z.imag == 0.0 and z.real >= 0.0: + return complex(_math.sqrt(z.real), 0.0) + r = abs(z) + ang = _math.atan2(z.imag, z.real) / 2.0 + m = _math.sqrt(r) + return complex(m * _math.cos(ang), m * _math.sin(ang)) + + +def cos(z): + z = _c(z) + return complex(_math.cos(z.real) * _math.cosh(z.imag), + -_math.sin(z.real) * _math.sinh(z.imag)) + + +def sin(z): + z = _c(z) + return complex(_math.sin(z.real) * _math.cosh(z.imag), + _math.cos(z.real) * _math.sinh(z.imag)) + + +def tan(z): + z = _c(z) + return sin(z) / cos(z) + + +def cosh(z): + z = _c(z) + return complex(_math.cosh(z.real) * _math.cos(z.imag), + _math.sinh(z.real) * _math.sin(z.imag)) + + +def sinh(z): + z = _c(z) + return complex(_math.sinh(z.real) * _math.cos(z.imag), + _math.cosh(z.real) * _math.sin(z.imag)) + + +def tanh(z): + z = _c(z) + return sinh(z) / cosh(z) + + +def asin(z): + z = _c(z) + return -1j * log(1j * z + sqrt(1 - z * z)) + + +def acos(z): + z = _c(z) + return -1j * log(z + 1j * sqrt(1 - z * z)) + + +def atan(z): + z = _c(z) + return (1j / 2) * (log(1 - 1j * z) - log(1 + 1j * z)) + + +def asinh(z): + z = _c(z) + return log(z + sqrt(z * z + 1)) + + +def acosh(z): + z = _c(z) + return log(z + sqrt(z - 1) * sqrt(z + 1)) + + +def atanh(z): + z = _c(z) + return (log(1 + z) - log(1 - z)) / 2 diff --git a/crates/weavepy-vm/src/stdlib/python/collections.py b/crates/weavepy-vm/src/stdlib/python/collections.py index ca48fe4..334402d 100644 --- a/crates/weavepy-vm/src/stdlib/python/collections.py +++ b/crates/weavepy-vm/src/stdlib/python/collections.py @@ -16,8 +16,16 @@ "Counter", "ChainMap", "namedtuple", + "UserDict", + "UserList", + "UserString", ] +# `UserDict`/`UserList`/`UserString` are verbatim CPython and depend on +# `collections.abc`, so they live in a sibling frozen module (imported at +# the end of this file, after the package is otherwise initialised) to keep +# the import graph acyclic. + def _count_elements(mapping, iterable): """Tally elements from the iterable. @@ -507,7 +515,7 @@ def __repr__(self): return "ChainMap(" + ", ".join(repr(m) for m in self.maps) + ")" -def namedtuple(typename, field_names, *, rename=False, defaults=None): +def namedtuple(typename, field_names, *, rename=False, defaults=None, module=None): """Return a new lightweight class with the given fields. The result mirrors CPython's ``namedtuple`` API surface — iteration, @@ -586,12 +594,20 @@ def _asdict(self): def _replace(self, **changes): values = list(self._values) - for k, v in changes.items(): - if k not in field_names: - raise ValueError("unknown field: " + k) - values[field_names.index(k)] = v + for i, name in enumerate(field_names): + if name in changes: + values[i] = changes.pop(name) + if changes: + # Match CPython: leftover keys are reported as a TypeError + # ("Got unexpected field names: [...]"). + raise TypeError( + "Got unexpected field names: " + repr(list(changes)) + ) return type(self)(*values) + def __replace__(self, **changes): + return self._replace(**changes) + def __repr__(self): parts = [] for name, value in zip(field_names, self._values): @@ -600,5 +616,11 @@ def __repr__(self): _NT.__name__ = typename _NT.__qualname__ = typename + if module is not None: + _NT.__module__ = module return _NT + + +# Pull in the abc-backed user wrappers last (see note near `__all__`). +from _collections_user import UserDict, UserList, UserString diff --git a/crates/weavepy-vm/src/stdlib/python/collections_abc.py b/crates/weavepy-vm/src/stdlib/python/collections_abc.py new file mode 100644 index 0000000..6e67bfb --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/collections_abc.py @@ -0,0 +1,10 @@ +"""``collections.abc`` — re-export of the ABCs defined in ``_collections_abc``. + +Carried verbatim from CPython 3.13. The implementation lives in +``_collections_abc`` (so the early-startup machinery can import the few +ABCs it needs without dragging in the whole ``collections`` package); this +module is the public spelling everyone actually imports. +""" + +from _collections_abc import * +from _collections_abc import __all__ # noqa: F401 diff --git a/crates/weavepy-vm/src/stdlib/python/contextlib.py b/crates/weavepy-vm/src/stdlib/python/contextlib.py index 28f4843..5b646fa 100644 --- a/crates/weavepy-vm/src/stdlib/python/contextlib.py +++ b/crates/weavepy-vm/src/stdlib/python/contextlib.py @@ -1,36 +1,80 @@ -"""WeavePy's pure-Python ``contextlib`` module. - -Provides the most common context-manager helpers: ``contextmanager``, -``closing``, ``suppress``, ``redirect_stdout``, ``redirect_stderr``, -``nullcontext``, and ``ExitStack``. -""" - +"""Utilities for with-statement contexts. See PEP 343.""" +import abc +import os import sys +import _collections_abc +from collections import deque from functools import wraps +from types import MethodType, GenericAlias + +__all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", + "AbstractContextManager", "AbstractAsyncContextManager", + "AsyncExitStack", "ContextDecorator", "ExitStack", + "redirect_stdout", "redirect_stderr", "suppress", "aclosing", + "chdir"] + + +class AbstractContextManager(abc.ABC): + + """An abstract base class for context managers.""" + + __class_getitem__ = classmethod(GenericAlias) + + __slots__ = () + + def __enter__(self): + """Return `self` upon entering the runtime context.""" + return self + + @abc.abstractmethod + def __exit__(self, exc_type, exc_value, traceback): + """Raise any exception triggered within the runtime context.""" + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is AbstractContextManager: + return _collections_abc._check_methods(C, "__enter__", "__exit__") + return NotImplemented + + +class AbstractAsyncContextManager(abc.ABC): + + """An abstract base class for asynchronous context managers.""" + __class_getitem__ = classmethod(GenericAlias) -__all__ = [ - "contextmanager", - "ContextDecorator", - "closing", - "suppress", - "redirect_stdout", - "redirect_stderr", - "nullcontext", - "ExitStack", -] + __slots__ = () + async def __aenter__(self): + """Return `self` upon entering the runtime context.""" + return self + + @abc.abstractmethod + async def __aexit__(self, exc_type, exc_value, traceback): + """Raise any exception triggered within the runtime context.""" + return None -class ContextDecorator: - """A base class or mixin that enables context managers to work as - decorators (PEP 343 / CPython ``contextlib.ContextDecorator``).""" + @classmethod + def __subclasshook__(cls, C): + if cls is AbstractAsyncContextManager: + return _collections_abc._check_methods(C, "__aenter__", + "__aexit__") + return NotImplemented + + +class ContextDecorator(object): + "A base class or mixin that enables context managers to work as decorators." def _recreate_cm(self): """Return a recreated instance of self. Allows an otherwise one-shot context manager like - ``_GeneratorContextManager`` to be used as a decorator over - multiple invocations of the wrapped function. + _GeneratorContextManager to support use as + a decorator via implicit recreation. + + This is a private interface just for _GeneratorContextManager. + See issue #11647 for details. """ return self @@ -42,100 +86,317 @@ def inner(*args, **kwds): return inner -class _GeneratorContextManager(ContextDecorator): - """Wrap a generator function turned into a context manager. +class AsyncContextDecorator(object): + "A base class or mixin that enables async context managers to work as decorators." - Inherits :class:`ContextDecorator` so a ``@contextmanager`` result - can itself be used as a decorator (``@cm`` over a function), with - ``_recreate_cm`` rebuilding the one-shot generator each call. - """ + def _recreate_cm(self): + """Return a recreated instance of self. + """ + return self + + def __call__(self, func): + @wraps(func) + async def inner(*args, **kwds): + async with self._recreate_cm(): + return await func(*args, **kwds) + return inner + + +class _GeneratorContextManagerBase: + """Shared functionality for @contextmanager and @asynccontextmanager.""" def __init__(self, func, args, kwds): self.gen = func(*args, **kwds) self.func, self.args, self.kwds = func, args, kwds - # Carry the wrapped function's docstring, matching CPython. + # Issue 19330: ensure context manager instances have good docstrings doc = getattr(func, "__doc__", None) if doc is None: doc = type(self).__doc__ self.__doc__ = doc + # Unfortunately, this still doesn't provide good help output when + # inspecting the created context manager instances, since pydoc + # currently bypasses the instance docstring and shows the docstring + # for the class instead. + # See http://bugs.python.org/issue19404 for more details. def _recreate_cm(self): + # _GCMB instances are one-shot context managers, so the + # CM must be recreated each time a decorated function is + # called return self.__class__(self.func, self.args, self.kwds) + +class _GeneratorContextManager( + _GeneratorContextManagerBase, + AbstractContextManager, + ContextDecorator, +): + """Helper for @contextmanager decorator.""" + def __enter__(self): + # do not keep args and kwds alive unnecessarily + # they are only needed for recreation, which is not possible anymore + del self.args, self.kwds, self.func try: return next(self.gen) except StopIteration: raise RuntimeError("generator didn't yield") from None - def __exit__(self, exc_type, exc_value, traceback): - if exc_type is None: + def __exit__(self, typ, value, traceback): + if typ is None: try: next(self.gen) except StopIteration: return False - raise RuntimeError("generator didn't stop") + else: + try: + raise RuntimeError("generator didn't stop") + finally: + self.gen.close() + else: + if value is None: + # Need to force instantiation so we can reliably + # tell if we get the same exception back + value = typ() + try: + self.gen.throw(value) + except StopIteration as exc: + # Suppress StopIteration *unless* it's the same exception that + # was passed to throw(). This prevents a StopIteration + # raised inside the "with" statement from being suppressed. + return exc is not value + except RuntimeError as exc: + # Don't re-raise the passed in exception. (issue27122) + if exc is value: + exc.__traceback__ = traceback + return False + # Avoid suppressing if a StopIteration exception + # was passed to throw() and later wrapped into a RuntimeError + # (see PEP 479 for sync generators; async generators also + # have this behavior). But do this only if the exception wrapped + # by the RuntimeError is actually Stop(Async)Iteration (see + # issue29692). + if ( + isinstance(value, StopIteration) + and exc.__cause__ is value + ): + value.__traceback__ = traceback + return False + raise + except BaseException as exc: + # only re-raise if it's *not* the exception that was + # passed to throw(), because __exit__() must not raise + # an exception unless __exit__() itself failed. But throw() + # has to raise the exception to signal propagation, so this + # fixes the impedance mismatch between the throw() protocol + # and the __exit__() protocol. + if exc is not value: + raise + exc.__traceback__ = traceback + return False + try: + raise RuntimeError("generator didn't stop after throw()") + finally: + self.gen.close() + +class _AsyncGeneratorContextManager( + _GeneratorContextManagerBase, + AbstractAsyncContextManager, + AsyncContextDecorator, +): + """Helper for @asynccontextmanager decorator.""" + + async def __aenter__(self): + # do not keep args and kwds alive unnecessarily + # they are only needed for recreation, which is not possible anymore + del self.args, self.kwds, self.func try: - self.gen.throw(exc_type, exc_value, traceback) - except StopIteration as stop: - return stop is not exc_value - except BaseException as exc: - if exc is exc_value: + return await anext(self.gen) + except StopAsyncIteration: + raise RuntimeError("generator didn't yield") from None + + async def __aexit__(self, typ, value, traceback): + if typ is None: + try: + await anext(self.gen) + except StopAsyncIteration: return False - raise - raise RuntimeError("generator didn't stop after throw()") + else: + try: + raise RuntimeError("generator didn't stop") + finally: + await self.gen.aclose() + else: + if value is None: + # Need to force instantiation so we can reliably + # tell if we get the same exception back + value = typ() + try: + await self.gen.athrow(value) + except StopAsyncIteration as exc: + # Suppress StopIteration *unless* it's the same exception that + # was passed to throw(). This prevents a StopIteration + # raised inside the "with" statement from being suppressed. + return exc is not value + except RuntimeError as exc: + # Don't re-raise the passed in exception. (issue27122) + if exc is value: + exc.__traceback__ = traceback + return False + # Avoid suppressing if a Stop(Async)Iteration exception + # was passed to athrow() and later wrapped into a RuntimeError + # (see PEP 479 for sync generators; async generators also + # have this behavior). But do this only if the exception wrapped + # by the RuntimeError is actually Stop(Async)Iteration (see + # issue29692). + if ( + isinstance(value, (StopIteration, StopAsyncIteration)) + and exc.__cause__ is value + ): + value.__traceback__ = traceback + return False + raise + except BaseException as exc: + # only re-raise if it's *not* the exception that was + # passed to throw(), because __exit__() must not raise + # an exception unless __exit__() itself failed. But throw() + # has to raise the exception to signal propagation, so this + # fixes the impedance mismatch between the throw() protocol + # and the __exit__() protocol. + if exc is not value: + raise + exc.__traceback__ = traceback + return False + try: + raise RuntimeError("generator didn't stop after athrow()") + finally: + await self.gen.aclose() def contextmanager(func): - """Decorator that turns a generator into a context-manager factory. + """@contextmanager decorator. - The returned ``helper`` builds a fresh :class:`_GeneratorContextManager` - per call and is ``functools.wraps``-decorated so the factory keeps the - wrapped function's name/qualname/doc. + Typical usage: + + @contextmanager + def some_generator(): + + try: + yield + finally: + + + This makes this: + + with some_generator() as : + + + equivalent to this: + + + try: + = + + finally: + + """ + @wraps(func) + def helper(*args, **kwds): + return _GeneratorContextManager(func, args, kwds) + return helper + + +def asynccontextmanager(func): + """@asynccontextmanager decorator. + + Typical usage: + + @asynccontextmanager + async def some_async_generator(): + + try: + yield + finally: + + + This makes this: + + async with some_async_generator() as : + + + equivalent to this: + + + try: + = + + finally: + """ @wraps(func) - def helper(*args, **kwargs): - return _GeneratorContextManager(func, args, kwargs) + def helper(*args, **kwds): + return _AsyncGeneratorContextManager(func, args, kwds) return helper -class closing: - """Context manager that calls ``close`` on its target.""" +class closing(AbstractContextManager): + """Context to automatically close something at the end of a block. + + Code like this: + with closing(.open()) as f: + + + is equivalent to this: + + f = .open() + try: + + finally: + f.close() + + """ def __init__(self, thing): self.thing = thing - def __enter__(self): return self.thing - - def __exit__(self, *exc): + def __exit__(self, *exc_info): self.thing.close() - return False -class suppress: - """Suppress one or more exception types.""" +class aclosing(AbstractAsyncContextManager): + """Async context manager for safely finalizing an asynchronously cleaned-up + resource such as an async generator, calling its ``aclose()`` method. - def __init__(self, *exceptions): - self._exceptions = exceptions + Code like this: - def __enter__(self): - return None + async with aclosing(.fetch()) as agen: + - def __exit__(self, exc_type, exc_value, traceback): - if exc_type is None: - return False - for exc in self._exceptions: - if issubclass(exc_type, exc): - return True - return False + is equivalent to this: + agen = .fetch() + try: + + finally: + await agen.aclose() + + """ + def __init__(self, thing): + self.thing = thing + async def __aenter__(self): + return self.thing + async def __aexit__(self, *exc_info): + await self.thing.aclose() + + +class _RedirectStream(AbstractContextManager): -class _RedirectStream: _stream = None def __init__(self, new_target): self._new_target = new_target + # We use a list of old targets to make this CM re-entrant self._old_targets = [] def __enter__(self): @@ -143,81 +404,411 @@ def __enter__(self): setattr(sys, self._stream, self._new_target) return self._new_target - def __exit__(self, *exc): + def __exit__(self, exctype, excinst, exctb): setattr(sys, self._stream, self._old_targets.pop()) - return False class redirect_stdout(_RedirectStream): + """Context manager for temporarily redirecting stdout to another file. + + # How to send help() to stderr + with redirect_stdout(sys.stderr): + help(dir) + + # How to write help() to a file + with open('help.txt', 'w') as f: + with redirect_stdout(f): + help(pow) + """ + _stream = "stdout" class redirect_stderr(_RedirectStream): + """Context manager for temporarily redirecting stderr to another file.""" + _stream = "stderr" -class nullcontext: - """Context manager that does nothing.""" +class suppress(AbstractContextManager): + """Context manager to suppress specified exceptions - def __init__(self, enter_result=None): - self.enter_result = enter_result + After the exception is suppressed, execution proceeds with the next + statement following the with statement. - def __enter__(self): - return self.enter_result + with suppress(FileNotFoundError): + os.remove(somefile) + # Execution still resumes here if the file was already removed + """ + + def __init__(self, *exceptions): + self._exceptions = exceptions - def __exit__(self, *exc): + def __enter__(self): + pass + + def __exit__(self, exctype, excinst, exctb): + # Unlike isinstance and issubclass, CPython exception handling + # currently only looks at the concrete type hierarchy (ignoring + # the instance and subclass checking hooks). While Guido considers + # that a bug rather than a feature, it's a fairly hard one to fix + # due to various internal implementation details. suppress provides + # the simpler issubclass based semantics, rather than trying to + # exactly reproduce the limitations of the CPython interpreter. + # + # See http://bugs.python.org/issue12029 for more details + if exctype is None: + return + if issubclass(exctype, self._exceptions): + return True + if issubclass(exctype, BaseExceptionGroup): + match, rest = excinst.split(self._exceptions) + if rest is None: + return True + raise rest return False -class ExitStack: - """Track and unwind multiple context managers.""" +class _BaseExitStack: + """A base class for ExitStack and AsyncExitStack.""" + + @staticmethod + def _create_exit_wrapper(cm, cm_exit): + return MethodType(cm_exit, cm) + + @staticmethod + def _create_cb_wrapper(callback, /, *args, **kwds): + def _exit_wrapper(exc_type, exc, tb): + callback(*args, **kwds) + return _exit_wrapper def __init__(self): - self._exit_callbacks = [] + self._exit_callbacks = deque() + + def pop_all(self): + """Preserve the context stack by transferring it to a new instance.""" + new_stack = type(self)() + new_stack._exit_callbacks = self._exit_callbacks + self._exit_callbacks = deque() + return new_stack + + def push(self, exit): + """Registers a callback with the standard __exit__ method signature. + + Can suppress exceptions the same way __exit__ method can. + Also accepts any object with an __exit__ method (registering a call + to the method instead of the object itself). + """ + # We use an unbound method rather than a bound method to follow + # the standard lookup behaviour for special methods. + _cb_type = type(exit) + + try: + exit_method = _cb_type.__exit__ + except AttributeError: + # Not a context manager, so assume it's a callable. + self._push_exit_callback(exit) + else: + self._push_cm_exit(exit, exit_method) + return exit # Allow use as a decorator. + + def enter_context(self, cm): + """Enters the supplied context manager. + + If successful, also pushes its __exit__ method as a callback and + returns the result of the __enter__ method. + """ + # We look up the special methods on the type to match the with + # statement. + cls = type(cm) + try: + _enter = cls.__enter__ + _exit = cls.__exit__ + except AttributeError: + raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " + f"not support the context manager protocol") from None + result = _enter(cm) + self._push_cm_exit(cm, _exit) + return result + + def callback(self, callback, /, *args, **kwds): + """Registers an arbitrary callback and arguments. + + Cannot suppress exceptions. + """ + _exit_wrapper = self._create_cb_wrapper(callback, *args, **kwds) + + # We changed the signature, so using @wraps is not appropriate, but + # setting __wrapped__ may still help with introspection. + _exit_wrapper.__wrapped__ = callback + self._push_exit_callback(_exit_wrapper) + return callback # Allow use as a decorator + + def _push_cm_exit(self, cm, cm_exit): + """Helper to correctly register callbacks to __exit__ methods.""" + _exit_wrapper = self._create_exit_wrapper(cm, cm_exit) + self._push_exit_callback(_exit_wrapper, True) + + def _push_exit_callback(self, callback, is_sync=True): + self._exit_callbacks.append((is_sync, callback)) + + +# Inspired by discussions on http://bugs.python.org/issue13585 +class ExitStack(_BaseExitStack, AbstractContextManager): + """Context manager for dynamic management of a stack of exit callbacks. + + For example: + with ExitStack() as stack: + files = [stack.enter_context(open(fname)) for fname in filenames] + # All opened files will automatically be closed at the end of + # the with statement, even if attempts to open files later + # in the list raise an exception. + """ def __enter__(self): return self - def __exit__(self, exc_type, exc_value, traceback): - suppressed = False + def __exit__(self, *exc_details): + exc = exc_details[1] + received_exc = exc is not None + + # We manipulate the exception state so it behaves as though + # we were actually nesting multiple with statements + frame_exc = sys.exception() + def _fix_exception_context(new_exc, old_exc): + # Context may not be correct, so find the end of the chain + while 1: + exc_context = new_exc.__context__ + if exc_context is None or exc_context is old_exc: + # Context is already set correctly (see issue 20317) + return + if exc_context is frame_exc: + break + new_exc = exc_context + # Change the end of the chain to point to the exception + # we expect it to reference + new_exc.__context__ = old_exc + + # Callbacks are invoked in LIFO order to match the behaviour of + # nested context managers + suppressed_exc = False + pending_raise = False while self._exit_callbacks: - cb = self._exit_callbacks.pop() + is_sync, cb = self._exit_callbacks.pop() + assert is_sync try: - if cb(exc_type, exc_value, traceback): - suppressed = True - exc_type = None - exc_value = None - traceback = None + if exc is None: + exc_details = None, None, None + else: + exc_details = type(exc), exc, exc.__traceback__ + if cb(*exc_details): + suppressed_exc = True + pending_raise = False + exc = None except BaseException as new_exc: - exc_type = type(new_exc) - exc_value = new_exc - traceback = None - suppressed = False - if exc_value is not None and not suppressed: - raise exc_value - return suppressed + # simulate the stack of exceptions by setting the context + _fix_exception_context(new_exc, exc) + pending_raise = True + exc = new_exc - def enter_context(self, cm): - result = cm.__enter__() - self._exit_callbacks.append(cm.__exit__) + if pending_raise: + try: + # bare "raise exc" replaces our carefully + # set-up context + fixed_ctx = exc.__context__ + raise exc + except BaseException: + exc.__context__ = fixed_ctx + raise + return received_exc and suppressed_exc + + def close(self): + """Immediately unwind the context stack.""" + self.__exit__(None, None, None) + + +# Inspired by discussions on https://bugs.python.org/issue29302 +class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): + """Async context manager for dynamic management of a stack of exit + callbacks. + + For example: + async with AsyncExitStack() as stack: + connections = [await stack.enter_async_context(get_connection()) + for i in range(5)] + # All opened connections will automatically be released at the + # end of the async with statement, even if attempts to open a + # connection later in the list raise an exception. + """ + + @staticmethod + def _create_async_exit_wrapper(cm, cm_exit): + return MethodType(cm_exit, cm) + + @staticmethod + def _create_async_cb_wrapper(callback, /, *args, **kwds): + async def _exit_wrapper(exc_type, exc, tb): + await callback(*args, **kwds) + return _exit_wrapper + + async def enter_async_context(self, cm): + """Enters the supplied async context manager. + + If successful, also pushes its __aexit__ method as a callback and + returns the result of the __aenter__ method. + """ + cls = type(cm) + try: + _enter = cls.__aenter__ + _exit = cls.__aexit__ + except AttributeError: + raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " + f"not support the asynchronous context manager protocol" + ) from None + result = await _enter(cm) + self._push_async_cm_exit(cm, _exit) return result - def callback(self, fn, *args, **kwargs): - def _cb(exc_type, exc_value, traceback): - fn(*args, **kwargs) - return False - self._exit_callbacks.append(_cb) - return fn + def push_async_exit(self, exit): + """Registers a coroutine function with the standard __aexit__ method + signature. + + Can suppress exceptions the same way __aexit__ method can. + Also accepts any object with an __aexit__ method (registering a call + to the method instead of the object itself). + """ + _cb_type = type(exit) + try: + exit_method = _cb_type.__aexit__ + except AttributeError: + # Not an async context manager, so assume it's a coroutine function + self._push_exit_callback(exit, False) + else: + self._push_async_cm_exit(exit, exit_method) + return exit # Allow use as a decorator + + def push_async_callback(self, callback, /, *args, **kwds): + """Registers an arbitrary coroutine function and arguments. + + Cannot suppress exceptions. + """ + _exit_wrapper = self._create_async_cb_wrapper(callback, *args, **kwds) + + # We changed the signature, so using @wraps is not appropriate, but + # setting __wrapped__ may still help with introspection. + _exit_wrapper.__wrapped__ = callback + self._push_exit_callback(_exit_wrapper, False) + return callback # Allow use as a decorator - def push(self, exit_method): - self._exit_callbacks.append(exit_method) - return exit_method + async def aclose(self): + """Immediately unwind the context stack.""" + await self.__aexit__(None, None, None) - def pop_all(self): - new_stack = ExitStack() - new_stack._exit_callbacks = self._exit_callbacks - self._exit_callbacks = [] - return new_stack + def _push_async_cm_exit(self, cm, cm_exit): + """Helper to correctly register coroutine function to __aexit__ + method.""" + _exit_wrapper = self._create_async_exit_wrapper(cm, cm_exit) + self._push_exit_callback(_exit_wrapper, False) - def close(self): - self.__exit__(None, None, None) + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_details): + exc = exc_details[1] + received_exc = exc is not None + + # We manipulate the exception state so it behaves as though + # we were actually nesting multiple with statements + frame_exc = sys.exception() + def _fix_exception_context(new_exc, old_exc): + # Context may not be correct, so find the end of the chain + while 1: + exc_context = new_exc.__context__ + if exc_context is None or exc_context is old_exc: + # Context is already set correctly (see issue 20317) + return + if exc_context is frame_exc: + break + new_exc = exc_context + # Change the end of the chain to point to the exception + # we expect it to reference + new_exc.__context__ = old_exc + + # Callbacks are invoked in LIFO order to match the behaviour of + # nested context managers + suppressed_exc = False + pending_raise = False + while self._exit_callbacks: + is_sync, cb = self._exit_callbacks.pop() + try: + if exc is None: + exc_details = None, None, None + else: + exc_details = type(exc), exc, exc.__traceback__ + if is_sync: + cb_suppress = cb(*exc_details) + else: + cb_suppress = await cb(*exc_details) + + if cb_suppress: + suppressed_exc = True + pending_raise = False + exc = None + except BaseException as new_exc: + # simulate the stack of exceptions by setting the context + _fix_exception_context(new_exc, exc) + pending_raise = True + exc = new_exc + + if pending_raise: + try: + # bare "raise exc" replaces our carefully + # set-up context + fixed_ctx = exc.__context__ + raise exc + except BaseException: + exc.__context__ = fixed_ctx + raise + return received_exc and suppressed_exc + + +class nullcontext(AbstractContextManager, AbstractAsyncContextManager): + """Context manager that does no additional processing. + + Used as a stand-in for a normal context manager, when a particular + block of code is only sometimes used with a normal context manager: + + cm = optional_cm if condition else nullcontext() + with cm: + # Perform operation, using optional_cm if condition is True + """ + + def __init__(self, enter_result=None): + self.enter_result = enter_result + + def __enter__(self): + return self.enter_result + + def __exit__(self, *excinfo): + pass + + async def __aenter__(self): + return self.enter_result + + async def __aexit__(self, *excinfo): + pass + + +class chdir(AbstractContextManager): + """Non thread-safe context manager to change the current working directory.""" + + def __init__(self, path): + self.path = path + self._old_cwd = [] + + def __enter__(self): + self._old_cwd.append(os.getcwd()) + os.chdir(self.path) + + def __exit__(self, *excinfo): + os.chdir(self._old_cwd.pop()) diff --git a/crates/weavepy-vm/src/stdlib/python/copy_mod.py b/crates/weavepy-vm/src/stdlib/python/copy_mod.py index a9956d0..4ed2b1c 100644 --- a/crates/weavepy-vm/src/stdlib/python/copy_mod.py +++ b/crates/weavepy-vm/src/stdlib/python/copy_mod.py @@ -1,42 +1,56 @@ """Shallow / deep copy operations — WeavePy port of CPython's ``copy``. -Mirrors the public surface: :func:`copy.copy` and -:func:`copy.deepcopy`, with the ``__copy__`` / ``__deepcopy__`` -protocol honoured. The dispatch tables for the immutable atomic -types match CPython. +A close port of CPython 3.13's :mod:`copy`: the public surface +(:func:`copy.copy`, :func:`copy.deepcopy`, :func:`copy.replace`), the +``__copy__`` / ``__deepcopy__`` / ``__replace__`` protocols, the +``copyreg.dispatch_table`` registry hook, and the per-type dispatch +tables for the immutable atomic types all match CPython. """ +import types +import weakref +from copyreg import dispatch_table + class Error(Exception): pass -error = Error +error = Error # backward compatibility -# Sentinel for the deepcopy memo lookup miss. -_nil = object() +__all__ = ["Error", "copy", "deepcopy", "replace"] def copy(x): + """Shallow copy operation on arbitrary Python objects.""" cls = type(x) copier = _copy_dispatch.get(cls) if copier: return copier(x) - copier = getattr(x, "__copy__", None) + if issubclass(cls, type): + # treat it as a regular class: + return _copy_immutable(x) + + copier = getattr(cls, "__copy__", None) if copier is not None: - return copier() + return copier(x) - reductor = getattr(x, "__reduce_ex__", None) + reductor = dispatch_table.get(cls) if reductor is not None: - rv = reductor(4) + rv = reductor(x) else: - reductor = getattr(x, "__reduce__", None) - if reductor: - rv = reductor() + reductor = getattr(x, "__reduce_ex__", None) + if reductor is not None: + rv = reductor(4) else: - raise Error("un(shallow)copyable object of type %s" % cls) + reductor = getattr(x, "__reduce__", None) + if reductor: + rv = reductor() + else: + raise Error("un(shallow)copyable object of type %s" % cls) + if isinstance(rv, str): return x return _reconstruct(x, None, *rv) @@ -49,113 +63,94 @@ def _copy_immutable(x): return x -for t in ( - type(None), - int, - float, - bool, - complex, - str, - tuple, - bytes, - frozenset, - type, - range, - slice, - type(Ellipsis), - type(NotImplemented), -): +for t in (types.NoneType, int, float, bool, complex, str, tuple, + bytes, frozenset, type, range, slice, property, + types.BuiltinFunctionType, types.EllipsisType, + types.NotImplementedType, types.FunctionType, types.CodeType, + weakref.ref): d[t] = _copy_immutable +d[list] = list.copy +d[dict] = dict.copy +d[set] = set.copy +# `bytearray.copy` is not exposed in WeavePy yet; slicing is equivalent. +d[bytearray] = lambda x: x[:] -def _copy_list(x): - return x.copy() - - -d[list] = _copy_list - - -def _copy_dict(x): - return x.copy() - - -d[dict] = _copy_dict - - -def _copy_set(x): - return x.copy() - - -d[set] = _copy_set - - -def _copy_bytearray(x): - return x[:] - - -d[bytearray] = _copy_bytearray +del d, t def deepcopy(x, memo=None, _nil=[]): + """Deep copy operation on arbitrary Python objects.""" + d = id(x) if memo is None: memo = {} - d = id(x) - y = memo.get(d, _nil) - if y is not _nil: - return y + else: + y = memo.get(d, _nil) + if y is not _nil: + return y + cls = type(x) + copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x, memo) else: - if cls is type: - y = x + if issubclass(cls, type): + y = _deepcopy_atomic(x, memo) else: copier = getattr(x, "__deepcopy__", None) if copier is not None: y = copier(memo) else: - reductor = getattr(x, "__reduce_ex__", None) + reductor = dispatch_table.get(cls) if reductor: - rv = reductor(4) + rv = reductor(x) else: - reductor = getattr(x, "__reduce__", None) - if reductor: - rv = reductor() + reductor = getattr(x, "__reduce_ex__", None) + if reductor is not None: + rv = reductor(4) else: - raise Error("un(deep)copyable object of type %s" % cls) + reductor = getattr(x, "__reduce__", None) + if reductor: + rv = reductor() + else: + raise Error( + "un(deep)copyable object of type %s" % cls) if isinstance(rv, str): y = x else: y = _reconstruct(x, memo, *rv) + # If is its own copy, don't memoize. if y is not x: memo[d] = y - _keep_alive(x, memo) + _keep_alive(x, memo) # Make sure x lives at least as long as d return y -_deepcopy_dispatch = dd = {} +_deepcopy_dispatch = d = {} def _deepcopy_atomic(x, memo): return x -for t in ( - type(None), - int, - float, - bool, - complex, - str, - bytes, - type, - range, - type(Ellipsis), - type(NotImplemented), -): - dd[t] = _deepcopy_atomic +d[types.NoneType] = _deepcopy_atomic +d[types.EllipsisType] = _deepcopy_atomic +d[types.NotImplementedType] = _deepcopy_atomic +d[int] = _deepcopy_atomic +d[float] = _deepcopy_atomic +d[bool] = _deepcopy_atomic +d[complex] = _deepcopy_atomic +d[bytes] = _deepcopy_atomic +d[str] = _deepcopy_atomic +d[types.CodeType] = _deepcopy_atomic +d[type] = _deepcopy_atomic +d[range] = _deepcopy_atomic +d[types.BuiltinFunctionType] = _deepcopy_atomic +d[types.FunctionType] = _deepcopy_atomic +d[weakref.ref] = _deepcopy_atomic +d[property] = _deepcopy_atomic def _deepcopy_list(x, memo, deepcopy=deepcopy): @@ -167,11 +162,13 @@ def _deepcopy_list(x, memo, deepcopy=deepcopy): return y -dd[list] = _deepcopy_list +d[list] = _deepcopy_list def _deepcopy_tuple(x, memo, deepcopy=deepcopy): y = [deepcopy(a, memo) for a in x] + # We're not going to put the tuple in the memo, but it's still important we + # check for it, in case the tuple contains recursive mutable structures. try: return memo[id(x)] except KeyError: @@ -182,11 +179,10 @@ def _deepcopy_tuple(x, memo, deepcopy=deepcopy): break else: y = x - memo[id(x)] = y return y -dd[tuple] = _deepcopy_tuple +d[tuple] = _deepcopy_tuple def _deepcopy_dict(x, memo, deepcopy=deepcopy): @@ -197,54 +193,47 @@ def _deepcopy_dict(x, memo, deepcopy=deepcopy): return y -dd[dict] = _deepcopy_dict +d[dict] = _deepcopy_dict -def _deepcopy_set(x, memo, deepcopy=deepcopy): - y = set() - memo[id(x)] = y - for a in x: - y.add(deepcopy(a, memo)) - return y +def _deepcopy_method(x, memo): # Copy instance methods + return type(x)(x.__func__, deepcopy(x.__self__, memo)) -dd[set] = _deepcopy_set +d[types.MethodType] = _deepcopy_method - -def _deepcopy_frozenset(x, memo, deepcopy=deepcopy): - return frozenset(deepcopy(a, memo) for a in x) - - -dd[frozenset] = _deepcopy_frozenset - - -def _deepcopy_bytearray(x, memo): - y = x[:] - memo[id(x)] = y - return y - - -dd[bytearray] = _deepcopy_bytearray +del d def _keep_alive(x, memo): + """Keeps a reference to the object x in the memo. + + Because we remember objects by their id, we have to assure that + possibly temporary objects are kept alive by referencing them. + We store a reference at the id of the memo, which should normally + not be used unless someone tries to deepcopy the memo itself... + """ try: memo[id(memo)].append(x) except KeyError: + # aha, this is the first one :-) memo[id(memo)] = [x] -def _reconstruct(x, memo, func, args, state=None, listiter=None, dictiter=None, deepcopy=deepcopy): +def _reconstruct(x, memo, func, args, + state=None, listiter=None, dictiter=None, + *, deepcopy=deepcopy): deep = memo is not None if deep and args: args = (deepcopy(arg, memo) for arg in args) y = func(*args) if deep: memo[id(x)] = y + if state is not None: if deep: state = deepcopy(state, memo) - if hasattr(y, "__setstate__"): + if hasattr(y, '__setstate__'): y.__setstate__(state) else: if isinstance(state, tuple) and len(state) == 2: @@ -252,12 +241,11 @@ def _reconstruct(x, memo, func, args, state=None, listiter=None, dictiter=None, else: slotstate = None if state is not None: - d = y.__dict__ - for key, value in state.items(): - d[key] = value + y.__dict__.update(state) if slotstate is not None: for key, value in slotstate.items(): setattr(y, key, value) + if listiter is not None: if deep: for item in listiter: @@ -278,4 +266,17 @@ def _reconstruct(x, memo, func, args, state=None, listiter=None, dictiter=None, return y -__all__ = ["Error", "copy", "deepcopy"] +del types, weakref + + +def replace(obj, /, **changes): + """Return a new object replacing specified fields with new values. + + This is especially useful for immutable objects, like named tuples or + frozen dataclasses. + """ + cls = obj.__class__ + func = getattr(cls, '__replace__', None) + if func is None: + raise TypeError(f"replace() does not support {cls.__name__} objects") + return func(obj, **changes) diff --git a/crates/weavepy-vm/src/stdlib/python/copyreg.py b/crates/weavepy-vm/src/stdlib/python/copyreg.py index 8379cc2..8d71a43 100644 --- a/crates/weavepy-vm/src/stdlib/python/copyreg.py +++ b/crates/weavepy-vm/src/stdlib/python/copyreg.py @@ -48,6 +48,70 @@ def __newobj_ex__(cls, args, kwargs): return cls.__new__(cls, *args, **kwargs) +def _default_getstate(obj): + """The CPython 3.11+ ``object.__getstate__`` default. + + Returns the instance ``__dict__`` (or ``None`` when empty), folded + together with any ``__slots__`` state as a ``(dict_state, slot_state)`` + pair when slots carry values. + """ + try: + d = obj.__dict__ + except AttributeError: + d = None + dict_state = d if d else None + slot_state = None + names = _slotnames(type(obj)) + if names: + slot_state = {} + for name in names: + try: + slot_state[name] = getattr(obj, name) + except AttributeError: + pass + if not slot_state: + slot_state = None + if slot_state is not None: + return (dict_state, slot_state) + return dict_state + + +def _reduce_newobj(obj, protocol): + """Port of CPython's ``object.__reduce_ex__`` protocol-2+ path + (``Objects/typeobject.c:reduce_newobj``). + + Produces the ``(callable, args, state, listitems, dictitems)`` tuple + that ``copy``/``pickle`` feed to ``copyreg._reconstruct`` to rebuild + the instance, honouring the ``__getnewargs_ex__`` / ``__getnewargs__`` + and ``__getstate__`` hooks. + """ + cls = type(obj) + getnewargs_ex = getattr(obj, "__getnewargs_ex__", None) + if getnewargs_ex is not None: + args, kwargs = getnewargs_ex() + else: + getnewargs = getattr(obj, "__getnewargs__", None) + args = getnewargs() if getnewargs is not None else () + kwargs = {} + + if kwargs: + newobj = __newobj_ex__ + newargs = (cls, tuple(args), kwargs) + else: + newobj = __newobj__ + newargs = (cls,) + tuple(args) + + getstate = getattr(obj, "__getstate__", None) + if getstate is not None: + state = getstate() + else: + state = _default_getstate(obj) + + listitems = iter(obj) if isinstance(obj, list) else None + dictitems = iter(obj.items()) if isinstance(obj, dict) else None + return newobj, newargs, state, listitems, dictitems + + def _slotnames(cls): """Return a (possibly cached) list of slot-style attribute names.""" slotnames = cls.__dict__.get("__slotnames__") @@ -67,8 +131,15 @@ def _slotnames(cls): for name in slots: if name in ("__dict__", "__weakref__"): continue + # mangled names — but a class named only with + # underscores (e.g. ``___``) strips to "" and the + # slot keeps its raw name (CPython parity). if name.startswith("__") and not name.endswith("__"): - names.append("_" + c.__name__ + name) + stripped = c.__name__.lstrip("_") + if stripped: + names.append("_%s%s" % (stripped, name)) + else: + names.append(name) else: names.append(name) slotnames = names @@ -79,6 +150,52 @@ def _slotnames(cls): return slotnames +# A registry of extension codes (ad-hoc pickle compression). Codes are +# positive ints in [1, 0x7fffffff]; 0 is reserved. These tables are a +# faithful port of CPython's copyreg extension registry — pickle grabs a +# reference at init, so the names must never be rebound. +_extension_registry = {} # key -> code +_inverted_registry = {} # code -> key +_extension_cache = {} # code -> object + + +def add_extension(module, name, code): + """Register an extension code.""" + code = int(code) + if not 1 <= code <= 0x7fffffff: + raise ValueError("code out of range") + key = (module, name) + if (_extension_registry.get(key) == code and + _inverted_registry.get(code) == key): + return # Redundant registrations are benign + if key in _extension_registry: + raise ValueError("key %s is already registered with code %s" % + (key, _extension_registry[key])) + if code in _inverted_registry: + raise ValueError("code %s is already in use for key %s" % + (code, _inverted_registry[code])) + _extension_registry[key] = code + _inverted_registry[code] = key + + +def remove_extension(module, name, code): + """Unregister an extension code. For testing only.""" + key = (module, name) + if (_extension_registry.get(key) != code or + _inverted_registry.get(code) != key): + raise ValueError("key %s is not registered with code %s" % + (key, code)) + del _extension_registry[key] + del _inverted_registry[code] + if code in _extension_cache: + del _extension_cache[code] + + +def clear_extension_cache(): + _extension_cache.clear() + + __all__ = ["pickle", "constructor", "dispatch_table", + "add_extension", "remove_extension", "clear_extension_cache", "__newobj__", "__newobj_ex__", "_reconstructor", "_slotnames"] diff --git a/crates/weavepy-vm/src/stdlib/python/dataclasses.py b/crates/weavepy-vm/src/stdlib/python/dataclasses.py index a2fc998..c67b2f3 100644 --- a/crates/weavepy-vm/src/stdlib/python/dataclasses.py +++ b/crates/weavepy-vm/src/stdlib/python/dataclasses.py @@ -196,10 +196,19 @@ def __init__(self, *args, **kwargs): f"__init__() takes {len(pos_fields) + 1} positional arguments " f"but {len(args) + 1} were given" ) + init_field_names = {fld.name for fld in init_fields} provided = {} for f, value in zip(pos_fields, args): provided[f.name] = value for key, value in kwargs.items(): + # An explicit-parameter `__init__` (CPython) rejects names that + # aren't init fields; mirror that so `C(unknown=…)` (and thus + # `copy.replace(obj, unknown=…)`) raises instead of silently + # dropping the value. + if key not in init_field_names: + raise TypeError( + f"__init__() got an unexpected keyword argument {key!r}" + ) if key in provided: raise TypeError( f"__init__() got multiple values for argument {key!r}" @@ -340,6 +349,10 @@ def _frozen_delattr(self, key): cls.__setattr__ = _frozen_setattr cls.__delattr__ = _frozen_delattr + # `copy.replace(obj)` (Python 3.13+) dispatches through `__replace__`. + if "__replace__" not in cls.__dict__: + cls.__replace__ = _replace + if slots: # CPython rebuilds the class so ``__slots__`` is in effect at # construction time; assigning ``cls.__slots__ = ...`` after @@ -491,12 +504,21 @@ def _astuple_inner(obj, tuple_factory): return obj +def _replace(self, /, **changes): + """`__replace__` implementation bound on each dataclass — delegates + to `replace` so `copy.replace(obj, **changes)` works (Python 3.13+).""" + return replace(self, **changes) + + def replace(obj, /, **changes): """Return a new dataclass instance with `changes` applied, all other fields copied from `obj`.""" if not is_dataclass(obj) or isinstance(obj, type): raise TypeError("replace() expects a dataclass instance") - kwargs = {} + # Fill in field values not being changed, mutating `changes` in place + # (CPython semantics). Any leftover keys that aren't init fields stay in + # `changes` and reach `__init__`, which rejects them with `TypeError` — + # so `replace(obj, not_a_field=…)` raises, as CPython requires. for f in fields(obj): if not f.init: if f.name in changes: @@ -504,11 +526,9 @@ def replace(obj, /, **changes): f"cannot replace non-init field {f.name!r}" ) continue - if f.name in changes: - kwargs[f.name] = changes[f.name] - else: - kwargs[f.name] = getattr(obj, f.name) - return type(obj)(**kwargs) + if f.name not in changes: + changes[f.name] = getattr(obj, f.name) + return type(obj)(**changes) def make_dataclass(cls_name, fields_spec, *, bases=(), namespace=None, **kwargs): diff --git a/crates/weavepy-vm/src/stdlib/python/decimal.py b/crates/weavepy-vm/src/stdlib/python/decimal.py index 36a5bff..ee3147f 100644 --- a/crates/weavepy-vm/src/stdlib/python/decimal.py +++ b/crates/weavepy-vm/src/stdlib/python/decimal.py @@ -1,517 +1,109 @@ -"""Public ``decimal`` module (RFC 0019). - -Pure-Python implementation of arbitrary-precision base-10 arithmetic -sitting directly on top of WeavePy's bignum ``int`` type. It mirrors -the most-used pieces of CPython's :class:`decimal.Decimal` API: - -* Construction from ``int``, ``str``, ``float``, ``Decimal``, and - ``(sign, digits_tuple, exponent)`` triples (matches CPython 3.x). -* ``__add__`` / ``__sub__`` / ``__mul__`` / ``__truediv__`` / - ``__neg__`` / ``__abs__`` / ``__pow__`` (integer exponents). -* Comparison operators, hashing, ``bool``, ``int``, ``float``. -* Quantize-style ``quantize(other, rounding=...)`` plus the standard - rounding mode constants and a thread-local ``getcontext()``. -* ``Decimal('3.14').as_tuple()`` and ``as_integer_ratio()``. - -It does *not* yet implement the full IEEE 754-2008 decimal context -(traps, signal flags, NaN/Infinity flavours), or the exhaustive -formatting / parsing rules CPython's ``_decimal`` module exposes. -Those are flagged as RFC follow-ups. +"""Decimal fixed-point and floating-point arithmetic. + +This is an implementation of decimal floating-point arithmetic based on +the General Decimal Arithmetic Specification: + + http://speleotrove.com/decimal/decarith.html + +and IEEE standard 854-1987: + + http://en.wikipedia.org/wiki/IEEE_854-1987 + +Decimal floating point has finite precision with arbitrarily large bounds. + +The purpose of this module is to support arithmetic using familiar +"schoolhouse" rules and to avoid some of the tricky representation +issues associated with binary floating point. The package is especially +useful for financial applications or for contexts where users have +expectations that are at odds with binary floating point (for instance, +in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead +of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected +Decimal('0.00')). + +Here are some examples of using the decimal module: + +>>> from decimal import * +>>> setcontext(ExtendedContext) +>>> Decimal(0) +Decimal('0') +>>> Decimal('1') +Decimal('1') +>>> Decimal('-.0123') +Decimal('-0.0123') +>>> Decimal(123456) +Decimal('123456') +>>> Decimal('123.45e12345678') +Decimal('1.2345E+12345680') +>>> Decimal('1.33') + Decimal('1.27') +Decimal('2.60') +>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') +Decimal('-2.20') +>>> dig = Decimal(1) +>>> print(dig / Decimal(3)) +0.333333333 +>>> getcontext().prec = 18 +>>> print(dig / Decimal(3)) +0.333333333333333333 +>>> print(dig.sqrt()) +1 +>>> print(Decimal(3).sqrt()) +1.73205080756887729 +>>> print(Decimal(3) ** 123) +4.85192780976896427E+58 +>>> inf = Decimal(1) / Decimal(0) +>>> print(inf) +Infinity +>>> neginf = Decimal(-1) / Decimal(0) +>>> print(neginf) +-Infinity +>>> print(neginf + inf) +NaN +>>> print(neginf * inf) +-Infinity +>>> print(dig / 0) +Infinity +>>> getcontext().traps[DivisionByZero] = 1 +>>> print(dig / 0) +Traceback (most recent call last): + ... + ... + ... +decimal.DivisionByZero: x / 0 +>>> c = Context() +>>> c.traps[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> c.divide(Decimal(0), Decimal(0)) +Decimal('NaN') +>>> c.traps[InvalidOperation] = 1 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> print(c.divide(Decimal(0), Decimal(0))) +Traceback (most recent call last): + ... + ... + ... +decimal.InvalidOperation: 0 / 0 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> c.traps[InvalidOperation] = 0 +>>> print(c.divide(Decimal(0), Decimal(0))) +NaN +>>> print(c.flags[InvalidOperation]) +1 +>>> """ -import math -import re - -ROUND_HALF_UP = "ROUND_HALF_UP" -ROUND_HALF_EVEN = "ROUND_HALF_EVEN" -ROUND_HALF_DOWN = "ROUND_HALF_DOWN" -ROUND_DOWN = "ROUND_DOWN" -ROUND_UP = "ROUND_UP" -ROUND_FLOOR = "ROUND_FLOOR" -ROUND_CEILING = "ROUND_CEILING" -ROUND_05UP = "ROUND_05UP" - -_PARSE = re.compile( - r"""\A - (?P[-+])? - (?: - (?P\d+)(?:\.(?P\d*))? - | - \.(?P\d+) - ) - (?:[eE](?P[-+]?\d+))? - \Z""", - re.VERBOSE, -) - - -class DecimalException(ArithmeticError): - pass - - -class InvalidOperation(DecimalException): - pass - - -class DivisionByZero(DecimalException, ZeroDivisionError): - pass - - -class Inexact(DecimalException): - pass - - -class Rounded(DecimalException): - pass - - -class Subnormal(DecimalException): - pass - - -class Overflow(DecimalException, OverflowError): - pass - - -class Underflow(DecimalException): - pass - - -class Clamped(DecimalException): - pass - - -class FloatOperation(DecimalException, TypeError): - pass - - -class _Context: - def __init__(self, prec=28, rounding=ROUND_HALF_EVEN): - self.prec = prec - self.rounding = rounding - - def copy(self): - return _Context(self.prec, self.rounding) - - -_default_context = _Context() - - -def getcontext(): - return _default_context - - -def setcontext(ctx): - global _default_context - _default_context = ctx - - -def localcontext(ctx=None): - return _LocalContext(ctx or _default_context.copy()) - - -class _LocalContext: - def __init__(self, ctx): - self.ctx = ctx - self._prev = None - - def __enter__(self): - global _default_context - self._prev = _default_context - _default_context = self.ctx - return self.ctx - - def __exit__(self, *exc): - global _default_context - _default_context = self._prev - return False - - -class Decimal: - """Arbitrary-precision decimal number.""" - - __slots__ = ("_sign", "_int", "_exp") - - def __new__(cls, value="0", context=None): - self = object.__new__(cls) - if isinstance(value, Decimal): - self._sign = value._sign - self._int = value._int - self._exp = value._exp - return self - if isinstance(value, int): - self._sign = 0 if value >= 0 else 1 - self._int = abs(value) - self._exp = 0 - return self - if isinstance(value, float): - return cls.from_float(value) - if isinstance(value, tuple): - if len(value) != 3: - raise InvalidOperation("Invalid Decimal tuple: %r" % (value,)) - sign, digits, exp = value - self._sign = sign - self._int = 0 - for d in digits: - self._int = self._int * 10 + int(d) - self._exp = exp - return self - if isinstance(value, str): - text = value.strip().replace("_", "") - if not text: - raise InvalidOperation("Invalid Decimal string: %r" % value) - m = _PARSE.match(text) - if not m: - lower = text.lower().lstrip("+-") - if lower in ("inf", "infinity"): - raise InvalidOperation( - "Decimal infinity not supported in this implementation") - if lower == "nan": - raise InvalidOperation( - "Decimal NaN not supported in this implementation") - raise InvalidOperation("Invalid Decimal string: %r" % value) - sign = 1 if m.group("sign") == "-" else 0 - int_part = m.group("int") or "" - frac_part = m.group("frac") or m.group("frac2") or "" - digits = (int_part + frac_part).lstrip("0") or "0" - exp = int(m.group("exp") or "0") - len(frac_part) - self._sign = sign - self._int = int(digits) - self._exp = exp - return self - raise TypeError("Cannot convert %r to Decimal" % value) - - @classmethod - def from_float(cls, f): - if math.isnan(f): - raise InvalidOperation("cannot convert NaN") - if math.isinf(f): - raise InvalidOperation("cannot convert infinity") - n, d = f.as_integer_ratio() - # Express n/d as exact decimal: n/d = n/(2^k * 5^j) — but we - # only have d as a power of 2. Multiply numerator by 5**k. - k = 0 - while d % 2 == 0: - d //= 2 - k += 1 - n *= 5 ** k - exp = -k - sign = 1 if n < 0 else 0 - n = abs(n) - return Decimal((sign, _digits(n), exp)) - - # -- accessors ------------------------------------------------- - - def as_tuple(self): - return (self._sign, _digits(self._int), self._exp) - - def as_integer_ratio(self): - if self._exp >= 0: - num = self._int * (10 ** self._exp) - den = 1 - else: - num = self._int - den = 10 ** (-self._exp) - from math import gcd - g = gcd(num, den) - if g > 1: - num //= g - den //= g - if self._sign: - num = -num - return (num, den) - - def is_zero(self): - return self._int == 0 - - # -- conversions ---------------------------------------------- - - def __repr__(self): - return "Decimal('%s')" % self - - def __str__(self): - sign = "-" if self._sign else "" - if self._exp >= 0: - digits = str(self._int) - return sign + digits + "0" * self._exp if self._exp else sign + digits - digits = str(self._int) - n = -self._exp - if n >= len(digits): - digits = "0" * (n - len(digits) + 1) + digits - return sign + digits[:-n] + "." + digits[-n:] - - def __int__(self): - if self._exp >= 0: - v = self._int * (10 ** self._exp) - else: - v = self._int // (10 ** -self._exp) - return -v if self._sign else v - - def __float__(self): - return float(str(self)) - - def __bool__(self): - return self._int != 0 - - def __hash__(self): - if self._exp >= 0: - return hash(int(self)) - return hash(self.as_integer_ratio()) - - # -- arithmetic ------------------------------------------------ - - def _to_signed_value(self): - """Return ``(value, exp)`` such that ``value * 10**exp == self``.""" - v = -self._int if self._sign else self._int - return v, self._exp - - @staticmethod - def _align(a, b): - """Bring two decimals to the same exponent.""" - ia, ea = a._to_signed_value() - ib, eb = b._to_signed_value() - if ea < eb: - ib *= 10 ** (eb - ea) - return ia, ib, ea - if eb < ea: - ia *= 10 ** (ea - eb) - return ia, ib, eb - return ia, ib, ea - - @staticmethod - def _from_signed(value, exp): - if value == 0: - return Decimal((0, (0,), exp)) - sign = 1 if value < 0 else 0 - return Decimal((sign, _digits(abs(value)), exp)) - - def __add__(self, other): - other = _coerce(other) - if other is NotImplemented: - return NotImplemented - ia, ib, e = Decimal._align(self, other) - return Decimal._from_signed(ia + ib, e) - - __radd__ = __add__ - - def __sub__(self, other): - other = _coerce(other) - if other is NotImplemented: - return NotImplemented - ia, ib, e = Decimal._align(self, other) - return Decimal._from_signed(ia - ib, e) - - def __rsub__(self, other): - other = _coerce(other) - if other is NotImplemented: - return NotImplemented - return other.__sub__(self) - - def __mul__(self, other): - other = _coerce(other) - if other is NotImplemented: - return NotImplemented - ia, ea = self._to_signed_value() - ib, eb = other._to_signed_value() - return Decimal._from_signed(ia * ib, ea + eb) - - __rmul__ = __mul__ - - def __truediv__(self, other): - other = _coerce(other) - if other is NotImplemented: - return NotImplemented - if other.is_zero(): - raise DivisionByZero("Decimal division by zero") - prec = getcontext().prec - a = -self._int if self._sign else self._int - b = -other._int if other._sign else other._int - ea, eb = self._exp, other._exp - # Scale a by 10^prec to retain precision. - scale = prec + 1 - q, _ = divmod(a * (10 ** scale), b) - return Decimal._from_signed(q, ea - eb - scale)._round(prec) - - def __rtruediv__(self, other): - other = _coerce(other) - if other is NotImplemented: - return NotImplemented - return other.__truediv__(self) - - def __neg__(self): - if self._int == 0: - return self - return Decimal((1 - self._sign, _digits(self._int), self._exp)) - - def __pos__(self): - return self - - def __abs__(self): - if self._sign == 0 or self._int == 0: - return self - return Decimal((0, _digits(self._int), self._exp)) - - def __pow__(self, other): - if not isinstance(other, int): - return NotImplemented - if other < 0: - return Decimal(1) / (self ** -other) - result = Decimal(1) - base = self - n = other - while n > 0: - if n & 1: - result = result * base - base = base * base - n >>= 1 - return result - - # -- comparisons ---------------------------------------------- - - def _cmp(self, other): - other = _coerce(other) - if other is NotImplemented: - return NotImplemented - ia, ib, _ = Decimal._align(self, other) - return (ia > ib) - (ia < ib) - - def __eq__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c == 0 - - def __lt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c < 0 - - def __le__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c <= 0 - - def __gt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c > 0 - - def __ge__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c >= 0 - - # -- precision tools ------------------------------------------ - - def _round(self, prec, rounding=None): - if self._int == 0: - return self - rounding = rounding or getcontext().rounding - digits = _digits(self._int) - excess = len(digits) - prec - if excess <= 0: - return self - kept = self._int // (10 ** excess) - rem = self._int % (10 ** excess) - threshold = 10 ** excess - if rounding == ROUND_DOWN: - new = kept - elif rounding == ROUND_UP: - new = kept + (1 if rem else 0) - elif rounding == ROUND_HALF_UP: - new = kept + (1 if rem * 2 >= threshold else 0) - elif rounding == ROUND_HALF_DOWN: - new = kept + (1 if rem * 2 > threshold else 0) - elif rounding == ROUND_HALF_EVEN: - doubled = rem * 2 - if doubled > threshold or (doubled == threshold and kept % 2): - new = kept + 1 - else: - new = kept - elif rounding == ROUND_FLOOR: - new = kept + (1 if (self._sign and rem) else 0) - elif rounding == ROUND_CEILING: - new = kept + (1 if (not self._sign and rem) else 0) - else: - new = kept - sign = self._sign - return Decimal((sign, _digits(new), self._exp + excess)) - - def quantize(self, exp, rounding=None): - if isinstance(exp, Decimal): - target = exp._exp - else: - target = int(exp) - if target < self._exp: - scale = self._exp - target - new_int = self._int * (10 ** scale) - return Decimal((self._sign, _digits(new_int), target)) - if target > self._exp: - shift = target - self._exp - scaled = self._int - divisor = 10 ** shift - quot, rem = divmod(scaled, divisor) - rounding = rounding or getcontext().rounding - doubled = rem * 2 - if rounding == ROUND_HALF_EVEN: - if doubled > divisor or (doubled == divisor and quot % 2): - quot += 1 - elif rounding == ROUND_HALF_UP: - if doubled >= divisor: - quot += 1 - elif rounding == ROUND_HALF_DOWN: - if doubled > divisor: - quot += 1 - elif rounding == ROUND_UP: - if rem != 0: - quot += 1 - elif rounding == ROUND_FLOOR: - if self._sign and rem: - quot += 1 - elif rounding == ROUND_CEILING: - if not self._sign and rem: - quot += 1 - return Decimal((self._sign, _digits(quot), target)) - return Decimal(self) - - def normalize(self): - if self._int == 0: - return Decimal((self._sign, (0,), 0)) - n = self._int - e = self._exp - while n % 10 == 0: - n //= 10 - e += 1 - return Decimal((self._sign, _digits(n), e)) - - -def _coerce(other): - if isinstance(other, Decimal): - return other - if isinstance(other, int): - return Decimal(other) - return NotImplemented - - -def _digits(n): - if n == 0: - return (0,) - out = [] - while n: - out.append(n % 10) - n //= 10 - out.reverse() - return tuple(out) - - -__all__ = ["Decimal", "DecimalException", "InvalidOperation", - "DivisionByZero", "Inexact", "Rounded", "Subnormal", - "Overflow", "Underflow", "Clamped", "FloatOperation", - "ROUND_HALF_UP", "ROUND_HALF_EVEN", "ROUND_HALF_DOWN", - "ROUND_DOWN", "ROUND_UP", "ROUND_FLOOR", "ROUND_CEILING", - "ROUND_05UP", - "getcontext", "setcontext", "localcontext"] +try: + from _decimal import * + from _decimal import __version__ + from _decimal import __libmpdec_version__ +except ImportError: + import _pydecimal + import sys + _pydecimal.__doc__ = __doc__ + sys.modules[__name__] = _pydecimal diff --git a/crates/weavepy-vm/src/stdlib/python/doctest.py b/crates/weavepy-vm/src/stdlib/python/doctest.py index 8bc5dd7..aa2f84b 100644 --- a/crates/weavepy-vm/src/stdlib/python/doctest.py +++ b/crates/weavepy-vm/src/stdlib/python/doctest.py @@ -1204,8 +1204,13 @@ def DocFileTest(path, module_relative=True, package=None, globs=None, def DocFileSuite(*paths, **kw): suite = _DocTestSuite() + # We do this here so that _normalize_module is called at the right + # level. If it were called in DocFileTest, then this function would be + # the caller and we might guess the package incorrectly. Depth 2 (the + # default) names the caller of DocFileSuite — i.e. the module whose + # directory the relative paths resolve against. if kw.get('module_relative', True): - kw['package'] = _normalize_module(kw.get('package'), 3) + kw['package'] = _normalize_module(kw.get('package')) for path in paths: suite.addTest(DocFileTest(path, **kw)) return suite diff --git a/crates/weavepy-vm/src/stdlib/python/enum.py b/crates/weavepy-vm/src/stdlib/python/enum.py index f0d8520..49de907 100644 --- a/crates/weavepy-vm/src/stdlib/python/enum.py +++ b/crates/weavepy-vm/src/stdlib/python/enum.py @@ -386,6 +386,33 @@ def unique(enumeration): return enumeration +def global_enum_repr(self): + """`repr` that references the member's *module* rather than its class — + used for enums hoisted into a module namespace via :func:`global_enum` + (e.g. ``calendar.JANUARY``).""" + return f"{self.__class__.__module__}.{self._name_}" + + +def global_str(self): + cls_name = self.__class__.__name__ + return f"{cls_name}.{self._name_}" + + +def global_enum(cls, update_str=False): + """Class decorator that exports an enum's members into its defining + module's global namespace and switches member ``repr`` to the + module-qualified form (CPython's ``enum.global_enum``). ``IntEnum`` + keeps ``int``'s ``__str__`` unless ``update_str`` is set.""" + import sys + cls.__repr__ = global_enum_repr + if update_str: + cls.__str__ = global_str + module = sys.modules.get(cls.__module__) + if module is not None: + module.__dict__.update(cls.__members__) + return cls + + __all__ = [ "auto", "EnumMeta", @@ -395,4 +422,5 @@ def unique(enumeration): "Flag", "IntFlag", "unique", + "global_enum", ] diff --git a/crates/weavepy-vm/src/stdlib/python/fractions.py b/crates/weavepy-vm/src/stdlib/python/fractions.py index df8b5c8..9d42e80 100644 --- a/crates/weavepy-vm/src/stdlib/python/fractions.py +++ b/crates/weavepy-vm/src/stdlib/python/fractions.py @@ -1,299 +1,1043 @@ -"""Public ``fractions`` module (RFC 0019). +# Originally contributed by Sjoerd Mullender. +# Significantly modified by Jeffrey Yasskin . -Implements rational arithmetic on top of WeavePy's arbitrary-precision -``int`` type. The surface mirrors CPython's :class:`fractions.Fraction` -for the universally-used operations: parsing, arithmetic, comparisons, -``limit_denominator``, ``__hash__``, and ``as_integer_ratio``. -""" +"""Fraction, infinite-precision, rational numbers.""" +from decimal import Decimal +import functools import math +import numbers +import operator import re +import sys -_PATTERN = re.compile( - r"\A\s*" - r"(?P[-+])?" - r"(?P\d+)" - r"(?:/(?P\d+))?" - r"\s*\Z" -) +__all__ = ['Fraction'] -def _gcd(a, b): - if a < 0: - a = -a - if b < 0: - b = -b - while b: - a, b = b, a % b - return a +# Constants related to the hash implementation; hash(x) is based +# on the reduction of x modulo the prime _PyHASH_MODULUS. +_PyHASH_MODULUS = sys.hash_info.modulus +# Value to be used for rationals that reduce to infinity modulo +# _PyHASH_MODULUS. +_PyHASH_INF = sys.hash_info.inf +@functools.lru_cache(maxsize = 1 << 14) +def _hash_algorithm(numerator, denominator): -def _normalize(num, den): - if den == 0: - raise ZeroDivisionError("Fraction(%d, 0)" % num) - if den < 0: - num, den = -num, -den - g = _gcd(num, den) - if g != 1: - num //= g - den //= g - return num, den + # To make sure that the hash of a Fraction agrees with the hash + # of a numerically equal integer, float or Decimal instance, we + # follow the rules for numeric hashes outlined in the + # documentation. (See library docs, 'Built-in Types'). + try: + dinv = pow(denominator, -1, _PyHASH_MODULUS) + except ValueError: + # ValueError means there is no modular inverse. + hash_ = _PyHASH_INF + else: + # The general algorithm now specifies that the absolute value of + # the hash is + # (|N| * dinv) % P + # where N is self._numerator and P is _PyHASH_MODULUS. That's + # optimized here in two ways: first, for a non-negative int i, + # hash(i) == i % P, but the int hash implementation doesn't need + # to divide, and is faster than doing % P explicitly. So we do + # hash(|N| * dinv) + # instead. Second, N is unbounded, so its product with dinv may + # be arbitrarily expensive to compute. The final answer is the + # same if we use the bounded |N| % P instead, which can again + # be done with an int hash() call. If 0 <= i < P, hash(i) == i, + # so this nested hash() call wastes a bit of time making a + # redundant copy when |N| < P, but can save an arbitrarily large + # amount of computation for large |N|. + hash_ = hash(hash(abs(numerator)) * dinv) + result = hash_ if numerator >= 0 else -hash_ + return -2 if result == -1 else result -class Fraction: - """Rational number represented as ``num/den``.""" +_RATIONAL_FORMAT = re.compile(r""" + \A\s* # optional whitespace at the start, + (?P[-+]?) # an optional sign, then + (?=\d|\.\d) # lookahead for digit or .digit + (?P\d*|\d+(_\d+)*) # numerator (possibly empty) + (?: # followed by + (?:\s*/\s*(?P\d+(_\d+)*))? # an optional denominator + | # or + (?:\.(?P\d*|\d+(_\d+)*))? # an optional fractional part + (?:E(?P[-+]?\d+(_\d+)*))? # and optional exponent + ) + \s*\Z # and optional whitespace to finish +""", re.VERBOSE | re.IGNORECASE) - __slots__ = ("_numerator", "_denominator") - def __new__(cls, numerator=0, denominator=None, *, _normalize=True): - self = object.__new__(cls) +# Helpers for formatting + +def _round_to_exponent(n, d, exponent, no_neg_zero=False): + """Round a rational number to the nearest multiple of a given power of 10. + + Rounds the rational number n/d to the nearest integer multiple of + 10**exponent, rounding to the nearest even integer multiple in the case of + a tie. Returns a pair (sign: bool, significand: int) representing the + rounded value (-1)**sign * significand * 10**exponent. + + If no_neg_zero is true, then the returned sign will always be False when + the significand is zero. Otherwise, the sign reflects the sign of the + input. + + d must be positive, but n and d need not be relatively prime. + """ + if exponent >= 0: + d *= 10**exponent + else: + n *= 10**-exponent + + # The divmod quotient is correct for round-ties-towards-positive-infinity; + # In the case of a tie, we zero out the least significant bit of q. + q, r = divmod(n + (d >> 1), d) + if r == 0 and d & 1 == 0: + q &= -2 + + sign = q < 0 if no_neg_zero else n < 0 + return sign, abs(q) + + +def _round_to_figures(n, d, figures): + """Round a rational number to a given number of significant figures. + + Rounds the rational number n/d to the given number of significant figures + using the round-ties-to-even rule, and returns a triple + (sign: bool, significand: int, exponent: int) representing the rounded + value (-1)**sign * significand * 10**exponent. + + In the special case where n = 0, returns a significand of zero and + an exponent of 1 - figures, for compatibility with formatting. + Otherwise, the returned significand satisfies + 10**(figures - 1) <= significand < 10**figures. + + d must be positive, but n and d need not be relatively prime. + figures must be positive. + """ + # Special case for n == 0. + if n == 0: + return False, 0, 1 - figures + + # Find integer m satisfying 10**(m - 1) <= abs(n)/d <= 10**m. (If abs(n)/d + # is a power of 10, either of the two possible values for m is fine.) + str_n, str_d = str(abs(n)), str(d) + m = len(str_n) - len(str_d) + (str_d <= str_n) + + # Round to a multiple of 10**(m - figures). The significand we get + # satisfies 10**(figures - 1) <= significand <= 10**figures. + exponent = m - figures + sign, significand = _round_to_exponent(n, d, exponent) + + # Adjust in the case where significand == 10**figures, to ensure that + # 10**(figures - 1) <= significand < 10**figures. + if len(str(significand)) == figures + 1: + significand //= 10 + exponent += 1 + + return sign, significand, exponent + + +# Pattern for matching non-float-style format specifications. +_GENERAL_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" + (?: + (?P.)? + (?P[<>=^]) + )? + (?P[-+ ]?) + # Alt flag forces a slash and denominator in the output, even for + # integer-valued Fraction objects. + (?P\#)? + # We don't implement the zeropad flag since there's no single obvious way + # to interpret it. + (?P0|[1-9][0-9]*)? + (?P[,_])? +""", re.DOTALL | re.VERBOSE).fullmatch + + +# Pattern for matching float-style format specifications; +# supports 'e', 'E', 'f', 'F', 'g', 'G' and '%' presentation types. +_FLOAT_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" + (?: + (?P.)? + (?P[<>=^]) + )? + (?P[-+ ]?) + (?Pz)? + (?P\#)? + # A '0' that's *not* followed by another digit is parsed as a minimum width + # rather than a zeropad flag. + (?P0(?=[0-9]))? + (?P0|[1-9][0-9]*)? + (?P[,_])? + (?:\.(?P0|[1-9][0-9]*))? + (?P[eEfFgG%]) +""", re.DOTALL | re.VERBOSE).fullmatch + + +class Fraction(numbers.Rational): + """This class implements rational numbers. + + In the two-argument form of the constructor, Fraction(8, 6) will + produce a rational number equivalent to 4/3. Both arguments must + be Rational. The numerator defaults to 0 and the denominator + defaults to 1 so that Fraction(3) == 3 and Fraction() == 0. + + Fractions can also be constructed from: + + - numeric strings similar to those accepted by the + float constructor (for example, '-2.3' or '1e10') + + - strings of the form '123/456' + + - float and Decimal instances + + - other Rational instances (including integers) + + """ + + __slots__ = ('_numerator', '_denominator') + + # We're immutable, so use __new__ not __init__ + def __new__(cls, numerator=0, denominator=None): + """Constructs a Rational. + + Takes a string like '3/2' or '1.5', another Rational instance, a + numerator/denominator pair, or a float. + + Examples + -------- + + >>> Fraction(10, -8) + Fraction(-5, 4) + >>> Fraction(Fraction(1, 7), 5) + Fraction(1, 35) + >>> Fraction(Fraction(1, 7), Fraction(2, 3)) + Fraction(3, 14) + >>> Fraction('314') + Fraction(314, 1) + >>> Fraction('-35/4') + Fraction(-35, 4) + >>> Fraction('3.1415') # conversion from numeric string + Fraction(6283, 2000) + >>> Fraction('-47e-2') # string may include a decimal exponent + Fraction(-47, 100) + >>> Fraction(1.47) # direct construction from float (exact conversion) + Fraction(6620291452234629, 4503599627370496) + >>> Fraction(2.25) + Fraction(9, 4) + >>> Fraction(Decimal('1.47')) + Fraction(147, 100) + + """ + self = super(Fraction, cls).__new__(cls) + if denominator is None: - if isinstance(numerator, int): + if type(numerator) is int: self._numerator = numerator self._denominator = 1 return self - if isinstance(numerator, float): - if not math.isfinite(numerator): - raise OverflowError( - "cannot convert non-finite float to Fraction") - num, den = numerator.as_integer_ratio() - self._numerator = num - self._denominator = den - return self - if isinstance(numerator, str): - m = _PATTERN.match(numerator) - if not m: - raise ValueError("Invalid fraction literal: %r" % numerator) - num = int(m.group("num")) - if m.group("denom"): - den = int(m.group("denom")) - else: - den = 1 - if m.group("sign") == "-": - num = -num - num, den = _norm(num, den) - self._numerator = num - self._denominator = den + + elif isinstance(numerator, numbers.Rational): + self._numerator = numerator.numerator + self._denominator = numerator.denominator return self - if isinstance(numerator, Fraction): - self._numerator = numerator._numerator - self._denominator = numerator._denominator + + elif isinstance(numerator, (float, Decimal)): + # Exact conversion + self._numerator, self._denominator = numerator.as_integer_ratio() return self - raise TypeError( - "argument should be a string or a number, not %r" % - type(numerator).__name__) - if not isinstance(numerator, int) or not isinstance(denominator, int): - raise TypeError("both numerator and denominator must be int") - num, den = _norm(numerator, denominator) - self._numerator = num - self._denominator = den + + elif isinstance(numerator, str): + # Handle construction from strings. + m = _RATIONAL_FORMAT.match(numerator) + if m is None: + raise ValueError('Invalid literal for Fraction: %r' % + numerator) + numerator = int(m.group('num') or '0') + denom = m.group('denom') + if denom: + denominator = int(denom) + else: + denominator = 1 + decimal = m.group('decimal') + if decimal: + decimal = decimal.replace('_', '') + scale = 10**len(decimal) + numerator = numerator * scale + int(decimal) + denominator *= scale + exp = m.group('exp') + if exp: + exp = int(exp) + if exp >= 0: + numerator *= 10**exp + else: + denominator *= 10**-exp + if m.group('sign') == '-': + numerator = -numerator + + else: + raise TypeError("argument should be a string " + "or a Rational instance") + + elif type(numerator) is int is type(denominator): + pass # *very* normal case + + elif (isinstance(numerator, numbers.Rational) and + isinstance(denominator, numbers.Rational)): + numerator, denominator = ( + numerator.numerator * denominator.denominator, + denominator.numerator * numerator.denominator + ) + else: + raise TypeError("both arguments should be " + "Rational instances") + + if denominator == 0: + raise ZeroDivisionError('Fraction(%s, 0)' % numerator) + g = math.gcd(numerator, denominator) + if denominator < 0: + g = -g + numerator //= g + denominator //= g + self._numerator = numerator + self._denominator = denominator return self - # -- properties ------------------------------------------------ + @classmethod + def from_float(cls, f): + """Converts a finite float to a rational number, exactly. - @property - def numerator(self): - return self._numerator + Beware that Fraction.from_float(0.3) != Fraction(3, 10). + + """ + if isinstance(f, numbers.Integral): + return cls(f) + elif not isinstance(f, float): + raise TypeError("%s.from_float() only takes floats, not %r (%s)" % + (cls.__name__, f, type(f).__name__)) + return cls._from_coprime_ints(*f.as_integer_ratio()) + + @classmethod + def from_decimal(cls, dec): + """Converts a finite Decimal instance to a rational number, exactly.""" + from decimal import Decimal + if isinstance(dec, numbers.Integral): + dec = Decimal(int(dec)) + elif not isinstance(dec, Decimal): + raise TypeError( + "%s.from_decimal() only takes Decimals, not %r (%s)" % + (cls.__name__, dec, type(dec).__name__)) + return cls._from_coprime_ints(*dec.as_integer_ratio()) + + @classmethod + def _from_coprime_ints(cls, numerator, denominator, /): + """Convert a pair of ints to a rational number, for internal use. + + The ratio of integers should be in lowest terms and the denominator + should be positive. + """ + obj = super(Fraction, cls).__new__(cls) + obj._numerator = numerator + obj._denominator = denominator + return obj + + def is_integer(self): + """Return True if the Fraction is an integer.""" + return self._denominator == 1 + + def as_integer_ratio(self): + """Return a pair of integers, whose ratio is equal to the original Fraction. + + The ratio is in lowest terms and has a positive denominator. + """ + return (self._numerator, self._denominator) + + def limit_denominator(self, max_denominator=1000000): + """Closest Fraction to self with denominator at most max_denominator. + + >>> Fraction('3.141592653589793').limit_denominator(10) + Fraction(22, 7) + >>> Fraction('3.141592653589793').limit_denominator(100) + Fraction(311, 99) + >>> Fraction(4321, 8765).limit_denominator(10000) + Fraction(4321, 8765) + + """ + # Algorithm notes: For any real number x, define a *best upper + # approximation* to x to be a rational number p/q such that: + # + # (1) p/q >= x, and + # (2) if p/q > r/s >= x then s > q, for any rational r/s. + # + # Define *best lower approximation* similarly. Then it can be + # proved that a rational number is a best upper or lower + # approximation to x if, and only if, it is a convergent or + # semiconvergent of the (unique shortest) continued fraction + # associated to x. + # + # To find a best rational approximation with denominator <= M, + # we find the best upper and lower approximations with + # denominator <= M and take whichever of these is closer to x. + # In the event of a tie, the bound with smaller denominator is + # chosen. If both denominators are equal (which can happen + # only when max_denominator == 1 and self is midway between + # two integers) the lower bound---i.e., the floor of self, is + # taken. + + if max_denominator < 1: + raise ValueError("max_denominator should be at least 1") + if self._denominator <= max_denominator: + return Fraction(self) + + p0, q0, p1, q1 = 0, 1, 1, 0 + n, d = self._numerator, self._denominator + while True: + a = n//d + q2 = q0+a*q1 + if q2 > max_denominator: + break + p0, q0, p1, q1 = p1, q1, p0+a*p1, q2 + n, d = d, n-a*d + k = (max_denominator-q0)//q1 + + # Determine which of the candidates (p0+k*p1)/(q0+k*q1) and p1/q1 is + # closer to self. The distance between them is 1/(q1*(q0+k*q1)), while + # the distance from p1/q1 to self is d/(q1*self._denominator). So we + # need to compare 2*(q0+k*q1) with self._denominator/d. + if 2*d*(q0+k*q1) <= self._denominator: + return Fraction._from_coprime_ints(p1, q1) + else: + return Fraction._from_coprime_ints(p0+k*p1, q0+k*q1) @property - def denominator(self): - return self._denominator + def numerator(a): + return a._numerator - # -- conversions ---------------------------------------------- + @property + def denominator(a): + return a._denominator def __repr__(self): - return "Fraction(%d, %d)" % (self._numerator, self._denominator) + """repr(self)""" + return '%s(%s, %s)' % (self.__class__.__name__, + self._numerator, self._denominator) def __str__(self): + """str(self)""" if self._denominator == 1: return str(self._numerator) - return "%d/%d" % (self._numerator, self._denominator) + else: + return '%s/%s' % (self._numerator, self._denominator) - def __float__(self): - return self._numerator / self._denominator + def _format_general(self, match): + """Helper method for __format__. - def __int__(self): - if self._numerator < 0: - return -(-self._numerator // self._denominator) - return self._numerator // self._denominator + Handles fill, alignment, signs, and thousands separators in the + case of no presentation type. + """ + # Validate and parse the format specifier. + fill = match["fill"] or " " + align = match["align"] or ">" + pos_sign = "" if match["sign"] == "-" else match["sign"] + alternate_form = bool(match["alt"]) + minimumwidth = int(match["minimumwidth"] or "0") + thousands_sep = match["thousands_sep"] or '' - def __bool__(self): - return self._numerator != 0 + # Determine the body and sign representation. + n, d = self._numerator, self._denominator + if d > 1 or alternate_form: + body = f"{abs(n):{thousands_sep}}/{d:{thousands_sep}}" + else: + body = f"{abs(n):{thousands_sep}}" + sign = '-' if n < 0 else pos_sign - def __hash__(self): - return hash((self._numerator, self._denominator)) + # Pad with fill character if necessary and return. + padding = fill * (minimumwidth - len(sign) - len(body)) + if align == ">": + return padding + sign + body + elif align == "<": + return sign + body + padding + elif align == "^": + half = len(padding) // 2 + return padding[:half] + sign + body + padding[half:] + else: # align == "=" + return sign + padding + body - def as_integer_ratio(self): - return (self._numerator, self._denominator) + def _format_float_style(self, match): + """Helper method for __format__; handles float presentation types.""" + fill = match["fill"] or " " + align = match["align"] or ">" + pos_sign = "" if match["sign"] == "-" else match["sign"] + no_neg_zero = bool(match["no_neg_zero"]) + alternate_form = bool(match["alt"]) + zeropad = bool(match["zeropad"]) + minimumwidth = int(match["minimumwidth"] or "0") + thousands_sep = match["thousands_sep"] + precision = int(match["precision"] or "6") + presentation_type = match["presentation_type"] + trim_zeros = presentation_type in "gG" and not alternate_form + trim_point = not alternate_form + exponent_indicator = "E" if presentation_type in "EFG" else "e" - # -- arithmetic ------------------------------------------------ + if align == '=' and fill == '0': + zeropad = True - def _coerce(self, other): - if isinstance(other, Fraction): - return other - if isinstance(other, int): - return Fraction(other, 1) - if isinstance(other, float): - return Fraction(other) - return NotImplemented + # Round to get the digits we need, figure out where to place the point, + # and decide whether to use scientific notation. 'point_pos' is the + # relative to the _end_ of the digit string: that is, it's the number + # of digits that should follow the point. + if presentation_type in "fF%": + exponent = -precision + if presentation_type == "%": + exponent -= 2 + negative, significand = _round_to_exponent( + self._numerator, self._denominator, exponent, no_neg_zero) + scientific = False + point_pos = precision + else: # presentation_type in "eEgG" + figures = ( + max(precision, 1) + if presentation_type in "gG" + else precision + 1 + ) + negative, significand, exponent = _round_to_figures( + self._numerator, self._denominator, figures) + scientific = ( + presentation_type in "eE" + or exponent > 0 + or exponent + figures <= -4 + ) + point_pos = figures - 1 if scientific else -exponent - def __add__(self, other): - o = self._coerce(other) - if o is NotImplemented: - return NotImplemented - n = self._numerator * o._denominator + o._numerator * self._denominator - d = self._denominator * o._denominator - return Fraction(n, d) + # Get the suffix - the part following the digits, if any. + if presentation_type == "%": + suffix = "%" + elif scientific: + suffix = f"{exponent_indicator}{exponent + point_pos:+03d}" + else: + suffix = "" - __radd__ = __add__ + # String of output digits, padded sufficiently with zeros on the left + # so that we'll have at least one digit before the decimal point. + digits = f"{significand:0{point_pos + 1}d}" - def __sub__(self, other): - o = self._coerce(other) - if o is NotImplemented: - return NotImplemented - n = self._numerator * o._denominator - o._numerator * self._denominator - d = self._denominator * o._denominator - return Fraction(n, d) + # Before padding, the output has the form f"{sign}{leading}{trailing}", + # where `leading` includes thousands separators if necessary and + # `trailing` includes the decimal separator where appropriate. + sign = "-" if negative else pos_sign + leading = digits[: len(digits) - point_pos] + frac_part = digits[len(digits) - point_pos :] + if trim_zeros: + frac_part = frac_part.rstrip("0") + separator = "" if trim_point and not frac_part else "." + trailing = separator + frac_part + suffix - def __rsub__(self, other): - o = self._coerce(other) - if o is NotImplemented: - return NotImplemented - return o.__sub__(self) + # Do zero padding if required. + if zeropad: + min_leading = minimumwidth - len(sign) - len(trailing) + # When adding thousands separators, they'll be added to the + # zero-padded portion too, so we need to compensate. + leading = leading.zfill( + 3 * min_leading // 4 + 1 if thousands_sep else min_leading + ) - def __mul__(self, other): - o = self._coerce(other) - if o is NotImplemented: - return NotImplemented - return Fraction(self._numerator * o._numerator, - self._denominator * o._denominator) + # Insert thousands separators if required. + if thousands_sep: + first_pos = 1 + (len(leading) - 1) % 3 + leading = leading[:first_pos] + "".join( + thousands_sep + leading[pos : pos + 3] + for pos in range(first_pos, len(leading), 3) + ) - __rmul__ = __mul__ + # We now have a sign and a body. Pad with fill character if necessary + # and return. + body = leading + trailing + padding = fill * (minimumwidth - len(sign) - len(body)) + if align == ">": + return padding + sign + body + elif align == "<": + return sign + body + padding + elif align == "^": + half = len(padding) // 2 + return padding[:half] + sign + body + padding[half:] + else: # align == "=" + return sign + padding + body - def __truediv__(self, other): - o = self._coerce(other) - if o is NotImplemented: - return NotImplemented - if o._numerator == 0: - raise ZeroDivisionError("Fraction(%d, 0)" % self._numerator) - return Fraction(self._numerator * o._denominator, - self._denominator * o._numerator) - - def __rtruediv__(self, other): - o = self._coerce(other) - if o is NotImplemented: - return NotImplemented - return o.__truediv__(self) + def __format__(self, format_spec, /): + """Format this fraction according to the given format specification.""" - def __neg__(self): - return Fraction(-self._numerator, self._denominator) + if match := _GENERAL_FORMAT_SPECIFICATION_MATCHER(format_spec): + return self._format_general(match) - def __pos__(self): - return self + if match := _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec): + # Refuse the temptation to guess if both alignment _and_ + # zero padding are specified. + if match["align"] is None or match["zeropad"] is None: + return self._format_float_style(match) + + raise ValueError( + f"Invalid format specifier {format_spec!r} " + f"for object of type {type(self).__name__!r}" + ) + + def _operator_fallbacks(monomorphic_operator, fallback_operator, + handle_complex=True): + """Generates forward and reverse operators given a purely-rational + operator and a function from the operator module. + + Use this like: + __op__, __rop__ = _operator_fallbacks(just_rational_op, operator.op) - def __abs__(self): - return Fraction(abs(self._numerator), self._denominator) + In general, we want to implement the arithmetic operations so + that mixed-mode operations either call an implementation whose + author knew about the types of both arguments, or convert both + to the nearest built in type and do the operation there. In + Fraction, that means that we define __add__ and __radd__ as: - def __pow__(self, exp): - if isinstance(exp, int): - if exp >= 0: - return Fraction(self._numerator ** exp, - self._denominator ** exp) + def __add__(self, other): + # Both types have numerators/denominator attributes, + # so do the operation directly + if isinstance(other, (int, Fraction)): + return Fraction(self.numerator * other.denominator + + other.numerator * self.denominator, + self.denominator * other.denominator) + # float and complex don't have those operations, but we + # know about those types, so special case them. + elif isinstance(other, float): + return float(self) + other + elif isinstance(other, complex): + return complex(self) + other + # Let the other type take over. + return NotImplemented + + def __radd__(self, other): + # radd handles more types than add because there's + # nothing left to fall back to. + if isinstance(other, numbers.Rational): + return Fraction(self.numerator * other.denominator + + other.numerator * self.denominator, + self.denominator * other.denominator) + elif isinstance(other, Real): + return float(other) + float(self) + elif isinstance(other, Complex): + return complex(other) + complex(self) + return NotImplemented + + + There are 5 different cases for a mixed-type addition on + Fraction. I'll refer to all of the above code that doesn't + refer to Fraction, float, or complex as "boilerplate". 'r' + will be an instance of Fraction, which is a subtype of + Rational (r : Fraction <: Rational), and b : B <: + Complex. The first three involve 'r + b': + + 1. If B <: Fraction, int, float, or complex, we handle + that specially, and all is well. + 2. If Fraction falls back to the boilerplate code, and it + were to return a value from __add__, we'd miss the + possibility that B defines a more intelligent __radd__, + so the boilerplate should return NotImplemented from + __add__. In particular, we don't handle Rational + here, even though we could get an exact answer, in case + the other type wants to do something special. + 3. If B <: Fraction, Python tries B.__radd__ before + Fraction.__add__. This is ok, because it was + implemented with knowledge of Fraction, so it can + handle those instances before delegating to Real or + Complex. + + The next two situations describe 'b + r'. We assume that b + didn't know about Fraction in its implementation, and that it + uses similar boilerplate code: + + 4. If B <: Rational, then __radd_ converts both to the + builtin rational type (hey look, that's us) and + proceeds. + 5. Otherwise, __radd__ tries to find the nearest common + base ABC, and fall back to its builtin type. Since this + class doesn't subclass a concrete type, there's no + implementation to fall back to, so we need to try as + hard as possible to return an actual value, or the user + will get a TypeError. + + """ + def forward(a, b): + if isinstance(b, Fraction): + return monomorphic_operator(a, b) + elif isinstance(b, int): + return monomorphic_operator(a, Fraction(b)) + elif isinstance(b, float): + return fallback_operator(float(a), b) + elif handle_complex and isinstance(b, complex): + return fallback_operator(complex(a), b) else: - if self._numerator == 0: - raise ZeroDivisionError("0 ** negative") - return Fraction(self._denominator ** -exp, - self._numerator ** -exp) - if isinstance(exp, (Fraction, float)): - return float(self) ** float(exp) - return NotImplemented - - # -- comparisons ---------------------------------------------- - - def _cmp(self, other): - if isinstance(other, Fraction): - a = self._numerator * other._denominator - b = other._numerator * self._denominator - return (a > b) - (a < b) - if isinstance(other, int): - return self._cmp(Fraction(other)) - if isinstance(other, float): - if math.isfinite(other): - return self._cmp(Fraction(other)) - return -1 if other > 0 else 1 - return NotImplemented - - def __eq__(self, other): - if isinstance(other, Fraction): - return (self._numerator == other._numerator and - self._denominator == other._denominator) - if isinstance(other, int): - return self._numerator == other and self._denominator == 1 - if isinstance(other, float): - if math.isfinite(other): - return self == Fraction(other) - return False - return NotImplemented - - def __lt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c < 0 + return NotImplemented + forward.__name__ = '__' + fallback_operator.__name__ + '__' + forward.__doc__ = monomorphic_operator.__doc__ + + def reverse(b, a): + if isinstance(a, numbers.Rational): + # Includes ints. + return monomorphic_operator(Fraction(a), b) + elif isinstance(a, numbers.Real): + return fallback_operator(float(a), float(b)) + elif handle_complex and isinstance(a, numbers.Complex): + return fallback_operator(complex(a), complex(b)) + else: + return NotImplemented + reverse.__name__ = '__r' + fallback_operator.__name__ + '__' + reverse.__doc__ = monomorphic_operator.__doc__ + + return forward, reverse + + # Rational arithmetic algorithms: Knuth, TAOCP, Volume 2, 4.5.1. + # + # Assume input fractions a and b are normalized. + # + # 1) Consider addition/subtraction. + # + # Let g = gcd(da, db). Then + # + # na nb na*db ± nb*da + # a ± b == -- ± -- == ------------- == + # da db da*db + # + # na*(db//g) ± nb*(da//g) t + # == ----------------------- == - + # (da*db)//g d + # + # Now, if g > 1, we're working with smaller integers. + # + # Note, that t, (da//g) and (db//g) are pairwise coprime. + # + # Indeed, (da//g) and (db//g) share no common factors (they were + # removed) and da is coprime with na (since input fractions are + # normalized), hence (da//g) and na are coprime. By symmetry, + # (db//g) and nb are coprime too. Then, + # + # gcd(t, da//g) == gcd(na*(db//g), da//g) == 1 + # gcd(t, db//g) == gcd(nb*(da//g), db//g) == 1 + # + # Above allows us optimize reduction of the result to lowest + # terms. Indeed, + # + # g2 = gcd(t, d) == gcd(t, (da//g)*(db//g)*g) == gcd(t, g) + # + # t//g2 t//g2 + # a ± b == ----------------------- == ---------------- + # (da//g)*(db//g)*(g//g2) (da//g)*(db//g2) + # + # is a normalized fraction. This is useful because the unnormalized + # denominator d could be much larger than g. + # + # We should special-case g == 1 (and g2 == 1), since 60.8% of + # randomly-chosen integers are coprime: + # https://en.wikipedia.org/wiki/Coprime_integers#Probability_of_coprimality + # Note, that g2 == 1 always for fractions, obtained from floats: here + # g is a power of 2 and the unnormalized numerator t is an odd integer. + # + # 2) Consider multiplication + # + # Let g1 = gcd(na, db) and g2 = gcd(nb, da), then + # + # na*nb na*nb (na//g1)*(nb//g2) + # a*b == ----- == ----- == ----------------- + # da*db db*da (db//g1)*(da//g2) + # + # Note, that after divisions we're multiplying smaller integers. + # + # Also, the resulting fraction is normalized, because each of + # two factors in the numerator is coprime to each of the two factors + # in the denominator. + # + # Indeed, pick (na//g1). It's coprime with (da//g2), because input + # fractions are normalized. It's also coprime with (db//g1), because + # common factors are removed by g1 == gcd(na, db). + # + # As for addition/subtraction, we should special-case g1 == 1 + # and g2 == 1 for same reason. That happens also for multiplying + # rationals, obtained from floats. + + def _add(a, b): + """a + b""" + na, da = a._numerator, a._denominator + nb, db = b._numerator, b._denominator + g = math.gcd(da, db) + if g == 1: + return Fraction._from_coprime_ints(na * db + da * nb, da * db) + s = da // g + t = na * (db // g) + nb * s + g2 = math.gcd(t, g) + if g2 == 1: + return Fraction._from_coprime_ints(t, s * db) + return Fraction._from_coprime_ints(t // g2, s * (db // g2)) + + __add__, __radd__ = _operator_fallbacks(_add, operator.add) + + def _sub(a, b): + """a - b""" + na, da = a._numerator, a._denominator + nb, db = b._numerator, b._denominator + g = math.gcd(da, db) + if g == 1: + return Fraction._from_coprime_ints(na * db - da * nb, da * db) + s = da // g + t = na * (db // g) - nb * s + g2 = math.gcd(t, g) + if g2 == 1: + return Fraction._from_coprime_ints(t, s * db) + return Fraction._from_coprime_ints(t // g2, s * (db // g2)) + + __sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub) + + def _mul(a, b): + """a * b""" + na, da = a._numerator, a._denominator + nb, db = b._numerator, b._denominator + g1 = math.gcd(na, db) + if g1 > 1: + na //= g1 + db //= g1 + g2 = math.gcd(nb, da) + if g2 > 1: + nb //= g2 + da //= g2 + return Fraction._from_coprime_ints(na * nb, db * da) + + __mul__, __rmul__ = _operator_fallbacks(_mul, operator.mul) + + def _div(a, b): + """a / b""" + # Same as _mul(), with inversed b. + nb, db = b._numerator, b._denominator + if nb == 0: + raise ZeroDivisionError('Fraction(%s, 0)' % db) + na, da = a._numerator, a._denominator + g1 = math.gcd(na, nb) + if g1 > 1: + na //= g1 + nb //= g1 + g2 = math.gcd(db, da) + if g2 > 1: + da //= g2 + db //= g2 + n, d = na * db, nb * da + if d < 0: + n, d = -n, -d + return Fraction._from_coprime_ints(n, d) + + __truediv__, __rtruediv__ = _operator_fallbacks(_div, operator.truediv) + + def _floordiv(a, b): + """a // b""" + return (a.numerator * b.denominator) // (a.denominator * b.numerator) + + __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv, False) + + def _divmod(a, b): + """(a // b, a % b)""" + da, db = a.denominator, b.denominator + div, n_mod = divmod(a.numerator * db, da * b.numerator) + return div, Fraction(n_mod, da * db) + + __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod, False) + + def _mod(a, b): + """a % b""" + da, db = a.denominator, b.denominator + return Fraction((a.numerator * db) % (b.numerator * da), da * db) + + __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) + + def __pow__(a, b): + """a ** b + + If b is not an integer, the result will be a float or complex + since roots are generally irrational. If b is an integer, the + result will be rational. - def __le__(self, other): - c = self._cmp(other) - if c is NotImplemented: + """ + if isinstance(b, numbers.Rational): + if b.denominator == 1: + power = b.numerator + if power >= 0: + return Fraction._from_coprime_ints(a._numerator ** power, + a._denominator ** power) + elif a._numerator > 0: + return Fraction._from_coprime_ints(a._denominator ** -power, + a._numerator ** -power) + elif a._numerator == 0: + raise ZeroDivisionError('Fraction(%s, 0)' % + a._denominator ** -power) + else: + return Fraction._from_coprime_ints((-a._denominator) ** -power, + (-a._numerator) ** -power) + else: + # A fractional power will generally produce an + # irrational number. + return float(a) ** float(b) + elif isinstance(b, (float, complex)): + return float(a) ** b + else: return NotImplemented - return c <= 0 - def __gt__(self, other): - c = self._cmp(other) - if c is NotImplemented: + def __rpow__(b, a): + """a ** b""" + if b._denominator == 1 and b._numerator >= 0: + # If a is an int, keep it that way if possible. + return a ** b._numerator + + if isinstance(a, numbers.Rational): + return Fraction(a.numerator, a.denominator) ** b + + if b._denominator == 1: + return a ** b._numerator + + return a ** float(b) + + def __pos__(a): + """+a: Coerces a subclass instance to Fraction""" + return Fraction._from_coprime_ints(a._numerator, a._denominator) + + def __neg__(a): + """-a""" + return Fraction._from_coprime_ints(-a._numerator, a._denominator) + + def __abs__(a): + """abs(a)""" + return Fraction._from_coprime_ints(abs(a._numerator), a._denominator) + + def __int__(a, _index=operator.index): + """int(a)""" + if a._numerator < 0: + return _index(-(-a._numerator // a._denominator)) + else: + return _index(a._numerator // a._denominator) + + def __trunc__(a): + """math.trunc(a)""" + if a._numerator < 0: + return -(-a._numerator // a._denominator) + else: + return a._numerator // a._denominator + + def __floor__(a): + """math.floor(a)""" + return a._numerator // a._denominator + + def __ceil__(a): + """math.ceil(a)""" + # The negations cleverly convince floordiv to return the ceiling. + return -(-a._numerator // a._denominator) + + def __round__(self, ndigits=None): + """round(self, ndigits) + + Rounds half toward even. + """ + if ndigits is None: + d = self._denominator + floor, remainder = divmod(self._numerator, d) + if remainder * 2 < d: + return floor + elif remainder * 2 > d: + return floor + 1 + # Deal with the half case: + elif floor % 2 == 0: + return floor + else: + return floor + 1 + shift = 10**abs(ndigits) + # See _operator_fallbacks.forward to check that the results of + # these operations will always be Fraction and therefore have + # round(). + if ndigits > 0: + return Fraction(round(self * shift), shift) + else: + return Fraction(round(self / shift) * shift) + + def __hash__(self): + """hash(self)""" + return _hash_algorithm(self._numerator, self._denominator) + + def __eq__(a, b): + """a == b""" + if type(b) is int: + return a._numerator == b and a._denominator == 1 + if isinstance(b, numbers.Rational): + return (a._numerator == b.numerator and + a._denominator == b.denominator) + if isinstance(b, numbers.Complex) and b.imag == 0: + b = b.real + if isinstance(b, float): + if math.isnan(b) or math.isinf(b): + # comparisons with an infinity or nan should behave in + # the same way for any finite a, so treat a as zero. + return 0.0 == b + else: + return a == a.from_float(b) + else: + # Since a doesn't know how to compare with b, let's give b + # a chance to compare itself with a. return NotImplemented - return c > 0 - def __ge__(self, other): - c = self._cmp(other) - if c is NotImplemented: + def _richcmp(self, other, op): + """Helper for comparison operators, for internal use only. + + Implement comparison between a Rational instance `self`, and + either another Rational instance or a float `other`. If + `other` is not a Rational instance or a float, return + NotImplemented. `op` should be one of the six standard + comparison operators. + + """ + # convert other to a Rational instance where reasonable. + if isinstance(other, numbers.Rational): + return op(self._numerator * other.denominator, + self._denominator * other.numerator) + if isinstance(other, float): + if math.isnan(other) or math.isinf(other): + return op(0.0, other) + else: + return op(self, self.from_float(other)) + else: return NotImplemented - return c >= 0 - # -- approximation --------------------------------------------- + def __lt__(a, b): + """a < b""" + return a._richcmp(b, operator.lt) - def limit_denominator(self, max_denominator=1000000): - if max_denominator < 1: - raise ValueError("max_denominator should be at least 1") - if self._denominator <= max_denominator: - return Fraction(self._numerator, self._denominator) - p0, q0, p1, q1 = 0, 1, 1, 0 - n, d = self._numerator, self._denominator - while True: - a = n // d - q2 = q0 + a * q1 - if q2 > max_denominator: - break - p0, q0, p1, q1 = p1, q1, p0 + a * p1, q2 - n, d = d, n - a * d - k = (max_denominator - q0) // q1 - bound1 = Fraction(p0 + k * p1, q0 + k * q1) - bound2 = Fraction(p1, q1) - # Pick the bound that is closer to self. We compare distances - # via signed-difference squared to avoid invoking abs() on - # Fraction (the VM does not yet dispatch abs() into user - # __abs__; we work around it here). - d1 = bound1 - self - d2 = bound2 - self - if d1._numerator < 0: - d1 = Fraction(-d1._numerator, d1._denominator) - if d2._numerator < 0: - d2 = Fraction(-d2._numerator, d2._denominator) - if d2 <= d1: - return bound2 - return bound1 - - -_norm = _normalize - - -__all__ = ["Fraction"] + def __gt__(a, b): + """a > b""" + return a._richcmp(b, operator.gt) + + def __le__(a, b): + """a <= b""" + return a._richcmp(b, operator.le) + + def __ge__(a, b): + """a >= b""" + return a._richcmp(b, operator.ge) + + def __bool__(a): + """a != 0""" + # bpo-39274: Use bool() because (a._numerator != 0) can return an + # object which is not a bool. + return bool(a._numerator) + + # support for pickling, copy, and deepcopy + + def __reduce__(self): + return (self.__class__, (self._numerator, self._denominator)) + + def __copy__(self): + if type(self) == Fraction: + return self # I'm immutable; therefore I am my own clone + return self.__class__(self._numerator, self._denominator) + + def __deepcopy__(self, memo): + if type(self) == Fraction: + return self # My components are also immutable + return self.__class__(self._numerator, self._denominator) diff --git a/crates/weavepy-vm/src/stdlib/python/functools.py b/crates/weavepy-vm/src/stdlib/python/functools.py index 59c1898..0f39a49 100644 --- a/crates/weavepy-vm/src/stdlib/python/functools.py +++ b/crates/weavepy-vm/src/stdlib/python/functools.py @@ -13,6 +13,7 @@ "wraps", "update_wrapper", "cmp_to_key", + "total_ordering", "singledispatch", "cached_property", ] @@ -45,40 +46,90 @@ def reduce(function, iterable, *initial): class partial: - """Callable that pre-applies positional and keyword arguments.""" + """Callable that pre-applies positional and keyword arguments. + + `func` is positional-only (CPython's `partial.__new__(cls, func, /, + *args, **keywords)`): without that, a keyword named `func`/`self` + passed through to the wrapped callable — e.g. `operator.methodcaller`'s + pickle reduce builds `partial(methodcaller, name, self=..., name=...)` + — would collide with the constructor's own parameter and raise + "got multiple values for argument 'self'". + """ - def __init__(self, func, *args, **kwargs): + def __new__(cls, func, /, *args, **keywords): + if not callable(func): + raise TypeError("the first argument must be callable") if isinstance(func, partial): args = func.args + args - new_kwargs = dict(func.keywords) - new_kwargs.update(kwargs) - kwargs = new_kwargs + keywords = {**func.keywords, **keywords} func = func.func + self = super(partial, cls).__new__(cls) self.func = func self.args = args - self.keywords = kwargs + self.keywords = keywords + return self - def __call__(self, *args, **kwargs): - merged = dict(self.keywords) - merged.update(kwargs) - return self.func(*self.args, *args, **merged) + def __call__(self, /, *args, **keywords): + keywords = {**self.keywords, **keywords} + return self.func(*self.args, *args, **keywords) def __repr__(self): + qualname = type(self).__qualname__ + module = type(self).__module__ or "functools" parts = [repr(self.func)] for a in self.args: parts.append(repr(a)) for k, v in self.keywords.items(): parts.append(k + "=" + repr(v)) - return "functools.partial(" + ", ".join(parts) + ")" + return module + "." + qualname + "(" + ", ".join(parts) + ")" + + def __reduce__(self): + return ( + type(self), + (self.func,), + (self.func, self.args, self.keywords or None, self.__dict__ or None), + ) + + def __setstate__(self, state): + if not isinstance(state, tuple): + raise TypeError("argument to __setstate__ must be a tuple") + if len(state) != 4: + raise TypeError("expected 4 items in state, got %d" % len(state)) + func, args, kwds, namespace = state + if (not callable(func) or not isinstance(args, tuple) or + (kwds is not None and not isinstance(kwds, dict)) or + (namespace is not None and not isinstance(namespace, dict))): + raise TypeError("invalid partial state") + args = tuple(args) + if kwds is None: + kwds = {} + elif type(kwds) is not dict: + kwds = dict(kwds) + if namespace is None: + namespace = {} + self.__dict__ = namespace + self.func = func + self.args = args + self.keywords = kwds class partialmethod: """Descriptor form of :class:`partial` for methods.""" - def __init__(self, func, *args, **kwargs): - self.func = func - self.args = args - self.keywords = kwargs + def __init__(self, func, /, *args, **keywords): + # `func` is positional-only (PEP 570) so a wrapped callable may itself + # take `self`/`func` keyword arguments without colliding — matches + # CPython's `partialmethod.__init__` signature. + if isinstance(func, partialmethod): + # Flatten nested partialmethods so cls/self stay ahead of all + # other arguments and only one underlying call happens. + self.func = func.func + self.args = func.args + args + self.keywords = {**func.keywords, **keywords} + else: + self.func = func + self.args = args + self.keywords = keywords def __get__(self, instance, owner=None): if instance is None: @@ -267,6 +318,123 @@ def __ne__(self, other): return K +# ---- total_ordering ---------------------------------------------------------- +# Verbatim CPython 3.13: fills in the missing rich-comparison methods from a +# single defined one (RFC 0037 WS8 functools edges). + +def _gt_from_lt(self, other, NotImplemented=NotImplemented): + 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).' + op_result = type(self).__lt__(self, other) + if op_result is NotImplemented: + return op_result + return not op_result and self != other + +def _le_from_lt(self, other, NotImplemented=NotImplemented): + 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).' + op_result = type(self).__lt__(self, other) + if op_result is NotImplemented: + return op_result + return op_result or self == other + +def _ge_from_lt(self, other, NotImplemented=NotImplemented): + 'Return a >= b. Computed by @total_ordering from (not a < b).' + op_result = type(self).__lt__(self, other) + if op_result is NotImplemented: + return op_result + return not op_result + +def _ge_from_le(self, other, NotImplemented=NotImplemented): + 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).' + op_result = type(self).__le__(self, other) + if op_result is NotImplemented: + return op_result + return not op_result or self == other + +def _lt_from_le(self, other, NotImplemented=NotImplemented): + 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).' + op_result = type(self).__le__(self, other) + if op_result is NotImplemented: + return op_result + return op_result and self != other + +def _gt_from_le(self, other, NotImplemented=NotImplemented): + 'Return a > b. Computed by @total_ordering from (not a <= b).' + op_result = type(self).__le__(self, other) + if op_result is NotImplemented: + return op_result + return not op_result + +def _lt_from_gt(self, other, NotImplemented=NotImplemented): + 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).' + op_result = type(self).__gt__(self, other) + if op_result is NotImplemented: + return op_result + return not op_result and self != other + +def _ge_from_gt(self, other, NotImplemented=NotImplemented): + 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).' + op_result = type(self).__gt__(self, other) + if op_result is NotImplemented: + return op_result + return op_result or self == other + +def _le_from_gt(self, other, NotImplemented=NotImplemented): + 'Return a <= b. Computed by @total_ordering from (not a > b).' + op_result = type(self).__gt__(self, other) + if op_result is NotImplemented: + return op_result + return not op_result + +def _le_from_ge(self, other, NotImplemented=NotImplemented): + 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).' + op_result = type(self).__ge__(self, other) + if op_result is NotImplemented: + return op_result + return not op_result or self == other + +def _gt_from_ge(self, other, NotImplemented=NotImplemented): + 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).' + op_result = type(self).__ge__(self, other) + if op_result is NotImplemented: + return op_result + return op_result and self != other + +def _lt_from_ge(self, other, NotImplemented=NotImplemented): + 'Return a < b. Computed by @total_ordering from (not a >= b).' + op_result = type(self).__ge__(self, other) + if op_result is NotImplemented: + return op_result + return not op_result + +_convert = { + '__lt__': [('__gt__', _gt_from_lt), + ('__le__', _le_from_lt), + ('__ge__', _ge_from_lt)], + '__le__': [('__ge__', _ge_from_le), + ('__lt__', _lt_from_le), + ('__gt__', _gt_from_le)], + '__gt__': [('__lt__', _lt_from_gt), + ('__ge__', _ge_from_gt), + ('__le__', _le_from_gt)], + '__ge__': [('__le__', _le_from_ge), + ('__gt__', _gt_from_ge), + ('__lt__', _lt_from_ge)], +} + +def total_ordering(cls): + """Class decorator that fills in missing ordering methods""" + # Find user-defined comparisons (not those inherited from object). + roots = {op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)} + if not roots: + raise ValueError('must define at least one ordering operation: < > <= >=') + root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ + for opname, opfunc in _convert[root]: + if opname not in roots: + opfunc.__name__ = opname + setattr(cls, opname, opfunc) + return cls + + # ---- single-dispatch generic functions -------------------------------------- diff --git a/crates/weavepy-vm/src/stdlib/python/future_module.py b/crates/weavepy-vm/src/stdlib/python/future_module.py new file mode 100644 index 0000000..39720a5 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/future_module.py @@ -0,0 +1,147 @@ +"""Record of phased-in incompatible language changes. + +Each line is of the form: + + FeatureName = "_Feature(" OptionalRelease "," MandatoryRelease "," + CompilerFlag ")" + +where, normally, OptionalRelease < MandatoryRelease, and both are 5-tuples +of the same form as sys.version_info: + + (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int + PY_MINOR_VERSION, # the 1; an int + PY_MICRO_VERSION, # the 0; an int + PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string + PY_RELEASE_SERIAL # the 3; an int + ) + +OptionalRelease records the first release in which + + from __future__ import FeatureName + +was accepted. + +In the case of MandatoryReleases that have not yet occurred, +MandatoryRelease predicts the release in which the feature will become part +of the language. + +Else MandatoryRelease records when the feature became part of the language; +in releases at or after that, modules no longer need + + from __future__ import FeatureName + +to use the feature in question, but may continue to use such imports. + +MandatoryRelease may also be None, meaning that a planned feature got +dropped or that the release version is undetermined. + +Instances of class _Feature have two corresponding methods, +.getOptionalRelease() and .getMandatoryRelease(). + +CompilerFlag is the (bitfield) flag that should be passed in the fourth +argument to the builtin function compile() to enable the feature in +dynamically compiled code. This flag is stored in the .compiler_flag +attribute on _Future instances. These values must match the appropriate +#defines of CO_xxx flags in Include/cpython/compile.h. + +No feature line is ever to be deleted from this file. +""" + +all_feature_names = [ + "nested_scopes", + "generators", + "division", + "absolute_import", + "with_statement", + "print_function", + "unicode_literals", + "barry_as_FLUFL", + "generator_stop", + "annotations", +] + +__all__ = ["all_feature_names"] + all_feature_names + +# The CO_xxx symbols are defined here under the same names defined in +# code.h and used by compile.h, so that an editor search will find them here. +# However, they're not exported in __all__, because they don't really belong to +# this module. +CO_NESTED = 0x0010 # nested_scopes +CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000) +CO_FUTURE_DIVISION = 0x20000 # division +CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default +CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement +CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function +CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals +CO_FUTURE_BARRY_AS_BDFL = 0x400000 +CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators +CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime + + +class _Feature: + + def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): + self.optional = optionalRelease + self.mandatory = mandatoryRelease + self.compiler_flag = compiler_flag + + def getOptionalRelease(self): + """Return first release in which this feature was recognized. + + This is a 5-tuple, of the same form as sys.version_info. + """ + return self.optional + + def getMandatoryRelease(self): + """Return release in which this feature will become mandatory. + + This is a 5-tuple, of the same form as sys.version_info, or, if + the feature was dropped, or the release date is undetermined, is None. + """ + return self.mandatory + + def __repr__(self): + return "_Feature" + repr((self.optional, + self.mandatory, + self.compiler_flag)) + + +nested_scopes = _Feature((2, 1, 0, "beta", 1), + (2, 2, 0, "alpha", 0), + CO_NESTED) + +generators = _Feature((2, 2, 0, "alpha", 1), + (2, 3, 0, "final", 0), + CO_GENERATOR_ALLOWED) + +division = _Feature((2, 2, 0, "alpha", 2), + (3, 0, 0, "alpha", 0), + CO_FUTURE_DIVISION) + +absolute_import = _Feature((2, 5, 0, "alpha", 1), + (3, 0, 0, "alpha", 0), + CO_FUTURE_ABSOLUTE_IMPORT) + +with_statement = _Feature((2, 5, 0, "alpha", 1), + (2, 6, 0, "alpha", 0), + CO_FUTURE_WITH_STATEMENT) + +print_function = _Feature((2, 6, 0, "alpha", 2), + (3, 0, 0, "alpha", 0), + CO_FUTURE_PRINT_FUNCTION) + +unicode_literals = _Feature((2, 6, 0, "alpha", 2), + (3, 0, 0, "alpha", 0), + CO_FUTURE_UNICODE_LITERALS) + +barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2), + (4, 0, 0, "alpha", 0), + CO_FUTURE_BARRY_AS_BDFL) + +generator_stop = _Feature((3, 5, 0, "beta", 1), + (3, 7, 0, "alpha", 0), + CO_FUTURE_GENERATOR_STOP) + +annotations = _Feature((3, 7, 0, "beta", 1), + None, + CO_FUTURE_ANNOTATIONS) diff --git a/crates/weavepy-vm/src/stdlib/python/html.py b/crates/weavepy-vm/src/stdlib/python/html.py index 21ec884..1543460 100644 --- a/crates/weavepy-vm/src/stdlib/python/html.py +++ b/crates/weavepy-vm/src/stdlib/python/html.py @@ -1,75 +1,132 @@ -"""WeavePy `html` — minimal escape / unescape helpers. - -Just the public top-level surface: `escape`, `unescape`. The -sub-packages `html.parser`, `html.entities` are intentionally not -shipped yet — the most-imported helpers here are the escape -functions. """ +General functions for HTML manipulation. +""" + +import re as _re +from html.entities import html5 as _html5 + + +__all__ = ['escape', 'unescape'] def escape(s, quote=True): - s = s.replace("&", "&") + """ + Replace special characters "&", "<" and ">" to HTML-safe sequences. + If the optional flag quote is true (the default), the quotation mark + characters, both double quote (") and single quote (') characters are also + translated. + """ + s = s.replace("&", "&") # Must be done first! s = s.replace("<", "<") s = s.replace(">", ">") if quote: s = s.replace('"', """) - s = s.replace("'", "'") + s = s.replace('\'', "'") return s -_NAMED = { - "amp": "&", - "lt": "<", - "gt": ">", - "quot": '"', - "apos": "'", - "nbsp": "\u00a0", - "copy": "\u00a9", - "reg": "\u00ae", - "deg": "\u00b0", - "trade": "\u2122", +# see https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state + +_invalid_charrefs = { + 0x00: '\ufffd', # REPLACEMENT CHARACTER + 0x0d: '\r', # CARRIAGE RETURN + 0x80: '\u20ac', # EURO SIGN + 0x81: '\x81', # + 0x82: '\u201a', # SINGLE LOW-9 QUOTATION MARK + 0x83: '\u0192', # LATIN SMALL LETTER F WITH HOOK + 0x84: '\u201e', # DOUBLE LOW-9 QUOTATION MARK + 0x85: '\u2026', # HORIZONTAL ELLIPSIS + 0x86: '\u2020', # DAGGER + 0x87: '\u2021', # DOUBLE DAGGER + 0x88: '\u02c6', # MODIFIER LETTER CIRCUMFLEX ACCENT + 0x89: '\u2030', # PER MILLE SIGN + 0x8a: '\u0160', # LATIN CAPITAL LETTER S WITH CARON + 0x8b: '\u2039', # SINGLE LEFT-POINTING ANGLE QUOTATION MARK + 0x8c: '\u0152', # LATIN CAPITAL LIGATURE OE + 0x8d: '\x8d', # + 0x8e: '\u017d', # LATIN CAPITAL LETTER Z WITH CARON + 0x8f: '\x8f', # + 0x90: '\x90', # + 0x91: '\u2018', # LEFT SINGLE QUOTATION MARK + 0x92: '\u2019', # RIGHT SINGLE QUOTATION MARK + 0x93: '\u201c', # LEFT DOUBLE QUOTATION MARK + 0x94: '\u201d', # RIGHT DOUBLE QUOTATION MARK + 0x95: '\u2022', # BULLET + 0x96: '\u2013', # EN DASH + 0x97: '\u2014', # EM DASH + 0x98: '\u02dc', # SMALL TILDE + 0x99: '\u2122', # TRADE MARK SIGN + 0x9a: '\u0161', # LATIN SMALL LETTER S WITH CARON + 0x9b: '\u203a', # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + 0x9c: '\u0153', # LATIN SMALL LIGATURE OE + 0x9d: '\x9d', # + 0x9e: '\u017e', # LATIN SMALL LETTER Z WITH CARON + 0x9f: '\u0178', # LATIN CAPITAL LETTER Y WITH DIAERESIS } +_invalid_codepoints = { + # 0x0001 to 0x0008 + 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, + # 0x000E to 0x001F + 0xe, 0xf, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, + 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + # 0x007F to 0x009F + 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + # 0xFDD0 to 0xFDEF + 0xfdd0, 0xfdd1, 0xfdd2, 0xfdd3, 0xfdd4, 0xfdd5, 0xfdd6, 0xfdd7, 0xfdd8, + 0xfdd9, 0xfdda, 0xfddb, 0xfddc, 0xfddd, 0xfdde, 0xfddf, 0xfde0, 0xfde1, + 0xfde2, 0xfde3, 0xfde4, 0xfde5, 0xfde6, 0xfde7, 0xfde8, 0xfde9, 0xfdea, + 0xfdeb, 0xfdec, 0xfded, 0xfdee, 0xfdef, + # others + 0xb, 0xfffe, 0xffff, 0x1fffe, 0x1ffff, 0x2fffe, 0x2ffff, 0x3fffe, 0x3ffff, + 0x4fffe, 0x4ffff, 0x5fffe, 0x5ffff, 0x6fffe, 0x6ffff, 0x7fffe, 0x7ffff, + 0x8fffe, 0x8ffff, 0x9fffe, 0x9ffff, 0xafffe, 0xaffff, 0xbfffe, 0xbffff, + 0xcfffe, 0xcffff, 0xdfffe, 0xdffff, 0xefffe, 0xeffff, 0xffffe, 0xfffff, + 0x10fffe, 0x10ffff +} -def unescape(s): - """Reverse `escape()` plus the common HTML entities.""" - # We walk the string manually since WeavePy's `re.sub` doesn't - # accept a callable `repl` yet. The state machine here is small - # enough that a hand-rolled loop is clear. - out = [] - i = 0 - n = len(s) - while i < n: - c = s[i] - if c != "&": - out.append(c) - i += 1 - continue - # Find the closing semicolon — bail out if absent. - semi = s.find(";", i + 1) - if semi == -1 or semi - i > 16: - out.append(c) - i += 1 - continue - body = s[i + 1:semi] - replaced = None - if body.startswith("#"): - try: - if body[1:2].lower() == "x": - replaced = chr(int(body[2:], 16)) - else: - replaced = chr(int(body[1:])) - except ValueError: - replaced = None + +def _replace_charref(s): + s = s.group(1) + if s[0] == '#': + # numeric charref + if s[1] in 'xX': + num = int(s[2:].rstrip(';'), 16) else: - replaced = _NAMED.get(body) - if replaced is None: - out.append(c) - i += 1 + num = int(s[1:].rstrip(';')) + if num in _invalid_charrefs: + return _invalid_charrefs[num] + if 0xD800 <= num <= 0xDFFF or num > 0x10FFFF: + return '\uFFFD' + if num in _invalid_codepoints: + return '' + return chr(num) + else: + # named charref + if s in _html5: + return _html5[s] + # find the longest matching name (as defined by the standard) + for x in range(len(s)-1, 1, -1): + if s[:x] in _html5: + return _html5[s[:x]] + s[x:] else: - out.append(replaced) - i = semi + 1 - return "".join(out) + return '&' + s -__all__ = ["escape", "unescape"] +_charref = _re.compile(r'&(#[0-9]+;?' + r'|#[xX][0-9a-fA-F]+;?' + r'|[^\t\n\f <&#;]{1,32};?)') + +def unescape(s): + """ + Convert all named and numeric character references (e.g. >, >, + &x3e;) in the string s to the corresponding unicode characters. + This function uses the rules defined by the HTML 5 standard + for both valid and invalid character references, and the list of + HTML 5 named character references defined in html.entities.html5. + """ + if '&' not in s: + return s + return _charref.sub(_replace_charref, s) diff --git a/crates/weavepy-vm/src/stdlib/python/html_entities.py b/crates/weavepy-vm/src/stdlib/python/html_entities.py new file mode 100644 index 0000000..eb6dc12 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/html_entities.py @@ -0,0 +1,2513 @@ +"""HTML character entity references.""" + +__all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs'] + + +# maps HTML4 entity name to the Unicode code point +name2codepoint = { + 'AElig': 0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1 + 'Aacute': 0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1 + 'Acirc': 0x00c2, # latin capital letter A with circumflex, U+00C2 ISOlat1 + 'Agrave': 0x00c0, # latin capital letter A with grave = latin capital letter A grave, U+00C0 ISOlat1 + 'Alpha': 0x0391, # greek capital letter alpha, U+0391 + 'Aring': 0x00c5, # latin capital letter A with ring above = latin capital letter A ring, U+00C5 ISOlat1 + 'Atilde': 0x00c3, # latin capital letter A with tilde, U+00C3 ISOlat1 + 'Auml': 0x00c4, # latin capital letter A with diaeresis, U+00C4 ISOlat1 + 'Beta': 0x0392, # greek capital letter beta, U+0392 + 'Ccedil': 0x00c7, # latin capital letter C with cedilla, U+00C7 ISOlat1 + 'Chi': 0x03a7, # greek capital letter chi, U+03A7 + 'Dagger': 0x2021, # double dagger, U+2021 ISOpub + 'Delta': 0x0394, # greek capital letter delta, U+0394 ISOgrk3 + 'ETH': 0x00d0, # latin capital letter ETH, U+00D0 ISOlat1 + 'Eacute': 0x00c9, # latin capital letter E with acute, U+00C9 ISOlat1 + 'Ecirc': 0x00ca, # latin capital letter E with circumflex, U+00CA ISOlat1 + 'Egrave': 0x00c8, # latin capital letter E with grave, U+00C8 ISOlat1 + 'Epsilon': 0x0395, # greek capital letter epsilon, U+0395 + 'Eta': 0x0397, # greek capital letter eta, U+0397 + 'Euml': 0x00cb, # latin capital letter E with diaeresis, U+00CB ISOlat1 + 'Gamma': 0x0393, # greek capital letter gamma, U+0393 ISOgrk3 + 'Iacute': 0x00cd, # latin capital letter I with acute, U+00CD ISOlat1 + 'Icirc': 0x00ce, # latin capital letter I with circumflex, U+00CE ISOlat1 + 'Igrave': 0x00cc, # latin capital letter I with grave, U+00CC ISOlat1 + 'Iota': 0x0399, # greek capital letter iota, U+0399 + 'Iuml': 0x00cf, # latin capital letter I with diaeresis, U+00CF ISOlat1 + 'Kappa': 0x039a, # greek capital letter kappa, U+039A + 'Lambda': 0x039b, # greek capital letter lambda, U+039B ISOgrk3 + 'Mu': 0x039c, # greek capital letter mu, U+039C + 'Ntilde': 0x00d1, # latin capital letter N with tilde, U+00D1 ISOlat1 + 'Nu': 0x039d, # greek capital letter nu, U+039D + 'OElig': 0x0152, # latin capital ligature OE, U+0152 ISOlat2 + 'Oacute': 0x00d3, # latin capital letter O with acute, U+00D3 ISOlat1 + 'Ocirc': 0x00d4, # latin capital letter O with circumflex, U+00D4 ISOlat1 + 'Ograve': 0x00d2, # latin capital letter O with grave, U+00D2 ISOlat1 + 'Omega': 0x03a9, # greek capital letter omega, U+03A9 ISOgrk3 + 'Omicron': 0x039f, # greek capital letter omicron, U+039F + 'Oslash': 0x00d8, # latin capital letter O with stroke = latin capital letter O slash, U+00D8 ISOlat1 + 'Otilde': 0x00d5, # latin capital letter O with tilde, U+00D5 ISOlat1 + 'Ouml': 0x00d6, # latin capital letter O with diaeresis, U+00D6 ISOlat1 + 'Phi': 0x03a6, # greek capital letter phi, U+03A6 ISOgrk3 + 'Pi': 0x03a0, # greek capital letter pi, U+03A0 ISOgrk3 + 'Prime': 0x2033, # double prime = seconds = inches, U+2033 ISOtech + 'Psi': 0x03a8, # greek capital letter psi, U+03A8 ISOgrk3 + 'Rho': 0x03a1, # greek capital letter rho, U+03A1 + 'Scaron': 0x0160, # latin capital letter S with caron, U+0160 ISOlat2 + 'Sigma': 0x03a3, # greek capital letter sigma, U+03A3 ISOgrk3 + 'THORN': 0x00de, # latin capital letter THORN, U+00DE ISOlat1 + 'Tau': 0x03a4, # greek capital letter tau, U+03A4 + 'Theta': 0x0398, # greek capital letter theta, U+0398 ISOgrk3 + 'Uacute': 0x00da, # latin capital letter U with acute, U+00DA ISOlat1 + 'Ucirc': 0x00db, # latin capital letter U with circumflex, U+00DB ISOlat1 + 'Ugrave': 0x00d9, # latin capital letter U with grave, U+00D9 ISOlat1 + 'Upsilon': 0x03a5, # greek capital letter upsilon, U+03A5 ISOgrk3 + 'Uuml': 0x00dc, # latin capital letter U with diaeresis, U+00DC ISOlat1 + 'Xi': 0x039e, # greek capital letter xi, U+039E ISOgrk3 + 'Yacute': 0x00dd, # latin capital letter Y with acute, U+00DD ISOlat1 + 'Yuml': 0x0178, # latin capital letter Y with diaeresis, U+0178 ISOlat2 + 'Zeta': 0x0396, # greek capital letter zeta, U+0396 + 'aacute': 0x00e1, # latin small letter a with acute, U+00E1 ISOlat1 + 'acirc': 0x00e2, # latin small letter a with circumflex, U+00E2 ISOlat1 + 'acute': 0x00b4, # acute accent = spacing acute, U+00B4 ISOdia + 'aelig': 0x00e6, # latin small letter ae = latin small ligature ae, U+00E6 ISOlat1 + 'agrave': 0x00e0, # latin small letter a with grave = latin small letter a grave, U+00E0 ISOlat1 + 'alefsym': 0x2135, # alef symbol = first transfinite cardinal, U+2135 NEW + 'alpha': 0x03b1, # greek small letter alpha, U+03B1 ISOgrk3 + 'amp': 0x0026, # ampersand, U+0026 ISOnum + 'and': 0x2227, # logical and = wedge, U+2227 ISOtech + 'ang': 0x2220, # angle, U+2220 ISOamso + 'aring': 0x00e5, # latin small letter a with ring above = latin small letter a ring, U+00E5 ISOlat1 + 'asymp': 0x2248, # almost equal to = asymptotic to, U+2248 ISOamsr + 'atilde': 0x00e3, # latin small letter a with tilde, U+00E3 ISOlat1 + 'auml': 0x00e4, # latin small letter a with diaeresis, U+00E4 ISOlat1 + 'bdquo': 0x201e, # double low-9 quotation mark, U+201E NEW + 'beta': 0x03b2, # greek small letter beta, U+03B2 ISOgrk3 + 'brvbar': 0x00a6, # broken bar = broken vertical bar, U+00A6 ISOnum + 'bull': 0x2022, # bullet = black small circle, U+2022 ISOpub + 'cap': 0x2229, # intersection = cap, U+2229 ISOtech + 'ccedil': 0x00e7, # latin small letter c with cedilla, U+00E7 ISOlat1 + 'cedil': 0x00b8, # cedilla = spacing cedilla, U+00B8 ISOdia + 'cent': 0x00a2, # cent sign, U+00A2 ISOnum + 'chi': 0x03c7, # greek small letter chi, U+03C7 ISOgrk3 + 'circ': 0x02c6, # modifier letter circumflex accent, U+02C6 ISOpub + 'clubs': 0x2663, # black club suit = shamrock, U+2663 ISOpub + 'cong': 0x2245, # approximately equal to, U+2245 ISOtech + 'copy': 0x00a9, # copyright sign, U+00A9 ISOnum + 'crarr': 0x21b5, # downwards arrow with corner leftwards = carriage return, U+21B5 NEW + 'cup': 0x222a, # union = cup, U+222A ISOtech + 'curren': 0x00a4, # currency sign, U+00A4 ISOnum + 'dArr': 0x21d3, # downwards double arrow, U+21D3 ISOamsa + 'dagger': 0x2020, # dagger, U+2020 ISOpub + 'darr': 0x2193, # downwards arrow, U+2193 ISOnum + 'deg': 0x00b0, # degree sign, U+00B0 ISOnum + 'delta': 0x03b4, # greek small letter delta, U+03B4 ISOgrk3 + 'diams': 0x2666, # black diamond suit, U+2666 ISOpub + 'divide': 0x00f7, # division sign, U+00F7 ISOnum + 'eacute': 0x00e9, # latin small letter e with acute, U+00E9 ISOlat1 + 'ecirc': 0x00ea, # latin small letter e with circumflex, U+00EA ISOlat1 + 'egrave': 0x00e8, # latin small letter e with grave, U+00E8 ISOlat1 + 'empty': 0x2205, # empty set = null set = diameter, U+2205 ISOamso + 'emsp': 0x2003, # em space, U+2003 ISOpub + 'ensp': 0x2002, # en space, U+2002 ISOpub + 'epsilon': 0x03b5, # greek small letter epsilon, U+03B5 ISOgrk3 + 'equiv': 0x2261, # identical to, U+2261 ISOtech + 'eta': 0x03b7, # greek small letter eta, U+03B7 ISOgrk3 + 'eth': 0x00f0, # latin small letter eth, U+00F0 ISOlat1 + 'euml': 0x00eb, # latin small letter e with diaeresis, U+00EB ISOlat1 + 'euro': 0x20ac, # euro sign, U+20AC NEW + 'exist': 0x2203, # there exists, U+2203 ISOtech + 'fnof': 0x0192, # latin small f with hook = function = florin, U+0192 ISOtech + 'forall': 0x2200, # for all, U+2200 ISOtech + 'frac12': 0x00bd, # vulgar fraction one half = fraction one half, U+00BD ISOnum + 'frac14': 0x00bc, # vulgar fraction one quarter = fraction one quarter, U+00BC ISOnum + 'frac34': 0x00be, # vulgar fraction three quarters = fraction three quarters, U+00BE ISOnum + 'frasl': 0x2044, # fraction slash, U+2044 NEW + 'gamma': 0x03b3, # greek small letter gamma, U+03B3 ISOgrk3 + 'ge': 0x2265, # greater-than or equal to, U+2265 ISOtech + 'gt': 0x003e, # greater-than sign, U+003E ISOnum + 'hArr': 0x21d4, # left right double arrow, U+21D4 ISOamsa + 'harr': 0x2194, # left right arrow, U+2194 ISOamsa + 'hearts': 0x2665, # black heart suit = valentine, U+2665 ISOpub + 'hellip': 0x2026, # horizontal ellipsis = three dot leader, U+2026 ISOpub + 'iacute': 0x00ed, # latin small letter i with acute, U+00ED ISOlat1 + 'icirc': 0x00ee, # latin small letter i with circumflex, U+00EE ISOlat1 + 'iexcl': 0x00a1, # inverted exclamation mark, U+00A1 ISOnum + 'igrave': 0x00ec, # latin small letter i with grave, U+00EC ISOlat1 + 'image': 0x2111, # blackletter capital I = imaginary part, U+2111 ISOamso + 'infin': 0x221e, # infinity, U+221E ISOtech + 'int': 0x222b, # integral, U+222B ISOtech + 'iota': 0x03b9, # greek small letter iota, U+03B9 ISOgrk3 + 'iquest': 0x00bf, # inverted question mark = turned question mark, U+00BF ISOnum + 'isin': 0x2208, # element of, U+2208 ISOtech + 'iuml': 0x00ef, # latin small letter i with diaeresis, U+00EF ISOlat1 + 'kappa': 0x03ba, # greek small letter kappa, U+03BA ISOgrk3 + 'lArr': 0x21d0, # leftwards double arrow, U+21D0 ISOtech + 'lambda': 0x03bb, # greek small letter lambda, U+03BB ISOgrk3 + 'lang': 0x2329, # left-pointing angle bracket = bra, U+2329 ISOtech + 'laquo': 0x00ab, # left-pointing double angle quotation mark = left pointing guillemet, U+00AB ISOnum + 'larr': 0x2190, # leftwards arrow, U+2190 ISOnum + 'lceil': 0x2308, # left ceiling = apl upstile, U+2308 ISOamsc + 'ldquo': 0x201c, # left double quotation mark, U+201C ISOnum + 'le': 0x2264, # less-than or equal to, U+2264 ISOtech + 'lfloor': 0x230a, # left floor = apl downstile, U+230A ISOamsc + 'lowast': 0x2217, # asterisk operator, U+2217 ISOtech + 'loz': 0x25ca, # lozenge, U+25CA ISOpub + 'lrm': 0x200e, # left-to-right mark, U+200E NEW RFC 2070 + 'lsaquo': 0x2039, # single left-pointing angle quotation mark, U+2039 ISO proposed + 'lsquo': 0x2018, # left single quotation mark, U+2018 ISOnum + 'lt': 0x003c, # less-than sign, U+003C ISOnum + 'macr': 0x00af, # macron = spacing macron = overline = APL overbar, U+00AF ISOdia + 'mdash': 0x2014, # em dash, U+2014 ISOpub + 'micro': 0x00b5, # micro sign, U+00B5 ISOnum + 'middot': 0x00b7, # middle dot = Georgian comma = Greek middle dot, U+00B7 ISOnum + 'minus': 0x2212, # minus sign, U+2212 ISOtech + 'mu': 0x03bc, # greek small letter mu, U+03BC ISOgrk3 + 'nabla': 0x2207, # nabla = backward difference, U+2207 ISOtech + 'nbsp': 0x00a0, # no-break space = non-breaking space, U+00A0 ISOnum + 'ndash': 0x2013, # en dash, U+2013 ISOpub + 'ne': 0x2260, # not equal to, U+2260 ISOtech + 'ni': 0x220b, # contains as member, U+220B ISOtech + 'not': 0x00ac, # not sign, U+00AC ISOnum + 'notin': 0x2209, # not an element of, U+2209 ISOtech + 'nsub': 0x2284, # not a subset of, U+2284 ISOamsn + 'ntilde': 0x00f1, # latin small letter n with tilde, U+00F1 ISOlat1 + 'nu': 0x03bd, # greek small letter nu, U+03BD ISOgrk3 + 'oacute': 0x00f3, # latin small letter o with acute, U+00F3 ISOlat1 + 'ocirc': 0x00f4, # latin small letter o with circumflex, U+00F4 ISOlat1 + 'oelig': 0x0153, # latin small ligature oe, U+0153 ISOlat2 + 'ograve': 0x00f2, # latin small letter o with grave, U+00F2 ISOlat1 + 'oline': 0x203e, # overline = spacing overscore, U+203E NEW + 'omega': 0x03c9, # greek small letter omega, U+03C9 ISOgrk3 + 'omicron': 0x03bf, # greek small letter omicron, U+03BF NEW + 'oplus': 0x2295, # circled plus = direct sum, U+2295 ISOamsb + 'or': 0x2228, # logical or = vee, U+2228 ISOtech + 'ordf': 0x00aa, # feminine ordinal indicator, U+00AA ISOnum + 'ordm': 0x00ba, # masculine ordinal indicator, U+00BA ISOnum + 'oslash': 0x00f8, # latin small letter o with stroke, = latin small letter o slash, U+00F8 ISOlat1 + 'otilde': 0x00f5, # latin small letter o with tilde, U+00F5 ISOlat1 + 'otimes': 0x2297, # circled times = vector product, U+2297 ISOamsb + 'ouml': 0x00f6, # latin small letter o with diaeresis, U+00F6 ISOlat1 + 'para': 0x00b6, # pilcrow sign = paragraph sign, U+00B6 ISOnum + 'part': 0x2202, # partial differential, U+2202 ISOtech + 'permil': 0x2030, # per mille sign, U+2030 ISOtech + 'perp': 0x22a5, # up tack = orthogonal to = perpendicular, U+22A5 ISOtech + 'phi': 0x03c6, # greek small letter phi, U+03C6 ISOgrk3 + 'pi': 0x03c0, # greek small letter pi, U+03C0 ISOgrk3 + 'piv': 0x03d6, # greek pi symbol, U+03D6 ISOgrk3 + 'plusmn': 0x00b1, # plus-minus sign = plus-or-minus sign, U+00B1 ISOnum + 'pound': 0x00a3, # pound sign, U+00A3 ISOnum + 'prime': 0x2032, # prime = minutes = feet, U+2032 ISOtech + 'prod': 0x220f, # n-ary product = product sign, U+220F ISOamsb + 'prop': 0x221d, # proportional to, U+221D ISOtech + 'psi': 0x03c8, # greek small letter psi, U+03C8 ISOgrk3 + 'quot': 0x0022, # quotation mark = APL quote, U+0022 ISOnum + 'rArr': 0x21d2, # rightwards double arrow, U+21D2 ISOtech + 'radic': 0x221a, # square root = radical sign, U+221A ISOtech + 'rang': 0x232a, # right-pointing angle bracket = ket, U+232A ISOtech + 'raquo': 0x00bb, # right-pointing double angle quotation mark = right pointing guillemet, U+00BB ISOnum + 'rarr': 0x2192, # rightwards arrow, U+2192 ISOnum + 'rceil': 0x2309, # right ceiling, U+2309 ISOamsc + 'rdquo': 0x201d, # right double quotation mark, U+201D ISOnum + 'real': 0x211c, # blackletter capital R = real part symbol, U+211C ISOamso + 'reg': 0x00ae, # registered sign = registered trade mark sign, U+00AE ISOnum + 'rfloor': 0x230b, # right floor, U+230B ISOamsc + 'rho': 0x03c1, # greek small letter rho, U+03C1 ISOgrk3 + 'rlm': 0x200f, # right-to-left mark, U+200F NEW RFC 2070 + 'rsaquo': 0x203a, # single right-pointing angle quotation mark, U+203A ISO proposed + 'rsquo': 0x2019, # right single quotation mark, U+2019 ISOnum + 'sbquo': 0x201a, # single low-9 quotation mark, U+201A NEW + 'scaron': 0x0161, # latin small letter s with caron, U+0161 ISOlat2 + 'sdot': 0x22c5, # dot operator, U+22C5 ISOamsb + 'sect': 0x00a7, # section sign, U+00A7 ISOnum + 'shy': 0x00ad, # soft hyphen = discretionary hyphen, U+00AD ISOnum + 'sigma': 0x03c3, # greek small letter sigma, U+03C3 ISOgrk3 + 'sigmaf': 0x03c2, # greek small letter final sigma, U+03C2 ISOgrk3 + 'sim': 0x223c, # tilde operator = varies with = similar to, U+223C ISOtech + 'spades': 0x2660, # black spade suit, U+2660 ISOpub + 'sub': 0x2282, # subset of, U+2282 ISOtech + 'sube': 0x2286, # subset of or equal to, U+2286 ISOtech + 'sum': 0x2211, # n-ary summation, U+2211 ISOamsb + 'sup': 0x2283, # superset of, U+2283 ISOtech + 'sup1': 0x00b9, # superscript one = superscript digit one, U+00B9 ISOnum + 'sup2': 0x00b2, # superscript two = superscript digit two = squared, U+00B2 ISOnum + 'sup3': 0x00b3, # superscript three = superscript digit three = cubed, U+00B3 ISOnum + 'supe': 0x2287, # superset of or equal to, U+2287 ISOtech + 'szlig': 0x00df, # latin small letter sharp s = ess-zed, U+00DF ISOlat1 + 'tau': 0x03c4, # greek small letter tau, U+03C4 ISOgrk3 + 'there4': 0x2234, # therefore, U+2234 ISOtech + 'theta': 0x03b8, # greek small letter theta, U+03B8 ISOgrk3 + 'thetasym': 0x03d1, # greek small letter theta symbol, U+03D1 NEW + 'thinsp': 0x2009, # thin space, U+2009 ISOpub + 'thorn': 0x00fe, # latin small letter thorn with, U+00FE ISOlat1 + 'tilde': 0x02dc, # small tilde, U+02DC ISOdia + 'times': 0x00d7, # multiplication sign, U+00D7 ISOnum + 'trade': 0x2122, # trade mark sign, U+2122 ISOnum + 'uArr': 0x21d1, # upwards double arrow, U+21D1 ISOamsa + 'uacute': 0x00fa, # latin small letter u with acute, U+00FA ISOlat1 + 'uarr': 0x2191, # upwards arrow, U+2191 ISOnum + 'ucirc': 0x00fb, # latin small letter u with circumflex, U+00FB ISOlat1 + 'ugrave': 0x00f9, # latin small letter u with grave, U+00F9 ISOlat1 + 'uml': 0x00a8, # diaeresis = spacing diaeresis, U+00A8 ISOdia + 'upsih': 0x03d2, # greek upsilon with hook symbol, U+03D2 NEW + 'upsilon': 0x03c5, # greek small letter upsilon, U+03C5 ISOgrk3 + 'uuml': 0x00fc, # latin small letter u with diaeresis, U+00FC ISOlat1 + 'weierp': 0x2118, # script capital P = power set = Weierstrass p, U+2118 ISOamso + 'xi': 0x03be, # greek small letter xi, U+03BE ISOgrk3 + 'yacute': 0x00fd, # latin small letter y with acute, U+00FD ISOlat1 + 'yen': 0x00a5, # yen sign = yuan sign, U+00A5 ISOnum + 'yuml': 0x00ff, # latin small letter y with diaeresis, U+00FF ISOlat1 + 'zeta': 0x03b6, # greek small letter zeta, U+03B6 ISOgrk3 + 'zwj': 0x200d, # zero width joiner, U+200D NEW RFC 2070 + 'zwnj': 0x200c, # zero width non-joiner, U+200C NEW RFC 2070 +} + + +# HTML5 named character references +# Generated by Tools/build/parse_html5_entities.py +# from https://html.spec.whatwg.org/entities.json and +# https://html.spec.whatwg.org/multipage/named-characters.html. +# Map HTML5 named character references to the equivalent Unicode character(s). +html5 = { + 'Aacute': '\xc1', + 'aacute': '\xe1', + 'Aacute;': '\xc1', + 'aacute;': '\xe1', + 'Abreve;': '\u0102', + 'abreve;': '\u0103', + 'ac;': '\u223e', + 'acd;': '\u223f', + 'acE;': '\u223e\u0333', + 'Acirc': '\xc2', + 'acirc': '\xe2', + 'Acirc;': '\xc2', + 'acirc;': '\xe2', + 'acute': '\xb4', + 'acute;': '\xb4', + 'Acy;': '\u0410', + 'acy;': '\u0430', + 'AElig': '\xc6', + 'aelig': '\xe6', + 'AElig;': '\xc6', + 'aelig;': '\xe6', + 'af;': '\u2061', + 'Afr;': '\U0001d504', + 'afr;': '\U0001d51e', + 'Agrave': '\xc0', + 'agrave': '\xe0', + 'Agrave;': '\xc0', + 'agrave;': '\xe0', + 'alefsym;': '\u2135', + 'aleph;': '\u2135', + 'Alpha;': '\u0391', + 'alpha;': '\u03b1', + 'Amacr;': '\u0100', + 'amacr;': '\u0101', + 'amalg;': '\u2a3f', + 'AMP': '&', + 'amp': '&', + 'AMP;': '&', + 'amp;': '&', + 'And;': '\u2a53', + 'and;': '\u2227', + 'andand;': '\u2a55', + 'andd;': '\u2a5c', + 'andslope;': '\u2a58', + 'andv;': '\u2a5a', + 'ang;': '\u2220', + 'ange;': '\u29a4', + 'angle;': '\u2220', + 'angmsd;': '\u2221', + 'angmsdaa;': '\u29a8', + 'angmsdab;': '\u29a9', + 'angmsdac;': '\u29aa', + 'angmsdad;': '\u29ab', + 'angmsdae;': '\u29ac', + 'angmsdaf;': '\u29ad', + 'angmsdag;': '\u29ae', + 'angmsdah;': '\u29af', + 'angrt;': '\u221f', + 'angrtvb;': '\u22be', + 'angrtvbd;': '\u299d', + 'angsph;': '\u2222', + 'angst;': '\xc5', + 'angzarr;': '\u237c', + 'Aogon;': '\u0104', + 'aogon;': '\u0105', + 'Aopf;': '\U0001d538', + 'aopf;': '\U0001d552', + 'ap;': '\u2248', + 'apacir;': '\u2a6f', + 'apE;': '\u2a70', + 'ape;': '\u224a', + 'apid;': '\u224b', + 'apos;': "'", + 'ApplyFunction;': '\u2061', + 'approx;': '\u2248', + 'approxeq;': '\u224a', + 'Aring': '\xc5', + 'aring': '\xe5', + 'Aring;': '\xc5', + 'aring;': '\xe5', + 'Ascr;': '\U0001d49c', + 'ascr;': '\U0001d4b6', + 'Assign;': '\u2254', + 'ast;': '*', + 'asymp;': '\u2248', + 'asympeq;': '\u224d', + 'Atilde': '\xc3', + 'atilde': '\xe3', + 'Atilde;': '\xc3', + 'atilde;': '\xe3', + 'Auml': '\xc4', + 'auml': '\xe4', + 'Auml;': '\xc4', + 'auml;': '\xe4', + 'awconint;': '\u2233', + 'awint;': '\u2a11', + 'backcong;': '\u224c', + 'backepsilon;': '\u03f6', + 'backprime;': '\u2035', + 'backsim;': '\u223d', + 'backsimeq;': '\u22cd', + 'Backslash;': '\u2216', + 'Barv;': '\u2ae7', + 'barvee;': '\u22bd', + 'Barwed;': '\u2306', + 'barwed;': '\u2305', + 'barwedge;': '\u2305', + 'bbrk;': '\u23b5', + 'bbrktbrk;': '\u23b6', + 'bcong;': '\u224c', + 'Bcy;': '\u0411', + 'bcy;': '\u0431', + 'bdquo;': '\u201e', + 'becaus;': '\u2235', + 'Because;': '\u2235', + 'because;': '\u2235', + 'bemptyv;': '\u29b0', + 'bepsi;': '\u03f6', + 'bernou;': '\u212c', + 'Bernoullis;': '\u212c', + 'Beta;': '\u0392', + 'beta;': '\u03b2', + 'beth;': '\u2136', + 'between;': '\u226c', + 'Bfr;': '\U0001d505', + 'bfr;': '\U0001d51f', + 'bigcap;': '\u22c2', + 'bigcirc;': '\u25ef', + 'bigcup;': '\u22c3', + 'bigodot;': '\u2a00', + 'bigoplus;': '\u2a01', + 'bigotimes;': '\u2a02', + 'bigsqcup;': '\u2a06', + 'bigstar;': '\u2605', + 'bigtriangledown;': '\u25bd', + 'bigtriangleup;': '\u25b3', + 'biguplus;': '\u2a04', + 'bigvee;': '\u22c1', + 'bigwedge;': '\u22c0', + 'bkarow;': '\u290d', + 'blacklozenge;': '\u29eb', + 'blacksquare;': '\u25aa', + 'blacktriangle;': '\u25b4', + 'blacktriangledown;': '\u25be', + 'blacktriangleleft;': '\u25c2', + 'blacktriangleright;': '\u25b8', + 'blank;': '\u2423', + 'blk12;': '\u2592', + 'blk14;': '\u2591', + 'blk34;': '\u2593', + 'block;': '\u2588', + 'bne;': '=\u20e5', + 'bnequiv;': '\u2261\u20e5', + 'bNot;': '\u2aed', + 'bnot;': '\u2310', + 'Bopf;': '\U0001d539', + 'bopf;': '\U0001d553', + 'bot;': '\u22a5', + 'bottom;': '\u22a5', + 'bowtie;': '\u22c8', + 'boxbox;': '\u29c9', + 'boxDL;': '\u2557', + 'boxDl;': '\u2556', + 'boxdL;': '\u2555', + 'boxdl;': '\u2510', + 'boxDR;': '\u2554', + 'boxDr;': '\u2553', + 'boxdR;': '\u2552', + 'boxdr;': '\u250c', + 'boxH;': '\u2550', + 'boxh;': '\u2500', + 'boxHD;': '\u2566', + 'boxHd;': '\u2564', + 'boxhD;': '\u2565', + 'boxhd;': '\u252c', + 'boxHU;': '\u2569', + 'boxHu;': '\u2567', + 'boxhU;': '\u2568', + 'boxhu;': '\u2534', + 'boxminus;': '\u229f', + 'boxplus;': '\u229e', + 'boxtimes;': '\u22a0', + 'boxUL;': '\u255d', + 'boxUl;': '\u255c', + 'boxuL;': '\u255b', + 'boxul;': '\u2518', + 'boxUR;': '\u255a', + 'boxUr;': '\u2559', + 'boxuR;': '\u2558', + 'boxur;': '\u2514', + 'boxV;': '\u2551', + 'boxv;': '\u2502', + 'boxVH;': '\u256c', + 'boxVh;': '\u256b', + 'boxvH;': '\u256a', + 'boxvh;': '\u253c', + 'boxVL;': '\u2563', + 'boxVl;': '\u2562', + 'boxvL;': '\u2561', + 'boxvl;': '\u2524', + 'boxVR;': '\u2560', + 'boxVr;': '\u255f', + 'boxvR;': '\u255e', + 'boxvr;': '\u251c', + 'bprime;': '\u2035', + 'Breve;': '\u02d8', + 'breve;': '\u02d8', + 'brvbar': '\xa6', + 'brvbar;': '\xa6', + 'Bscr;': '\u212c', + 'bscr;': '\U0001d4b7', + 'bsemi;': '\u204f', + 'bsim;': '\u223d', + 'bsime;': '\u22cd', + 'bsol;': '\\', + 'bsolb;': '\u29c5', + 'bsolhsub;': '\u27c8', + 'bull;': '\u2022', + 'bullet;': '\u2022', + 'bump;': '\u224e', + 'bumpE;': '\u2aae', + 'bumpe;': '\u224f', + 'Bumpeq;': '\u224e', + 'bumpeq;': '\u224f', + 'Cacute;': '\u0106', + 'cacute;': '\u0107', + 'Cap;': '\u22d2', + 'cap;': '\u2229', + 'capand;': '\u2a44', + 'capbrcup;': '\u2a49', + 'capcap;': '\u2a4b', + 'capcup;': '\u2a47', + 'capdot;': '\u2a40', + 'CapitalDifferentialD;': '\u2145', + 'caps;': '\u2229\ufe00', + 'caret;': '\u2041', + 'caron;': '\u02c7', + 'Cayleys;': '\u212d', + 'ccaps;': '\u2a4d', + 'Ccaron;': '\u010c', + 'ccaron;': '\u010d', + 'Ccedil': '\xc7', + 'ccedil': '\xe7', + 'Ccedil;': '\xc7', + 'ccedil;': '\xe7', + 'Ccirc;': '\u0108', + 'ccirc;': '\u0109', + 'Cconint;': '\u2230', + 'ccups;': '\u2a4c', + 'ccupssm;': '\u2a50', + 'Cdot;': '\u010a', + 'cdot;': '\u010b', + 'cedil': '\xb8', + 'cedil;': '\xb8', + 'Cedilla;': '\xb8', + 'cemptyv;': '\u29b2', + 'cent': '\xa2', + 'cent;': '\xa2', + 'CenterDot;': '\xb7', + 'centerdot;': '\xb7', + 'Cfr;': '\u212d', + 'cfr;': '\U0001d520', + 'CHcy;': '\u0427', + 'chcy;': '\u0447', + 'check;': '\u2713', + 'checkmark;': '\u2713', + 'Chi;': '\u03a7', + 'chi;': '\u03c7', + 'cir;': '\u25cb', + 'circ;': '\u02c6', + 'circeq;': '\u2257', + 'circlearrowleft;': '\u21ba', + 'circlearrowright;': '\u21bb', + 'circledast;': '\u229b', + 'circledcirc;': '\u229a', + 'circleddash;': '\u229d', + 'CircleDot;': '\u2299', + 'circledR;': '\xae', + 'circledS;': '\u24c8', + 'CircleMinus;': '\u2296', + 'CirclePlus;': '\u2295', + 'CircleTimes;': '\u2297', + 'cirE;': '\u29c3', + 'cire;': '\u2257', + 'cirfnint;': '\u2a10', + 'cirmid;': '\u2aef', + 'cirscir;': '\u29c2', + 'ClockwiseContourIntegral;': '\u2232', + 'CloseCurlyDoubleQuote;': '\u201d', + 'CloseCurlyQuote;': '\u2019', + 'clubs;': '\u2663', + 'clubsuit;': '\u2663', + 'Colon;': '\u2237', + 'colon;': ':', + 'Colone;': '\u2a74', + 'colone;': '\u2254', + 'coloneq;': '\u2254', + 'comma;': ',', + 'commat;': '@', + 'comp;': '\u2201', + 'compfn;': '\u2218', + 'complement;': '\u2201', + 'complexes;': '\u2102', + 'cong;': '\u2245', + 'congdot;': '\u2a6d', + 'Congruent;': '\u2261', + 'Conint;': '\u222f', + 'conint;': '\u222e', + 'ContourIntegral;': '\u222e', + 'Copf;': '\u2102', + 'copf;': '\U0001d554', + 'coprod;': '\u2210', + 'Coproduct;': '\u2210', + 'COPY': '\xa9', + 'copy': '\xa9', + 'COPY;': '\xa9', + 'copy;': '\xa9', + 'copysr;': '\u2117', + 'CounterClockwiseContourIntegral;': '\u2233', + 'crarr;': '\u21b5', + 'Cross;': '\u2a2f', + 'cross;': '\u2717', + 'Cscr;': '\U0001d49e', + 'cscr;': '\U0001d4b8', + 'csub;': '\u2acf', + 'csube;': '\u2ad1', + 'csup;': '\u2ad0', + 'csupe;': '\u2ad2', + 'ctdot;': '\u22ef', + 'cudarrl;': '\u2938', + 'cudarrr;': '\u2935', + 'cuepr;': '\u22de', + 'cuesc;': '\u22df', + 'cularr;': '\u21b6', + 'cularrp;': '\u293d', + 'Cup;': '\u22d3', + 'cup;': '\u222a', + 'cupbrcap;': '\u2a48', + 'CupCap;': '\u224d', + 'cupcap;': '\u2a46', + 'cupcup;': '\u2a4a', + 'cupdot;': '\u228d', + 'cupor;': '\u2a45', + 'cups;': '\u222a\ufe00', + 'curarr;': '\u21b7', + 'curarrm;': '\u293c', + 'curlyeqprec;': '\u22de', + 'curlyeqsucc;': '\u22df', + 'curlyvee;': '\u22ce', + 'curlywedge;': '\u22cf', + 'curren': '\xa4', + 'curren;': '\xa4', + 'curvearrowleft;': '\u21b6', + 'curvearrowright;': '\u21b7', + 'cuvee;': '\u22ce', + 'cuwed;': '\u22cf', + 'cwconint;': '\u2232', + 'cwint;': '\u2231', + 'cylcty;': '\u232d', + 'Dagger;': '\u2021', + 'dagger;': '\u2020', + 'daleth;': '\u2138', + 'Darr;': '\u21a1', + 'dArr;': '\u21d3', + 'darr;': '\u2193', + 'dash;': '\u2010', + 'Dashv;': '\u2ae4', + 'dashv;': '\u22a3', + 'dbkarow;': '\u290f', + 'dblac;': '\u02dd', + 'Dcaron;': '\u010e', + 'dcaron;': '\u010f', + 'Dcy;': '\u0414', + 'dcy;': '\u0434', + 'DD;': '\u2145', + 'dd;': '\u2146', + 'ddagger;': '\u2021', + 'ddarr;': '\u21ca', + 'DDotrahd;': '\u2911', + 'ddotseq;': '\u2a77', + 'deg': '\xb0', + 'deg;': '\xb0', + 'Del;': '\u2207', + 'Delta;': '\u0394', + 'delta;': '\u03b4', + 'demptyv;': '\u29b1', + 'dfisht;': '\u297f', + 'Dfr;': '\U0001d507', + 'dfr;': '\U0001d521', + 'dHar;': '\u2965', + 'dharl;': '\u21c3', + 'dharr;': '\u21c2', + 'DiacriticalAcute;': '\xb4', + 'DiacriticalDot;': '\u02d9', + 'DiacriticalDoubleAcute;': '\u02dd', + 'DiacriticalGrave;': '`', + 'DiacriticalTilde;': '\u02dc', + 'diam;': '\u22c4', + 'Diamond;': '\u22c4', + 'diamond;': '\u22c4', + 'diamondsuit;': '\u2666', + 'diams;': '\u2666', + 'die;': '\xa8', + 'DifferentialD;': '\u2146', + 'digamma;': '\u03dd', + 'disin;': '\u22f2', + 'div;': '\xf7', + 'divide': '\xf7', + 'divide;': '\xf7', + 'divideontimes;': '\u22c7', + 'divonx;': '\u22c7', + 'DJcy;': '\u0402', + 'djcy;': '\u0452', + 'dlcorn;': '\u231e', + 'dlcrop;': '\u230d', + 'dollar;': '$', + 'Dopf;': '\U0001d53b', + 'dopf;': '\U0001d555', + 'Dot;': '\xa8', + 'dot;': '\u02d9', + 'DotDot;': '\u20dc', + 'doteq;': '\u2250', + 'doteqdot;': '\u2251', + 'DotEqual;': '\u2250', + 'dotminus;': '\u2238', + 'dotplus;': '\u2214', + 'dotsquare;': '\u22a1', + 'doublebarwedge;': '\u2306', + 'DoubleContourIntegral;': '\u222f', + 'DoubleDot;': '\xa8', + 'DoubleDownArrow;': '\u21d3', + 'DoubleLeftArrow;': '\u21d0', + 'DoubleLeftRightArrow;': '\u21d4', + 'DoubleLeftTee;': '\u2ae4', + 'DoubleLongLeftArrow;': '\u27f8', + 'DoubleLongLeftRightArrow;': '\u27fa', + 'DoubleLongRightArrow;': '\u27f9', + 'DoubleRightArrow;': '\u21d2', + 'DoubleRightTee;': '\u22a8', + 'DoubleUpArrow;': '\u21d1', + 'DoubleUpDownArrow;': '\u21d5', + 'DoubleVerticalBar;': '\u2225', + 'DownArrow;': '\u2193', + 'Downarrow;': '\u21d3', + 'downarrow;': '\u2193', + 'DownArrowBar;': '\u2913', + 'DownArrowUpArrow;': '\u21f5', + 'DownBreve;': '\u0311', + 'downdownarrows;': '\u21ca', + 'downharpoonleft;': '\u21c3', + 'downharpoonright;': '\u21c2', + 'DownLeftRightVector;': '\u2950', + 'DownLeftTeeVector;': '\u295e', + 'DownLeftVector;': '\u21bd', + 'DownLeftVectorBar;': '\u2956', + 'DownRightTeeVector;': '\u295f', + 'DownRightVector;': '\u21c1', + 'DownRightVectorBar;': '\u2957', + 'DownTee;': '\u22a4', + 'DownTeeArrow;': '\u21a7', + 'drbkarow;': '\u2910', + 'drcorn;': '\u231f', + 'drcrop;': '\u230c', + 'Dscr;': '\U0001d49f', + 'dscr;': '\U0001d4b9', + 'DScy;': '\u0405', + 'dscy;': '\u0455', + 'dsol;': '\u29f6', + 'Dstrok;': '\u0110', + 'dstrok;': '\u0111', + 'dtdot;': '\u22f1', + 'dtri;': '\u25bf', + 'dtrif;': '\u25be', + 'duarr;': '\u21f5', + 'duhar;': '\u296f', + 'dwangle;': '\u29a6', + 'DZcy;': '\u040f', + 'dzcy;': '\u045f', + 'dzigrarr;': '\u27ff', + 'Eacute': '\xc9', + 'eacute': '\xe9', + 'Eacute;': '\xc9', + 'eacute;': '\xe9', + 'easter;': '\u2a6e', + 'Ecaron;': '\u011a', + 'ecaron;': '\u011b', + 'ecir;': '\u2256', + 'Ecirc': '\xca', + 'ecirc': '\xea', + 'Ecirc;': '\xca', + 'ecirc;': '\xea', + 'ecolon;': '\u2255', + 'Ecy;': '\u042d', + 'ecy;': '\u044d', + 'eDDot;': '\u2a77', + 'Edot;': '\u0116', + 'eDot;': '\u2251', + 'edot;': '\u0117', + 'ee;': '\u2147', + 'efDot;': '\u2252', + 'Efr;': '\U0001d508', + 'efr;': '\U0001d522', + 'eg;': '\u2a9a', + 'Egrave': '\xc8', + 'egrave': '\xe8', + 'Egrave;': '\xc8', + 'egrave;': '\xe8', + 'egs;': '\u2a96', + 'egsdot;': '\u2a98', + 'el;': '\u2a99', + 'Element;': '\u2208', + 'elinters;': '\u23e7', + 'ell;': '\u2113', + 'els;': '\u2a95', + 'elsdot;': '\u2a97', + 'Emacr;': '\u0112', + 'emacr;': '\u0113', + 'empty;': '\u2205', + 'emptyset;': '\u2205', + 'EmptySmallSquare;': '\u25fb', + 'emptyv;': '\u2205', + 'EmptyVerySmallSquare;': '\u25ab', + 'emsp13;': '\u2004', + 'emsp14;': '\u2005', + 'emsp;': '\u2003', + 'ENG;': '\u014a', + 'eng;': '\u014b', + 'ensp;': '\u2002', + 'Eogon;': '\u0118', + 'eogon;': '\u0119', + 'Eopf;': '\U0001d53c', + 'eopf;': '\U0001d556', + 'epar;': '\u22d5', + 'eparsl;': '\u29e3', + 'eplus;': '\u2a71', + 'epsi;': '\u03b5', + 'Epsilon;': '\u0395', + 'epsilon;': '\u03b5', + 'epsiv;': '\u03f5', + 'eqcirc;': '\u2256', + 'eqcolon;': '\u2255', + 'eqsim;': '\u2242', + 'eqslantgtr;': '\u2a96', + 'eqslantless;': '\u2a95', + 'Equal;': '\u2a75', + 'equals;': '=', + 'EqualTilde;': '\u2242', + 'equest;': '\u225f', + 'Equilibrium;': '\u21cc', + 'equiv;': '\u2261', + 'equivDD;': '\u2a78', + 'eqvparsl;': '\u29e5', + 'erarr;': '\u2971', + 'erDot;': '\u2253', + 'Escr;': '\u2130', + 'escr;': '\u212f', + 'esdot;': '\u2250', + 'Esim;': '\u2a73', + 'esim;': '\u2242', + 'Eta;': '\u0397', + 'eta;': '\u03b7', + 'ETH': '\xd0', + 'eth': '\xf0', + 'ETH;': '\xd0', + 'eth;': '\xf0', + 'Euml': '\xcb', + 'euml': '\xeb', + 'Euml;': '\xcb', + 'euml;': '\xeb', + 'euro;': '\u20ac', + 'excl;': '!', + 'exist;': '\u2203', + 'Exists;': '\u2203', + 'expectation;': '\u2130', + 'ExponentialE;': '\u2147', + 'exponentiale;': '\u2147', + 'fallingdotseq;': '\u2252', + 'Fcy;': '\u0424', + 'fcy;': '\u0444', + 'female;': '\u2640', + 'ffilig;': '\ufb03', + 'fflig;': '\ufb00', + 'ffllig;': '\ufb04', + 'Ffr;': '\U0001d509', + 'ffr;': '\U0001d523', + 'filig;': '\ufb01', + 'FilledSmallSquare;': '\u25fc', + 'FilledVerySmallSquare;': '\u25aa', + 'fjlig;': 'fj', + 'flat;': '\u266d', + 'fllig;': '\ufb02', + 'fltns;': '\u25b1', + 'fnof;': '\u0192', + 'Fopf;': '\U0001d53d', + 'fopf;': '\U0001d557', + 'ForAll;': '\u2200', + 'forall;': '\u2200', + 'fork;': '\u22d4', + 'forkv;': '\u2ad9', + 'Fouriertrf;': '\u2131', + 'fpartint;': '\u2a0d', + 'frac12': '\xbd', + 'frac12;': '\xbd', + 'frac13;': '\u2153', + 'frac14': '\xbc', + 'frac14;': '\xbc', + 'frac15;': '\u2155', + 'frac16;': '\u2159', + 'frac18;': '\u215b', + 'frac23;': '\u2154', + 'frac25;': '\u2156', + 'frac34': '\xbe', + 'frac34;': '\xbe', + 'frac35;': '\u2157', + 'frac38;': '\u215c', + 'frac45;': '\u2158', + 'frac56;': '\u215a', + 'frac58;': '\u215d', + 'frac78;': '\u215e', + 'frasl;': '\u2044', + 'frown;': '\u2322', + 'Fscr;': '\u2131', + 'fscr;': '\U0001d4bb', + 'gacute;': '\u01f5', + 'Gamma;': '\u0393', + 'gamma;': '\u03b3', + 'Gammad;': '\u03dc', + 'gammad;': '\u03dd', + 'gap;': '\u2a86', + 'Gbreve;': '\u011e', + 'gbreve;': '\u011f', + 'Gcedil;': '\u0122', + 'Gcirc;': '\u011c', + 'gcirc;': '\u011d', + 'Gcy;': '\u0413', + 'gcy;': '\u0433', + 'Gdot;': '\u0120', + 'gdot;': '\u0121', + 'gE;': '\u2267', + 'ge;': '\u2265', + 'gEl;': '\u2a8c', + 'gel;': '\u22db', + 'geq;': '\u2265', + 'geqq;': '\u2267', + 'geqslant;': '\u2a7e', + 'ges;': '\u2a7e', + 'gescc;': '\u2aa9', + 'gesdot;': '\u2a80', + 'gesdoto;': '\u2a82', + 'gesdotol;': '\u2a84', + 'gesl;': '\u22db\ufe00', + 'gesles;': '\u2a94', + 'Gfr;': '\U0001d50a', + 'gfr;': '\U0001d524', + 'Gg;': '\u22d9', + 'gg;': '\u226b', + 'ggg;': '\u22d9', + 'gimel;': '\u2137', + 'GJcy;': '\u0403', + 'gjcy;': '\u0453', + 'gl;': '\u2277', + 'gla;': '\u2aa5', + 'glE;': '\u2a92', + 'glj;': '\u2aa4', + 'gnap;': '\u2a8a', + 'gnapprox;': '\u2a8a', + 'gnE;': '\u2269', + 'gne;': '\u2a88', + 'gneq;': '\u2a88', + 'gneqq;': '\u2269', + 'gnsim;': '\u22e7', + 'Gopf;': '\U0001d53e', + 'gopf;': '\U0001d558', + 'grave;': '`', + 'GreaterEqual;': '\u2265', + 'GreaterEqualLess;': '\u22db', + 'GreaterFullEqual;': '\u2267', + 'GreaterGreater;': '\u2aa2', + 'GreaterLess;': '\u2277', + 'GreaterSlantEqual;': '\u2a7e', + 'GreaterTilde;': '\u2273', + 'Gscr;': '\U0001d4a2', + 'gscr;': '\u210a', + 'gsim;': '\u2273', + 'gsime;': '\u2a8e', + 'gsiml;': '\u2a90', + 'GT': '>', + 'gt': '>', + 'GT;': '>', + 'Gt;': '\u226b', + 'gt;': '>', + 'gtcc;': '\u2aa7', + 'gtcir;': '\u2a7a', + 'gtdot;': '\u22d7', + 'gtlPar;': '\u2995', + 'gtquest;': '\u2a7c', + 'gtrapprox;': '\u2a86', + 'gtrarr;': '\u2978', + 'gtrdot;': '\u22d7', + 'gtreqless;': '\u22db', + 'gtreqqless;': '\u2a8c', + 'gtrless;': '\u2277', + 'gtrsim;': '\u2273', + 'gvertneqq;': '\u2269\ufe00', + 'gvnE;': '\u2269\ufe00', + 'Hacek;': '\u02c7', + 'hairsp;': '\u200a', + 'half;': '\xbd', + 'hamilt;': '\u210b', + 'HARDcy;': '\u042a', + 'hardcy;': '\u044a', + 'hArr;': '\u21d4', + 'harr;': '\u2194', + 'harrcir;': '\u2948', + 'harrw;': '\u21ad', + 'Hat;': '^', + 'hbar;': '\u210f', + 'Hcirc;': '\u0124', + 'hcirc;': '\u0125', + 'hearts;': '\u2665', + 'heartsuit;': '\u2665', + 'hellip;': '\u2026', + 'hercon;': '\u22b9', + 'Hfr;': '\u210c', + 'hfr;': '\U0001d525', + 'HilbertSpace;': '\u210b', + 'hksearow;': '\u2925', + 'hkswarow;': '\u2926', + 'hoarr;': '\u21ff', + 'homtht;': '\u223b', + 'hookleftarrow;': '\u21a9', + 'hookrightarrow;': '\u21aa', + 'Hopf;': '\u210d', + 'hopf;': '\U0001d559', + 'horbar;': '\u2015', + 'HorizontalLine;': '\u2500', + 'Hscr;': '\u210b', + 'hscr;': '\U0001d4bd', + 'hslash;': '\u210f', + 'Hstrok;': '\u0126', + 'hstrok;': '\u0127', + 'HumpDownHump;': '\u224e', + 'HumpEqual;': '\u224f', + 'hybull;': '\u2043', + 'hyphen;': '\u2010', + 'Iacute': '\xcd', + 'iacute': '\xed', + 'Iacute;': '\xcd', + 'iacute;': '\xed', + 'ic;': '\u2063', + 'Icirc': '\xce', + 'icirc': '\xee', + 'Icirc;': '\xce', + 'icirc;': '\xee', + 'Icy;': '\u0418', + 'icy;': '\u0438', + 'Idot;': '\u0130', + 'IEcy;': '\u0415', + 'iecy;': '\u0435', + 'iexcl': '\xa1', + 'iexcl;': '\xa1', + 'iff;': '\u21d4', + 'Ifr;': '\u2111', + 'ifr;': '\U0001d526', + 'Igrave': '\xcc', + 'igrave': '\xec', + 'Igrave;': '\xcc', + 'igrave;': '\xec', + 'ii;': '\u2148', + 'iiiint;': '\u2a0c', + 'iiint;': '\u222d', + 'iinfin;': '\u29dc', + 'iiota;': '\u2129', + 'IJlig;': '\u0132', + 'ijlig;': '\u0133', + 'Im;': '\u2111', + 'Imacr;': '\u012a', + 'imacr;': '\u012b', + 'image;': '\u2111', + 'ImaginaryI;': '\u2148', + 'imagline;': '\u2110', + 'imagpart;': '\u2111', + 'imath;': '\u0131', + 'imof;': '\u22b7', + 'imped;': '\u01b5', + 'Implies;': '\u21d2', + 'in;': '\u2208', + 'incare;': '\u2105', + 'infin;': '\u221e', + 'infintie;': '\u29dd', + 'inodot;': '\u0131', + 'Int;': '\u222c', + 'int;': '\u222b', + 'intcal;': '\u22ba', + 'integers;': '\u2124', + 'Integral;': '\u222b', + 'intercal;': '\u22ba', + 'Intersection;': '\u22c2', + 'intlarhk;': '\u2a17', + 'intprod;': '\u2a3c', + 'InvisibleComma;': '\u2063', + 'InvisibleTimes;': '\u2062', + 'IOcy;': '\u0401', + 'iocy;': '\u0451', + 'Iogon;': '\u012e', + 'iogon;': '\u012f', + 'Iopf;': '\U0001d540', + 'iopf;': '\U0001d55a', + 'Iota;': '\u0399', + 'iota;': '\u03b9', + 'iprod;': '\u2a3c', + 'iquest': '\xbf', + 'iquest;': '\xbf', + 'Iscr;': '\u2110', + 'iscr;': '\U0001d4be', + 'isin;': '\u2208', + 'isindot;': '\u22f5', + 'isinE;': '\u22f9', + 'isins;': '\u22f4', + 'isinsv;': '\u22f3', + 'isinv;': '\u2208', + 'it;': '\u2062', + 'Itilde;': '\u0128', + 'itilde;': '\u0129', + 'Iukcy;': '\u0406', + 'iukcy;': '\u0456', + 'Iuml': '\xcf', + 'iuml': '\xef', + 'Iuml;': '\xcf', + 'iuml;': '\xef', + 'Jcirc;': '\u0134', + 'jcirc;': '\u0135', + 'Jcy;': '\u0419', + 'jcy;': '\u0439', + 'Jfr;': '\U0001d50d', + 'jfr;': '\U0001d527', + 'jmath;': '\u0237', + 'Jopf;': '\U0001d541', + 'jopf;': '\U0001d55b', + 'Jscr;': '\U0001d4a5', + 'jscr;': '\U0001d4bf', + 'Jsercy;': '\u0408', + 'jsercy;': '\u0458', + 'Jukcy;': '\u0404', + 'jukcy;': '\u0454', + 'Kappa;': '\u039a', + 'kappa;': '\u03ba', + 'kappav;': '\u03f0', + 'Kcedil;': '\u0136', + 'kcedil;': '\u0137', + 'Kcy;': '\u041a', + 'kcy;': '\u043a', + 'Kfr;': '\U0001d50e', + 'kfr;': '\U0001d528', + 'kgreen;': '\u0138', + 'KHcy;': '\u0425', + 'khcy;': '\u0445', + 'KJcy;': '\u040c', + 'kjcy;': '\u045c', + 'Kopf;': '\U0001d542', + 'kopf;': '\U0001d55c', + 'Kscr;': '\U0001d4a6', + 'kscr;': '\U0001d4c0', + 'lAarr;': '\u21da', + 'Lacute;': '\u0139', + 'lacute;': '\u013a', + 'laemptyv;': '\u29b4', + 'lagran;': '\u2112', + 'Lambda;': '\u039b', + 'lambda;': '\u03bb', + 'Lang;': '\u27ea', + 'lang;': '\u27e8', + 'langd;': '\u2991', + 'langle;': '\u27e8', + 'lap;': '\u2a85', + 'Laplacetrf;': '\u2112', + 'laquo': '\xab', + 'laquo;': '\xab', + 'Larr;': '\u219e', + 'lArr;': '\u21d0', + 'larr;': '\u2190', + 'larrb;': '\u21e4', + 'larrbfs;': '\u291f', + 'larrfs;': '\u291d', + 'larrhk;': '\u21a9', + 'larrlp;': '\u21ab', + 'larrpl;': '\u2939', + 'larrsim;': '\u2973', + 'larrtl;': '\u21a2', + 'lat;': '\u2aab', + 'lAtail;': '\u291b', + 'latail;': '\u2919', + 'late;': '\u2aad', + 'lates;': '\u2aad\ufe00', + 'lBarr;': '\u290e', + 'lbarr;': '\u290c', + 'lbbrk;': '\u2772', + 'lbrace;': '{', + 'lbrack;': '[', + 'lbrke;': '\u298b', + 'lbrksld;': '\u298f', + 'lbrkslu;': '\u298d', + 'Lcaron;': '\u013d', + 'lcaron;': '\u013e', + 'Lcedil;': '\u013b', + 'lcedil;': '\u013c', + 'lceil;': '\u2308', + 'lcub;': '{', + 'Lcy;': '\u041b', + 'lcy;': '\u043b', + 'ldca;': '\u2936', + 'ldquo;': '\u201c', + 'ldquor;': '\u201e', + 'ldrdhar;': '\u2967', + 'ldrushar;': '\u294b', + 'ldsh;': '\u21b2', + 'lE;': '\u2266', + 'le;': '\u2264', + 'LeftAngleBracket;': '\u27e8', + 'LeftArrow;': '\u2190', + 'Leftarrow;': '\u21d0', + 'leftarrow;': '\u2190', + 'LeftArrowBar;': '\u21e4', + 'LeftArrowRightArrow;': '\u21c6', + 'leftarrowtail;': '\u21a2', + 'LeftCeiling;': '\u2308', + 'LeftDoubleBracket;': '\u27e6', + 'LeftDownTeeVector;': '\u2961', + 'LeftDownVector;': '\u21c3', + 'LeftDownVectorBar;': '\u2959', + 'LeftFloor;': '\u230a', + 'leftharpoondown;': '\u21bd', + 'leftharpoonup;': '\u21bc', + 'leftleftarrows;': '\u21c7', + 'LeftRightArrow;': '\u2194', + 'Leftrightarrow;': '\u21d4', + 'leftrightarrow;': '\u2194', + 'leftrightarrows;': '\u21c6', + 'leftrightharpoons;': '\u21cb', + 'leftrightsquigarrow;': '\u21ad', + 'LeftRightVector;': '\u294e', + 'LeftTee;': '\u22a3', + 'LeftTeeArrow;': '\u21a4', + 'LeftTeeVector;': '\u295a', + 'leftthreetimes;': '\u22cb', + 'LeftTriangle;': '\u22b2', + 'LeftTriangleBar;': '\u29cf', + 'LeftTriangleEqual;': '\u22b4', + 'LeftUpDownVector;': '\u2951', + 'LeftUpTeeVector;': '\u2960', + 'LeftUpVector;': '\u21bf', + 'LeftUpVectorBar;': '\u2958', + 'LeftVector;': '\u21bc', + 'LeftVectorBar;': '\u2952', + 'lEg;': '\u2a8b', + 'leg;': '\u22da', + 'leq;': '\u2264', + 'leqq;': '\u2266', + 'leqslant;': '\u2a7d', + 'les;': '\u2a7d', + 'lescc;': '\u2aa8', + 'lesdot;': '\u2a7f', + 'lesdoto;': '\u2a81', + 'lesdotor;': '\u2a83', + 'lesg;': '\u22da\ufe00', + 'lesges;': '\u2a93', + 'lessapprox;': '\u2a85', + 'lessdot;': '\u22d6', + 'lesseqgtr;': '\u22da', + 'lesseqqgtr;': '\u2a8b', + 'LessEqualGreater;': '\u22da', + 'LessFullEqual;': '\u2266', + 'LessGreater;': '\u2276', + 'lessgtr;': '\u2276', + 'LessLess;': '\u2aa1', + 'lesssim;': '\u2272', + 'LessSlantEqual;': '\u2a7d', + 'LessTilde;': '\u2272', + 'lfisht;': '\u297c', + 'lfloor;': '\u230a', + 'Lfr;': '\U0001d50f', + 'lfr;': '\U0001d529', + 'lg;': '\u2276', + 'lgE;': '\u2a91', + 'lHar;': '\u2962', + 'lhard;': '\u21bd', + 'lharu;': '\u21bc', + 'lharul;': '\u296a', + 'lhblk;': '\u2584', + 'LJcy;': '\u0409', + 'ljcy;': '\u0459', + 'Ll;': '\u22d8', + 'll;': '\u226a', + 'llarr;': '\u21c7', + 'llcorner;': '\u231e', + 'Lleftarrow;': '\u21da', + 'llhard;': '\u296b', + 'lltri;': '\u25fa', + 'Lmidot;': '\u013f', + 'lmidot;': '\u0140', + 'lmoust;': '\u23b0', + 'lmoustache;': '\u23b0', + 'lnap;': '\u2a89', + 'lnapprox;': '\u2a89', + 'lnE;': '\u2268', + 'lne;': '\u2a87', + 'lneq;': '\u2a87', + 'lneqq;': '\u2268', + 'lnsim;': '\u22e6', + 'loang;': '\u27ec', + 'loarr;': '\u21fd', + 'lobrk;': '\u27e6', + 'LongLeftArrow;': '\u27f5', + 'Longleftarrow;': '\u27f8', + 'longleftarrow;': '\u27f5', + 'LongLeftRightArrow;': '\u27f7', + 'Longleftrightarrow;': '\u27fa', + 'longleftrightarrow;': '\u27f7', + 'longmapsto;': '\u27fc', + 'LongRightArrow;': '\u27f6', + 'Longrightarrow;': '\u27f9', + 'longrightarrow;': '\u27f6', + 'looparrowleft;': '\u21ab', + 'looparrowright;': '\u21ac', + 'lopar;': '\u2985', + 'Lopf;': '\U0001d543', + 'lopf;': '\U0001d55d', + 'loplus;': '\u2a2d', + 'lotimes;': '\u2a34', + 'lowast;': '\u2217', + 'lowbar;': '_', + 'LowerLeftArrow;': '\u2199', + 'LowerRightArrow;': '\u2198', + 'loz;': '\u25ca', + 'lozenge;': '\u25ca', + 'lozf;': '\u29eb', + 'lpar;': '(', + 'lparlt;': '\u2993', + 'lrarr;': '\u21c6', + 'lrcorner;': '\u231f', + 'lrhar;': '\u21cb', + 'lrhard;': '\u296d', + 'lrm;': '\u200e', + 'lrtri;': '\u22bf', + 'lsaquo;': '\u2039', + 'Lscr;': '\u2112', + 'lscr;': '\U0001d4c1', + 'Lsh;': '\u21b0', + 'lsh;': '\u21b0', + 'lsim;': '\u2272', + 'lsime;': '\u2a8d', + 'lsimg;': '\u2a8f', + 'lsqb;': '[', + 'lsquo;': '\u2018', + 'lsquor;': '\u201a', + 'Lstrok;': '\u0141', + 'lstrok;': '\u0142', + 'LT': '<', + 'lt': '<', + 'LT;': '<', + 'Lt;': '\u226a', + 'lt;': '<', + 'ltcc;': '\u2aa6', + 'ltcir;': '\u2a79', + 'ltdot;': '\u22d6', + 'lthree;': '\u22cb', + 'ltimes;': '\u22c9', + 'ltlarr;': '\u2976', + 'ltquest;': '\u2a7b', + 'ltri;': '\u25c3', + 'ltrie;': '\u22b4', + 'ltrif;': '\u25c2', + 'ltrPar;': '\u2996', + 'lurdshar;': '\u294a', + 'luruhar;': '\u2966', + 'lvertneqq;': '\u2268\ufe00', + 'lvnE;': '\u2268\ufe00', + 'macr': '\xaf', + 'macr;': '\xaf', + 'male;': '\u2642', + 'malt;': '\u2720', + 'maltese;': '\u2720', + 'Map;': '\u2905', + 'map;': '\u21a6', + 'mapsto;': '\u21a6', + 'mapstodown;': '\u21a7', + 'mapstoleft;': '\u21a4', + 'mapstoup;': '\u21a5', + 'marker;': '\u25ae', + 'mcomma;': '\u2a29', + 'Mcy;': '\u041c', + 'mcy;': '\u043c', + 'mdash;': '\u2014', + 'mDDot;': '\u223a', + 'measuredangle;': '\u2221', + 'MediumSpace;': '\u205f', + 'Mellintrf;': '\u2133', + 'Mfr;': '\U0001d510', + 'mfr;': '\U0001d52a', + 'mho;': '\u2127', + 'micro': '\xb5', + 'micro;': '\xb5', + 'mid;': '\u2223', + 'midast;': '*', + 'midcir;': '\u2af0', + 'middot': '\xb7', + 'middot;': '\xb7', + 'minus;': '\u2212', + 'minusb;': '\u229f', + 'minusd;': '\u2238', + 'minusdu;': '\u2a2a', + 'MinusPlus;': '\u2213', + 'mlcp;': '\u2adb', + 'mldr;': '\u2026', + 'mnplus;': '\u2213', + 'models;': '\u22a7', + 'Mopf;': '\U0001d544', + 'mopf;': '\U0001d55e', + 'mp;': '\u2213', + 'Mscr;': '\u2133', + 'mscr;': '\U0001d4c2', + 'mstpos;': '\u223e', + 'Mu;': '\u039c', + 'mu;': '\u03bc', + 'multimap;': '\u22b8', + 'mumap;': '\u22b8', + 'nabla;': '\u2207', + 'Nacute;': '\u0143', + 'nacute;': '\u0144', + 'nang;': '\u2220\u20d2', + 'nap;': '\u2249', + 'napE;': '\u2a70\u0338', + 'napid;': '\u224b\u0338', + 'napos;': '\u0149', + 'napprox;': '\u2249', + 'natur;': '\u266e', + 'natural;': '\u266e', + 'naturals;': '\u2115', + 'nbsp': '\xa0', + 'nbsp;': '\xa0', + 'nbump;': '\u224e\u0338', + 'nbumpe;': '\u224f\u0338', + 'ncap;': '\u2a43', + 'Ncaron;': '\u0147', + 'ncaron;': '\u0148', + 'Ncedil;': '\u0145', + 'ncedil;': '\u0146', + 'ncong;': '\u2247', + 'ncongdot;': '\u2a6d\u0338', + 'ncup;': '\u2a42', + 'Ncy;': '\u041d', + 'ncy;': '\u043d', + 'ndash;': '\u2013', + 'ne;': '\u2260', + 'nearhk;': '\u2924', + 'neArr;': '\u21d7', + 'nearr;': '\u2197', + 'nearrow;': '\u2197', + 'nedot;': '\u2250\u0338', + 'NegativeMediumSpace;': '\u200b', + 'NegativeThickSpace;': '\u200b', + 'NegativeThinSpace;': '\u200b', + 'NegativeVeryThinSpace;': '\u200b', + 'nequiv;': '\u2262', + 'nesear;': '\u2928', + 'nesim;': '\u2242\u0338', + 'NestedGreaterGreater;': '\u226b', + 'NestedLessLess;': '\u226a', + 'NewLine;': '\n', + 'nexist;': '\u2204', + 'nexists;': '\u2204', + 'Nfr;': '\U0001d511', + 'nfr;': '\U0001d52b', + 'ngE;': '\u2267\u0338', + 'nge;': '\u2271', + 'ngeq;': '\u2271', + 'ngeqq;': '\u2267\u0338', + 'ngeqslant;': '\u2a7e\u0338', + 'nges;': '\u2a7e\u0338', + 'nGg;': '\u22d9\u0338', + 'ngsim;': '\u2275', + 'nGt;': '\u226b\u20d2', + 'ngt;': '\u226f', + 'ngtr;': '\u226f', + 'nGtv;': '\u226b\u0338', + 'nhArr;': '\u21ce', + 'nharr;': '\u21ae', + 'nhpar;': '\u2af2', + 'ni;': '\u220b', + 'nis;': '\u22fc', + 'nisd;': '\u22fa', + 'niv;': '\u220b', + 'NJcy;': '\u040a', + 'njcy;': '\u045a', + 'nlArr;': '\u21cd', + 'nlarr;': '\u219a', + 'nldr;': '\u2025', + 'nlE;': '\u2266\u0338', + 'nle;': '\u2270', + 'nLeftarrow;': '\u21cd', + 'nleftarrow;': '\u219a', + 'nLeftrightarrow;': '\u21ce', + 'nleftrightarrow;': '\u21ae', + 'nleq;': '\u2270', + 'nleqq;': '\u2266\u0338', + 'nleqslant;': '\u2a7d\u0338', + 'nles;': '\u2a7d\u0338', + 'nless;': '\u226e', + 'nLl;': '\u22d8\u0338', + 'nlsim;': '\u2274', + 'nLt;': '\u226a\u20d2', + 'nlt;': '\u226e', + 'nltri;': '\u22ea', + 'nltrie;': '\u22ec', + 'nLtv;': '\u226a\u0338', + 'nmid;': '\u2224', + 'NoBreak;': '\u2060', + 'NonBreakingSpace;': '\xa0', + 'Nopf;': '\u2115', + 'nopf;': '\U0001d55f', + 'not': '\xac', + 'Not;': '\u2aec', + 'not;': '\xac', + 'NotCongruent;': '\u2262', + 'NotCupCap;': '\u226d', + 'NotDoubleVerticalBar;': '\u2226', + 'NotElement;': '\u2209', + 'NotEqual;': '\u2260', + 'NotEqualTilde;': '\u2242\u0338', + 'NotExists;': '\u2204', + 'NotGreater;': '\u226f', + 'NotGreaterEqual;': '\u2271', + 'NotGreaterFullEqual;': '\u2267\u0338', + 'NotGreaterGreater;': '\u226b\u0338', + 'NotGreaterLess;': '\u2279', + 'NotGreaterSlantEqual;': '\u2a7e\u0338', + 'NotGreaterTilde;': '\u2275', + 'NotHumpDownHump;': '\u224e\u0338', + 'NotHumpEqual;': '\u224f\u0338', + 'notin;': '\u2209', + 'notindot;': '\u22f5\u0338', + 'notinE;': '\u22f9\u0338', + 'notinva;': '\u2209', + 'notinvb;': '\u22f7', + 'notinvc;': '\u22f6', + 'NotLeftTriangle;': '\u22ea', + 'NotLeftTriangleBar;': '\u29cf\u0338', + 'NotLeftTriangleEqual;': '\u22ec', + 'NotLess;': '\u226e', + 'NotLessEqual;': '\u2270', + 'NotLessGreater;': '\u2278', + 'NotLessLess;': '\u226a\u0338', + 'NotLessSlantEqual;': '\u2a7d\u0338', + 'NotLessTilde;': '\u2274', + 'NotNestedGreaterGreater;': '\u2aa2\u0338', + 'NotNestedLessLess;': '\u2aa1\u0338', + 'notni;': '\u220c', + 'notniva;': '\u220c', + 'notnivb;': '\u22fe', + 'notnivc;': '\u22fd', + 'NotPrecedes;': '\u2280', + 'NotPrecedesEqual;': '\u2aaf\u0338', + 'NotPrecedesSlantEqual;': '\u22e0', + 'NotReverseElement;': '\u220c', + 'NotRightTriangle;': '\u22eb', + 'NotRightTriangleBar;': '\u29d0\u0338', + 'NotRightTriangleEqual;': '\u22ed', + 'NotSquareSubset;': '\u228f\u0338', + 'NotSquareSubsetEqual;': '\u22e2', + 'NotSquareSuperset;': '\u2290\u0338', + 'NotSquareSupersetEqual;': '\u22e3', + 'NotSubset;': '\u2282\u20d2', + 'NotSubsetEqual;': '\u2288', + 'NotSucceeds;': '\u2281', + 'NotSucceedsEqual;': '\u2ab0\u0338', + 'NotSucceedsSlantEqual;': '\u22e1', + 'NotSucceedsTilde;': '\u227f\u0338', + 'NotSuperset;': '\u2283\u20d2', + 'NotSupersetEqual;': '\u2289', + 'NotTilde;': '\u2241', + 'NotTildeEqual;': '\u2244', + 'NotTildeFullEqual;': '\u2247', + 'NotTildeTilde;': '\u2249', + 'NotVerticalBar;': '\u2224', + 'npar;': '\u2226', + 'nparallel;': '\u2226', + 'nparsl;': '\u2afd\u20e5', + 'npart;': '\u2202\u0338', + 'npolint;': '\u2a14', + 'npr;': '\u2280', + 'nprcue;': '\u22e0', + 'npre;': '\u2aaf\u0338', + 'nprec;': '\u2280', + 'npreceq;': '\u2aaf\u0338', + 'nrArr;': '\u21cf', + 'nrarr;': '\u219b', + 'nrarrc;': '\u2933\u0338', + 'nrarrw;': '\u219d\u0338', + 'nRightarrow;': '\u21cf', + 'nrightarrow;': '\u219b', + 'nrtri;': '\u22eb', + 'nrtrie;': '\u22ed', + 'nsc;': '\u2281', + 'nsccue;': '\u22e1', + 'nsce;': '\u2ab0\u0338', + 'Nscr;': '\U0001d4a9', + 'nscr;': '\U0001d4c3', + 'nshortmid;': '\u2224', + 'nshortparallel;': '\u2226', + 'nsim;': '\u2241', + 'nsime;': '\u2244', + 'nsimeq;': '\u2244', + 'nsmid;': '\u2224', + 'nspar;': '\u2226', + 'nsqsube;': '\u22e2', + 'nsqsupe;': '\u22e3', + 'nsub;': '\u2284', + 'nsubE;': '\u2ac5\u0338', + 'nsube;': '\u2288', + 'nsubset;': '\u2282\u20d2', + 'nsubseteq;': '\u2288', + 'nsubseteqq;': '\u2ac5\u0338', + 'nsucc;': '\u2281', + 'nsucceq;': '\u2ab0\u0338', + 'nsup;': '\u2285', + 'nsupE;': '\u2ac6\u0338', + 'nsupe;': '\u2289', + 'nsupset;': '\u2283\u20d2', + 'nsupseteq;': '\u2289', + 'nsupseteqq;': '\u2ac6\u0338', + 'ntgl;': '\u2279', + 'Ntilde': '\xd1', + 'ntilde': '\xf1', + 'Ntilde;': '\xd1', + 'ntilde;': '\xf1', + 'ntlg;': '\u2278', + 'ntriangleleft;': '\u22ea', + 'ntrianglelefteq;': '\u22ec', + 'ntriangleright;': '\u22eb', + 'ntrianglerighteq;': '\u22ed', + 'Nu;': '\u039d', + 'nu;': '\u03bd', + 'num;': '#', + 'numero;': '\u2116', + 'numsp;': '\u2007', + 'nvap;': '\u224d\u20d2', + 'nVDash;': '\u22af', + 'nVdash;': '\u22ae', + 'nvDash;': '\u22ad', + 'nvdash;': '\u22ac', + 'nvge;': '\u2265\u20d2', + 'nvgt;': '>\u20d2', + 'nvHarr;': '\u2904', + 'nvinfin;': '\u29de', + 'nvlArr;': '\u2902', + 'nvle;': '\u2264\u20d2', + 'nvlt;': '<\u20d2', + 'nvltrie;': '\u22b4\u20d2', + 'nvrArr;': '\u2903', + 'nvrtrie;': '\u22b5\u20d2', + 'nvsim;': '\u223c\u20d2', + 'nwarhk;': '\u2923', + 'nwArr;': '\u21d6', + 'nwarr;': '\u2196', + 'nwarrow;': '\u2196', + 'nwnear;': '\u2927', + 'Oacute': '\xd3', + 'oacute': '\xf3', + 'Oacute;': '\xd3', + 'oacute;': '\xf3', + 'oast;': '\u229b', + 'ocir;': '\u229a', + 'Ocirc': '\xd4', + 'ocirc': '\xf4', + 'Ocirc;': '\xd4', + 'ocirc;': '\xf4', + 'Ocy;': '\u041e', + 'ocy;': '\u043e', + 'odash;': '\u229d', + 'Odblac;': '\u0150', + 'odblac;': '\u0151', + 'odiv;': '\u2a38', + 'odot;': '\u2299', + 'odsold;': '\u29bc', + 'OElig;': '\u0152', + 'oelig;': '\u0153', + 'ofcir;': '\u29bf', + 'Ofr;': '\U0001d512', + 'ofr;': '\U0001d52c', + 'ogon;': '\u02db', + 'Ograve': '\xd2', + 'ograve': '\xf2', + 'Ograve;': '\xd2', + 'ograve;': '\xf2', + 'ogt;': '\u29c1', + 'ohbar;': '\u29b5', + 'ohm;': '\u03a9', + 'oint;': '\u222e', + 'olarr;': '\u21ba', + 'olcir;': '\u29be', + 'olcross;': '\u29bb', + 'oline;': '\u203e', + 'olt;': '\u29c0', + 'Omacr;': '\u014c', + 'omacr;': '\u014d', + 'Omega;': '\u03a9', + 'omega;': '\u03c9', + 'Omicron;': '\u039f', + 'omicron;': '\u03bf', + 'omid;': '\u29b6', + 'ominus;': '\u2296', + 'Oopf;': '\U0001d546', + 'oopf;': '\U0001d560', + 'opar;': '\u29b7', + 'OpenCurlyDoubleQuote;': '\u201c', + 'OpenCurlyQuote;': '\u2018', + 'operp;': '\u29b9', + 'oplus;': '\u2295', + 'Or;': '\u2a54', + 'or;': '\u2228', + 'orarr;': '\u21bb', + 'ord;': '\u2a5d', + 'order;': '\u2134', + 'orderof;': '\u2134', + 'ordf': '\xaa', + 'ordf;': '\xaa', + 'ordm': '\xba', + 'ordm;': '\xba', + 'origof;': '\u22b6', + 'oror;': '\u2a56', + 'orslope;': '\u2a57', + 'orv;': '\u2a5b', + 'oS;': '\u24c8', + 'Oscr;': '\U0001d4aa', + 'oscr;': '\u2134', + 'Oslash': '\xd8', + 'oslash': '\xf8', + 'Oslash;': '\xd8', + 'oslash;': '\xf8', + 'osol;': '\u2298', + 'Otilde': '\xd5', + 'otilde': '\xf5', + 'Otilde;': '\xd5', + 'otilde;': '\xf5', + 'Otimes;': '\u2a37', + 'otimes;': '\u2297', + 'otimesas;': '\u2a36', + 'Ouml': '\xd6', + 'ouml': '\xf6', + 'Ouml;': '\xd6', + 'ouml;': '\xf6', + 'ovbar;': '\u233d', + 'OverBar;': '\u203e', + 'OverBrace;': '\u23de', + 'OverBracket;': '\u23b4', + 'OverParenthesis;': '\u23dc', + 'par;': '\u2225', + 'para': '\xb6', + 'para;': '\xb6', + 'parallel;': '\u2225', + 'parsim;': '\u2af3', + 'parsl;': '\u2afd', + 'part;': '\u2202', + 'PartialD;': '\u2202', + 'Pcy;': '\u041f', + 'pcy;': '\u043f', + 'percnt;': '%', + 'period;': '.', + 'permil;': '\u2030', + 'perp;': '\u22a5', + 'pertenk;': '\u2031', + 'Pfr;': '\U0001d513', + 'pfr;': '\U0001d52d', + 'Phi;': '\u03a6', + 'phi;': '\u03c6', + 'phiv;': '\u03d5', + 'phmmat;': '\u2133', + 'phone;': '\u260e', + 'Pi;': '\u03a0', + 'pi;': '\u03c0', + 'pitchfork;': '\u22d4', + 'piv;': '\u03d6', + 'planck;': '\u210f', + 'planckh;': '\u210e', + 'plankv;': '\u210f', + 'plus;': '+', + 'plusacir;': '\u2a23', + 'plusb;': '\u229e', + 'pluscir;': '\u2a22', + 'plusdo;': '\u2214', + 'plusdu;': '\u2a25', + 'pluse;': '\u2a72', + 'PlusMinus;': '\xb1', + 'plusmn': '\xb1', + 'plusmn;': '\xb1', + 'plussim;': '\u2a26', + 'plustwo;': '\u2a27', + 'pm;': '\xb1', + 'Poincareplane;': '\u210c', + 'pointint;': '\u2a15', + 'Popf;': '\u2119', + 'popf;': '\U0001d561', + 'pound': '\xa3', + 'pound;': '\xa3', + 'Pr;': '\u2abb', + 'pr;': '\u227a', + 'prap;': '\u2ab7', + 'prcue;': '\u227c', + 'prE;': '\u2ab3', + 'pre;': '\u2aaf', + 'prec;': '\u227a', + 'precapprox;': '\u2ab7', + 'preccurlyeq;': '\u227c', + 'Precedes;': '\u227a', + 'PrecedesEqual;': '\u2aaf', + 'PrecedesSlantEqual;': '\u227c', + 'PrecedesTilde;': '\u227e', + 'preceq;': '\u2aaf', + 'precnapprox;': '\u2ab9', + 'precneqq;': '\u2ab5', + 'precnsim;': '\u22e8', + 'precsim;': '\u227e', + 'Prime;': '\u2033', + 'prime;': '\u2032', + 'primes;': '\u2119', + 'prnap;': '\u2ab9', + 'prnE;': '\u2ab5', + 'prnsim;': '\u22e8', + 'prod;': '\u220f', + 'Product;': '\u220f', + 'profalar;': '\u232e', + 'profline;': '\u2312', + 'profsurf;': '\u2313', + 'prop;': '\u221d', + 'Proportion;': '\u2237', + 'Proportional;': '\u221d', + 'propto;': '\u221d', + 'prsim;': '\u227e', + 'prurel;': '\u22b0', + 'Pscr;': '\U0001d4ab', + 'pscr;': '\U0001d4c5', + 'Psi;': '\u03a8', + 'psi;': '\u03c8', + 'puncsp;': '\u2008', + 'Qfr;': '\U0001d514', + 'qfr;': '\U0001d52e', + 'qint;': '\u2a0c', + 'Qopf;': '\u211a', + 'qopf;': '\U0001d562', + 'qprime;': '\u2057', + 'Qscr;': '\U0001d4ac', + 'qscr;': '\U0001d4c6', + 'quaternions;': '\u210d', + 'quatint;': '\u2a16', + 'quest;': '?', + 'questeq;': '\u225f', + 'QUOT': '"', + 'quot': '"', + 'QUOT;': '"', + 'quot;': '"', + 'rAarr;': '\u21db', + 'race;': '\u223d\u0331', + 'Racute;': '\u0154', + 'racute;': '\u0155', + 'radic;': '\u221a', + 'raemptyv;': '\u29b3', + 'Rang;': '\u27eb', + 'rang;': '\u27e9', + 'rangd;': '\u2992', + 'range;': '\u29a5', + 'rangle;': '\u27e9', + 'raquo': '\xbb', + 'raquo;': '\xbb', + 'Rarr;': '\u21a0', + 'rArr;': '\u21d2', + 'rarr;': '\u2192', + 'rarrap;': '\u2975', + 'rarrb;': '\u21e5', + 'rarrbfs;': '\u2920', + 'rarrc;': '\u2933', + 'rarrfs;': '\u291e', + 'rarrhk;': '\u21aa', + 'rarrlp;': '\u21ac', + 'rarrpl;': '\u2945', + 'rarrsim;': '\u2974', + 'Rarrtl;': '\u2916', + 'rarrtl;': '\u21a3', + 'rarrw;': '\u219d', + 'rAtail;': '\u291c', + 'ratail;': '\u291a', + 'ratio;': '\u2236', + 'rationals;': '\u211a', + 'RBarr;': '\u2910', + 'rBarr;': '\u290f', + 'rbarr;': '\u290d', + 'rbbrk;': '\u2773', + 'rbrace;': '}', + 'rbrack;': ']', + 'rbrke;': '\u298c', + 'rbrksld;': '\u298e', + 'rbrkslu;': '\u2990', + 'Rcaron;': '\u0158', + 'rcaron;': '\u0159', + 'Rcedil;': '\u0156', + 'rcedil;': '\u0157', + 'rceil;': '\u2309', + 'rcub;': '}', + 'Rcy;': '\u0420', + 'rcy;': '\u0440', + 'rdca;': '\u2937', + 'rdldhar;': '\u2969', + 'rdquo;': '\u201d', + 'rdquor;': '\u201d', + 'rdsh;': '\u21b3', + 'Re;': '\u211c', + 'real;': '\u211c', + 'realine;': '\u211b', + 'realpart;': '\u211c', + 'reals;': '\u211d', + 'rect;': '\u25ad', + 'REG': '\xae', + 'reg': '\xae', + 'REG;': '\xae', + 'reg;': '\xae', + 'ReverseElement;': '\u220b', + 'ReverseEquilibrium;': '\u21cb', + 'ReverseUpEquilibrium;': '\u296f', + 'rfisht;': '\u297d', + 'rfloor;': '\u230b', + 'Rfr;': '\u211c', + 'rfr;': '\U0001d52f', + 'rHar;': '\u2964', + 'rhard;': '\u21c1', + 'rharu;': '\u21c0', + 'rharul;': '\u296c', + 'Rho;': '\u03a1', + 'rho;': '\u03c1', + 'rhov;': '\u03f1', + 'RightAngleBracket;': '\u27e9', + 'RightArrow;': '\u2192', + 'Rightarrow;': '\u21d2', + 'rightarrow;': '\u2192', + 'RightArrowBar;': '\u21e5', + 'RightArrowLeftArrow;': '\u21c4', + 'rightarrowtail;': '\u21a3', + 'RightCeiling;': '\u2309', + 'RightDoubleBracket;': '\u27e7', + 'RightDownTeeVector;': '\u295d', + 'RightDownVector;': '\u21c2', + 'RightDownVectorBar;': '\u2955', + 'RightFloor;': '\u230b', + 'rightharpoondown;': '\u21c1', + 'rightharpoonup;': '\u21c0', + 'rightleftarrows;': '\u21c4', + 'rightleftharpoons;': '\u21cc', + 'rightrightarrows;': '\u21c9', + 'rightsquigarrow;': '\u219d', + 'RightTee;': '\u22a2', + 'RightTeeArrow;': '\u21a6', + 'RightTeeVector;': '\u295b', + 'rightthreetimes;': '\u22cc', + 'RightTriangle;': '\u22b3', + 'RightTriangleBar;': '\u29d0', + 'RightTriangleEqual;': '\u22b5', + 'RightUpDownVector;': '\u294f', + 'RightUpTeeVector;': '\u295c', + 'RightUpVector;': '\u21be', + 'RightUpVectorBar;': '\u2954', + 'RightVector;': '\u21c0', + 'RightVectorBar;': '\u2953', + 'ring;': '\u02da', + 'risingdotseq;': '\u2253', + 'rlarr;': '\u21c4', + 'rlhar;': '\u21cc', + 'rlm;': '\u200f', + 'rmoust;': '\u23b1', + 'rmoustache;': '\u23b1', + 'rnmid;': '\u2aee', + 'roang;': '\u27ed', + 'roarr;': '\u21fe', + 'robrk;': '\u27e7', + 'ropar;': '\u2986', + 'Ropf;': '\u211d', + 'ropf;': '\U0001d563', + 'roplus;': '\u2a2e', + 'rotimes;': '\u2a35', + 'RoundImplies;': '\u2970', + 'rpar;': ')', + 'rpargt;': '\u2994', + 'rppolint;': '\u2a12', + 'rrarr;': '\u21c9', + 'Rrightarrow;': '\u21db', + 'rsaquo;': '\u203a', + 'Rscr;': '\u211b', + 'rscr;': '\U0001d4c7', + 'Rsh;': '\u21b1', + 'rsh;': '\u21b1', + 'rsqb;': ']', + 'rsquo;': '\u2019', + 'rsquor;': '\u2019', + 'rthree;': '\u22cc', + 'rtimes;': '\u22ca', + 'rtri;': '\u25b9', + 'rtrie;': '\u22b5', + 'rtrif;': '\u25b8', + 'rtriltri;': '\u29ce', + 'RuleDelayed;': '\u29f4', + 'ruluhar;': '\u2968', + 'rx;': '\u211e', + 'Sacute;': '\u015a', + 'sacute;': '\u015b', + 'sbquo;': '\u201a', + 'Sc;': '\u2abc', + 'sc;': '\u227b', + 'scap;': '\u2ab8', + 'Scaron;': '\u0160', + 'scaron;': '\u0161', + 'sccue;': '\u227d', + 'scE;': '\u2ab4', + 'sce;': '\u2ab0', + 'Scedil;': '\u015e', + 'scedil;': '\u015f', + 'Scirc;': '\u015c', + 'scirc;': '\u015d', + 'scnap;': '\u2aba', + 'scnE;': '\u2ab6', + 'scnsim;': '\u22e9', + 'scpolint;': '\u2a13', + 'scsim;': '\u227f', + 'Scy;': '\u0421', + 'scy;': '\u0441', + 'sdot;': '\u22c5', + 'sdotb;': '\u22a1', + 'sdote;': '\u2a66', + 'searhk;': '\u2925', + 'seArr;': '\u21d8', + 'searr;': '\u2198', + 'searrow;': '\u2198', + 'sect': '\xa7', + 'sect;': '\xa7', + 'semi;': ';', + 'seswar;': '\u2929', + 'setminus;': '\u2216', + 'setmn;': '\u2216', + 'sext;': '\u2736', + 'Sfr;': '\U0001d516', + 'sfr;': '\U0001d530', + 'sfrown;': '\u2322', + 'sharp;': '\u266f', + 'SHCHcy;': '\u0429', + 'shchcy;': '\u0449', + 'SHcy;': '\u0428', + 'shcy;': '\u0448', + 'ShortDownArrow;': '\u2193', + 'ShortLeftArrow;': '\u2190', + 'shortmid;': '\u2223', + 'shortparallel;': '\u2225', + 'ShortRightArrow;': '\u2192', + 'ShortUpArrow;': '\u2191', + 'shy': '\xad', + 'shy;': '\xad', + 'Sigma;': '\u03a3', + 'sigma;': '\u03c3', + 'sigmaf;': '\u03c2', + 'sigmav;': '\u03c2', + 'sim;': '\u223c', + 'simdot;': '\u2a6a', + 'sime;': '\u2243', + 'simeq;': '\u2243', + 'simg;': '\u2a9e', + 'simgE;': '\u2aa0', + 'siml;': '\u2a9d', + 'simlE;': '\u2a9f', + 'simne;': '\u2246', + 'simplus;': '\u2a24', + 'simrarr;': '\u2972', + 'slarr;': '\u2190', + 'SmallCircle;': '\u2218', + 'smallsetminus;': '\u2216', + 'smashp;': '\u2a33', + 'smeparsl;': '\u29e4', + 'smid;': '\u2223', + 'smile;': '\u2323', + 'smt;': '\u2aaa', + 'smte;': '\u2aac', + 'smtes;': '\u2aac\ufe00', + 'SOFTcy;': '\u042c', + 'softcy;': '\u044c', + 'sol;': '/', + 'solb;': '\u29c4', + 'solbar;': '\u233f', + 'Sopf;': '\U0001d54a', + 'sopf;': '\U0001d564', + 'spades;': '\u2660', + 'spadesuit;': '\u2660', + 'spar;': '\u2225', + 'sqcap;': '\u2293', + 'sqcaps;': '\u2293\ufe00', + 'sqcup;': '\u2294', + 'sqcups;': '\u2294\ufe00', + 'Sqrt;': '\u221a', + 'sqsub;': '\u228f', + 'sqsube;': '\u2291', + 'sqsubset;': '\u228f', + 'sqsubseteq;': '\u2291', + 'sqsup;': '\u2290', + 'sqsupe;': '\u2292', + 'sqsupset;': '\u2290', + 'sqsupseteq;': '\u2292', + 'squ;': '\u25a1', + 'Square;': '\u25a1', + 'square;': '\u25a1', + 'SquareIntersection;': '\u2293', + 'SquareSubset;': '\u228f', + 'SquareSubsetEqual;': '\u2291', + 'SquareSuperset;': '\u2290', + 'SquareSupersetEqual;': '\u2292', + 'SquareUnion;': '\u2294', + 'squarf;': '\u25aa', + 'squf;': '\u25aa', + 'srarr;': '\u2192', + 'Sscr;': '\U0001d4ae', + 'sscr;': '\U0001d4c8', + 'ssetmn;': '\u2216', + 'ssmile;': '\u2323', + 'sstarf;': '\u22c6', + 'Star;': '\u22c6', + 'star;': '\u2606', + 'starf;': '\u2605', + 'straightepsilon;': '\u03f5', + 'straightphi;': '\u03d5', + 'strns;': '\xaf', + 'Sub;': '\u22d0', + 'sub;': '\u2282', + 'subdot;': '\u2abd', + 'subE;': '\u2ac5', + 'sube;': '\u2286', + 'subedot;': '\u2ac3', + 'submult;': '\u2ac1', + 'subnE;': '\u2acb', + 'subne;': '\u228a', + 'subplus;': '\u2abf', + 'subrarr;': '\u2979', + 'Subset;': '\u22d0', + 'subset;': '\u2282', + 'subseteq;': '\u2286', + 'subseteqq;': '\u2ac5', + 'SubsetEqual;': '\u2286', + 'subsetneq;': '\u228a', + 'subsetneqq;': '\u2acb', + 'subsim;': '\u2ac7', + 'subsub;': '\u2ad5', + 'subsup;': '\u2ad3', + 'succ;': '\u227b', + 'succapprox;': '\u2ab8', + 'succcurlyeq;': '\u227d', + 'Succeeds;': '\u227b', + 'SucceedsEqual;': '\u2ab0', + 'SucceedsSlantEqual;': '\u227d', + 'SucceedsTilde;': '\u227f', + 'succeq;': '\u2ab0', + 'succnapprox;': '\u2aba', + 'succneqq;': '\u2ab6', + 'succnsim;': '\u22e9', + 'succsim;': '\u227f', + 'SuchThat;': '\u220b', + 'Sum;': '\u2211', + 'sum;': '\u2211', + 'sung;': '\u266a', + 'sup1': '\xb9', + 'sup1;': '\xb9', + 'sup2': '\xb2', + 'sup2;': '\xb2', + 'sup3': '\xb3', + 'sup3;': '\xb3', + 'Sup;': '\u22d1', + 'sup;': '\u2283', + 'supdot;': '\u2abe', + 'supdsub;': '\u2ad8', + 'supE;': '\u2ac6', + 'supe;': '\u2287', + 'supedot;': '\u2ac4', + 'Superset;': '\u2283', + 'SupersetEqual;': '\u2287', + 'suphsol;': '\u27c9', + 'suphsub;': '\u2ad7', + 'suplarr;': '\u297b', + 'supmult;': '\u2ac2', + 'supnE;': '\u2acc', + 'supne;': '\u228b', + 'supplus;': '\u2ac0', + 'Supset;': '\u22d1', + 'supset;': '\u2283', + 'supseteq;': '\u2287', + 'supseteqq;': '\u2ac6', + 'supsetneq;': '\u228b', + 'supsetneqq;': '\u2acc', + 'supsim;': '\u2ac8', + 'supsub;': '\u2ad4', + 'supsup;': '\u2ad6', + 'swarhk;': '\u2926', + 'swArr;': '\u21d9', + 'swarr;': '\u2199', + 'swarrow;': '\u2199', + 'swnwar;': '\u292a', + 'szlig': '\xdf', + 'szlig;': '\xdf', + 'Tab;': '\t', + 'target;': '\u2316', + 'Tau;': '\u03a4', + 'tau;': '\u03c4', + 'tbrk;': '\u23b4', + 'Tcaron;': '\u0164', + 'tcaron;': '\u0165', + 'Tcedil;': '\u0162', + 'tcedil;': '\u0163', + 'Tcy;': '\u0422', + 'tcy;': '\u0442', + 'tdot;': '\u20db', + 'telrec;': '\u2315', + 'Tfr;': '\U0001d517', + 'tfr;': '\U0001d531', + 'there4;': '\u2234', + 'Therefore;': '\u2234', + 'therefore;': '\u2234', + 'Theta;': '\u0398', + 'theta;': '\u03b8', + 'thetasym;': '\u03d1', + 'thetav;': '\u03d1', + 'thickapprox;': '\u2248', + 'thicksim;': '\u223c', + 'ThickSpace;': '\u205f\u200a', + 'thinsp;': '\u2009', + 'ThinSpace;': '\u2009', + 'thkap;': '\u2248', + 'thksim;': '\u223c', + 'THORN': '\xde', + 'thorn': '\xfe', + 'THORN;': '\xde', + 'thorn;': '\xfe', + 'Tilde;': '\u223c', + 'tilde;': '\u02dc', + 'TildeEqual;': '\u2243', + 'TildeFullEqual;': '\u2245', + 'TildeTilde;': '\u2248', + 'times': '\xd7', + 'times;': '\xd7', + 'timesb;': '\u22a0', + 'timesbar;': '\u2a31', + 'timesd;': '\u2a30', + 'tint;': '\u222d', + 'toea;': '\u2928', + 'top;': '\u22a4', + 'topbot;': '\u2336', + 'topcir;': '\u2af1', + 'Topf;': '\U0001d54b', + 'topf;': '\U0001d565', + 'topfork;': '\u2ada', + 'tosa;': '\u2929', + 'tprime;': '\u2034', + 'TRADE;': '\u2122', + 'trade;': '\u2122', + 'triangle;': '\u25b5', + 'triangledown;': '\u25bf', + 'triangleleft;': '\u25c3', + 'trianglelefteq;': '\u22b4', + 'triangleq;': '\u225c', + 'triangleright;': '\u25b9', + 'trianglerighteq;': '\u22b5', + 'tridot;': '\u25ec', + 'trie;': '\u225c', + 'triminus;': '\u2a3a', + 'TripleDot;': '\u20db', + 'triplus;': '\u2a39', + 'trisb;': '\u29cd', + 'tritime;': '\u2a3b', + 'trpezium;': '\u23e2', + 'Tscr;': '\U0001d4af', + 'tscr;': '\U0001d4c9', + 'TScy;': '\u0426', + 'tscy;': '\u0446', + 'TSHcy;': '\u040b', + 'tshcy;': '\u045b', + 'Tstrok;': '\u0166', + 'tstrok;': '\u0167', + 'twixt;': '\u226c', + 'twoheadleftarrow;': '\u219e', + 'twoheadrightarrow;': '\u21a0', + 'Uacute': '\xda', + 'uacute': '\xfa', + 'Uacute;': '\xda', + 'uacute;': '\xfa', + 'Uarr;': '\u219f', + 'uArr;': '\u21d1', + 'uarr;': '\u2191', + 'Uarrocir;': '\u2949', + 'Ubrcy;': '\u040e', + 'ubrcy;': '\u045e', + 'Ubreve;': '\u016c', + 'ubreve;': '\u016d', + 'Ucirc': '\xdb', + 'ucirc': '\xfb', + 'Ucirc;': '\xdb', + 'ucirc;': '\xfb', + 'Ucy;': '\u0423', + 'ucy;': '\u0443', + 'udarr;': '\u21c5', + 'Udblac;': '\u0170', + 'udblac;': '\u0171', + 'udhar;': '\u296e', + 'ufisht;': '\u297e', + 'Ufr;': '\U0001d518', + 'ufr;': '\U0001d532', + 'Ugrave': '\xd9', + 'ugrave': '\xf9', + 'Ugrave;': '\xd9', + 'ugrave;': '\xf9', + 'uHar;': '\u2963', + 'uharl;': '\u21bf', + 'uharr;': '\u21be', + 'uhblk;': '\u2580', + 'ulcorn;': '\u231c', + 'ulcorner;': '\u231c', + 'ulcrop;': '\u230f', + 'ultri;': '\u25f8', + 'Umacr;': '\u016a', + 'umacr;': '\u016b', + 'uml': '\xa8', + 'uml;': '\xa8', + 'UnderBar;': '_', + 'UnderBrace;': '\u23df', + 'UnderBracket;': '\u23b5', + 'UnderParenthesis;': '\u23dd', + 'Union;': '\u22c3', + 'UnionPlus;': '\u228e', + 'Uogon;': '\u0172', + 'uogon;': '\u0173', + 'Uopf;': '\U0001d54c', + 'uopf;': '\U0001d566', + 'UpArrow;': '\u2191', + 'Uparrow;': '\u21d1', + 'uparrow;': '\u2191', + 'UpArrowBar;': '\u2912', + 'UpArrowDownArrow;': '\u21c5', + 'UpDownArrow;': '\u2195', + 'Updownarrow;': '\u21d5', + 'updownarrow;': '\u2195', + 'UpEquilibrium;': '\u296e', + 'upharpoonleft;': '\u21bf', + 'upharpoonright;': '\u21be', + 'uplus;': '\u228e', + 'UpperLeftArrow;': '\u2196', + 'UpperRightArrow;': '\u2197', + 'Upsi;': '\u03d2', + 'upsi;': '\u03c5', + 'upsih;': '\u03d2', + 'Upsilon;': '\u03a5', + 'upsilon;': '\u03c5', + 'UpTee;': '\u22a5', + 'UpTeeArrow;': '\u21a5', + 'upuparrows;': '\u21c8', + 'urcorn;': '\u231d', + 'urcorner;': '\u231d', + 'urcrop;': '\u230e', + 'Uring;': '\u016e', + 'uring;': '\u016f', + 'urtri;': '\u25f9', + 'Uscr;': '\U0001d4b0', + 'uscr;': '\U0001d4ca', + 'utdot;': '\u22f0', + 'Utilde;': '\u0168', + 'utilde;': '\u0169', + 'utri;': '\u25b5', + 'utrif;': '\u25b4', + 'uuarr;': '\u21c8', + 'Uuml': '\xdc', + 'uuml': '\xfc', + 'Uuml;': '\xdc', + 'uuml;': '\xfc', + 'uwangle;': '\u29a7', + 'vangrt;': '\u299c', + 'varepsilon;': '\u03f5', + 'varkappa;': '\u03f0', + 'varnothing;': '\u2205', + 'varphi;': '\u03d5', + 'varpi;': '\u03d6', + 'varpropto;': '\u221d', + 'vArr;': '\u21d5', + 'varr;': '\u2195', + 'varrho;': '\u03f1', + 'varsigma;': '\u03c2', + 'varsubsetneq;': '\u228a\ufe00', + 'varsubsetneqq;': '\u2acb\ufe00', + 'varsupsetneq;': '\u228b\ufe00', + 'varsupsetneqq;': '\u2acc\ufe00', + 'vartheta;': '\u03d1', + 'vartriangleleft;': '\u22b2', + 'vartriangleright;': '\u22b3', + 'Vbar;': '\u2aeb', + 'vBar;': '\u2ae8', + 'vBarv;': '\u2ae9', + 'Vcy;': '\u0412', + 'vcy;': '\u0432', + 'VDash;': '\u22ab', + 'Vdash;': '\u22a9', + 'vDash;': '\u22a8', + 'vdash;': '\u22a2', + 'Vdashl;': '\u2ae6', + 'Vee;': '\u22c1', + 'vee;': '\u2228', + 'veebar;': '\u22bb', + 'veeeq;': '\u225a', + 'vellip;': '\u22ee', + 'Verbar;': '\u2016', + 'verbar;': '|', + 'Vert;': '\u2016', + 'vert;': '|', + 'VerticalBar;': '\u2223', + 'VerticalLine;': '|', + 'VerticalSeparator;': '\u2758', + 'VerticalTilde;': '\u2240', + 'VeryThinSpace;': '\u200a', + 'Vfr;': '\U0001d519', + 'vfr;': '\U0001d533', + 'vltri;': '\u22b2', + 'vnsub;': '\u2282\u20d2', + 'vnsup;': '\u2283\u20d2', + 'Vopf;': '\U0001d54d', + 'vopf;': '\U0001d567', + 'vprop;': '\u221d', + 'vrtri;': '\u22b3', + 'Vscr;': '\U0001d4b1', + 'vscr;': '\U0001d4cb', + 'vsubnE;': '\u2acb\ufe00', + 'vsubne;': '\u228a\ufe00', + 'vsupnE;': '\u2acc\ufe00', + 'vsupne;': '\u228b\ufe00', + 'Vvdash;': '\u22aa', + 'vzigzag;': '\u299a', + 'Wcirc;': '\u0174', + 'wcirc;': '\u0175', + 'wedbar;': '\u2a5f', + 'Wedge;': '\u22c0', + 'wedge;': '\u2227', + 'wedgeq;': '\u2259', + 'weierp;': '\u2118', + 'Wfr;': '\U0001d51a', + 'wfr;': '\U0001d534', + 'Wopf;': '\U0001d54e', + 'wopf;': '\U0001d568', + 'wp;': '\u2118', + 'wr;': '\u2240', + 'wreath;': '\u2240', + 'Wscr;': '\U0001d4b2', + 'wscr;': '\U0001d4cc', + 'xcap;': '\u22c2', + 'xcirc;': '\u25ef', + 'xcup;': '\u22c3', + 'xdtri;': '\u25bd', + 'Xfr;': '\U0001d51b', + 'xfr;': '\U0001d535', + 'xhArr;': '\u27fa', + 'xharr;': '\u27f7', + 'Xi;': '\u039e', + 'xi;': '\u03be', + 'xlArr;': '\u27f8', + 'xlarr;': '\u27f5', + 'xmap;': '\u27fc', + 'xnis;': '\u22fb', + 'xodot;': '\u2a00', + 'Xopf;': '\U0001d54f', + 'xopf;': '\U0001d569', + 'xoplus;': '\u2a01', + 'xotime;': '\u2a02', + 'xrArr;': '\u27f9', + 'xrarr;': '\u27f6', + 'Xscr;': '\U0001d4b3', + 'xscr;': '\U0001d4cd', + 'xsqcup;': '\u2a06', + 'xuplus;': '\u2a04', + 'xutri;': '\u25b3', + 'xvee;': '\u22c1', + 'xwedge;': '\u22c0', + 'Yacute': '\xdd', + 'yacute': '\xfd', + 'Yacute;': '\xdd', + 'yacute;': '\xfd', + 'YAcy;': '\u042f', + 'yacy;': '\u044f', + 'Ycirc;': '\u0176', + 'ycirc;': '\u0177', + 'Ycy;': '\u042b', + 'ycy;': '\u044b', + 'yen': '\xa5', + 'yen;': '\xa5', + 'Yfr;': '\U0001d51c', + 'yfr;': '\U0001d536', + 'YIcy;': '\u0407', + 'yicy;': '\u0457', + 'Yopf;': '\U0001d550', + 'yopf;': '\U0001d56a', + 'Yscr;': '\U0001d4b4', + 'yscr;': '\U0001d4ce', + 'YUcy;': '\u042e', + 'yucy;': '\u044e', + 'yuml': '\xff', + 'Yuml;': '\u0178', + 'yuml;': '\xff', + 'Zacute;': '\u0179', + 'zacute;': '\u017a', + 'Zcaron;': '\u017d', + 'zcaron;': '\u017e', + 'Zcy;': '\u0417', + 'zcy;': '\u0437', + 'Zdot;': '\u017b', + 'zdot;': '\u017c', + 'zeetrf;': '\u2128', + 'ZeroWidthSpace;': '\u200b', + 'Zeta;': '\u0396', + 'zeta;': '\u03b6', + 'Zfr;': '\u2128', + 'zfr;': '\U0001d537', + 'ZHcy;': '\u0416', + 'zhcy;': '\u0436', + 'zigrarr;': '\u21dd', + 'Zopf;': '\u2124', + 'zopf;': '\U0001d56b', + 'Zscr;': '\U0001d4b5', + 'zscr;': '\U0001d4cf', + 'zwj;': '\u200d', + 'zwnj;': '\u200c', +} + +# maps the Unicode code point to the HTML entity name +codepoint2name = {} + +# maps the HTML entity name to the character +# (or a character reference if the character is outside the Latin-1 range) +entitydefs = {} + +for (name, codepoint) in name2codepoint.items(): + codepoint2name[codepoint] = name + entitydefs[name] = chr(codepoint) + +del name, codepoint diff --git a/crates/weavepy-vm/src/stdlib/python/inspect.py b/crates/weavepy-vm/src/stdlib/python/inspect.py index 389fbe2..1c2c1f8 100644 --- a/crates/weavepy-vm/src/stdlib/python/inspect.py +++ b/crates/weavepy-vm/src/stdlib/python/inspect.py @@ -723,17 +723,33 @@ def _bind(self, args, kwargs, partial): return BoundArguments(self, arguments) def __str__(self): - parts = [] - kind_seen = None + result = [] + render_pos_only_separator = False + render_kw_only_separator = True for p in self._parameters.values(): - if p.kind == Parameter.KEYWORD_ONLY and kind_seen != Parameter.VAR_POSITIONAL and kind_seen != Parameter.KEYWORD_ONLY: - parts.append("*") - parts.append(str(p)) - kind_seen = p.kind + formatted = str(p) + kind = p.kind + if kind == Parameter.POSITIONAL_ONLY: + render_pos_only_separator = True + elif render_pos_only_separator: + # We have a separator, and we've just got to a non-pos-only param. + result.append("/") + render_pos_only_separator = False + if kind == Parameter.VAR_POSITIONAL: + # OK, we have an '*args'-like parameter, so we won't need '*'. + render_kw_only_separator = False + elif kind == Parameter.KEYWORD_ONLY and render_kw_only_separator: + result.append("*") + render_kw_only_separator = False + result.append(formatted) + if render_pos_only_separator: + # There were only positional-only parameters, hence the flag was + # not reset to 'False'. + result.append("/") ret = "" if self._return_annotation is not _empty: ret = f" -> {self._return_annotation!r}" - return "(" + ", ".join(parts) + ")" + ret + return "(" + ", ".join(result) + ")" + ret @classmethod def from_callable(cls, func): @@ -742,17 +758,31 @@ def from_callable(cls, func): def signature(callable_): if isclass(callable_): + # Prefer __new__ when it is overridden (e.g. functools.partial), then + # fall back to __init__. A class signature carries no return annotation. + new = getattr(callable_, "__new__", None) + if new is not None and new is not object.__new__: + sig = signature(new) + params = [p for name, p in sig.parameters.items() if name != "cls"] + return Signature(params) init = getattr(callable_, "__init__", None) if init is not None and init is not object.__init__: sig = signature(init) params = [p for name, p in sig.parameters.items() if name != "self"] - return Signature(params, return_annotation=callable_) + return Signature(params) return Signature([]) if ismethod(callable_): sig = signature(callable_.__func__) params = [p for name, p in sig.parameters.items() if name != "self"] return Signature(params, return_annotation=sig.return_annotation) if not isfunction(callable_): + # A callable instance (defines __call__ on its type): derive the + # signature from the type's __call__, dropping the bound `self`. + call = getattr(type(callable_), "__call__", None) + if call is not None and (isfunction(call) or ismethod(call)): + sig = signature(call) + params = [p for name, p in sig.parameters.items() if name != "self"] + return Signature(params, return_annotation=sig.return_annotation) # Best effort: return an "unknown" signature. return Signature([Parameter("args", Parameter.VAR_POSITIONAL), Parameter("kwargs", Parameter.VAR_KEYWORD)]) @@ -761,13 +791,16 @@ def signature(callable_): defaults = spec.defaults or () n_defaults = len(defaults) n_args = len(spec.args) + f = _func_of(callable_) + posonly = getattr(f.__code__, "co_posonlyargcount", 0) if f is not None else 0 for i, name in enumerate(spec.args): if i >= n_args - n_defaults: default = defaults[i - (n_args - n_defaults)] else: default = _empty annotation = spec.annotations.get(name, _empty) - params.append(Parameter(name, Parameter.POSITIONAL_OR_KEYWORD, + kind = Parameter.POSITIONAL_ONLY if i < posonly else Parameter.POSITIONAL_OR_KEYWORD + params.append(Parameter(name, kind, default=default, annotation=annotation)) if spec.varargs: params.append(Parameter(spec.varargs, Parameter.VAR_POSITIONAL, diff --git a/crates/weavepy-vm/src/stdlib/python/locale.py b/crates/weavepy-vm/src/stdlib/python/locale.py new file mode 100644 index 0000000..0aa8ad6 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/locale.py @@ -0,0 +1,186 @@ +"""Minimal but faithful ``locale`` for the portable 'C'/'POSIX' locale. + +CPython backs :mod:`locale` with the C ``_locale`` extension. WeavePy ships +a pure-Python module implementing the public surface against the default +'C' locale: querying always succeeds, switching to ``''``/``'C'``/``'POSIX'`` +succeeds, and any other (uninstalled) locale raises :class:`Error` exactly +as CPython does on a host where that locale is unavailable (RFC 0037 WS8). +""" + +CHAR_MAX = 127 + +# Category constants. The numeric values are an internal, stable choice +# (CPython's come from the C library and are platform-specific); code should +# use the names, never the literals. +LC_CTYPE = 0 +LC_NUMERIC = 1 +LC_TIME = 2 +LC_COLLATE = 3 +LC_MONETARY = 4 +LC_MESSAGES = 5 +LC_ALL = 6 + +_ALL_CATEGORIES = (LC_CTYPE, LC_NUMERIC, LC_TIME, LC_COLLATE, LC_MONETARY, + LC_MESSAGES) + +__all__ = [ + "getlocale", "getdefaultlocale", "getpreferredencoding", "Error", + "setlocale", "resetlocale", "localeconv", "strcoll", "strxfrm", + "str", "atof", "atoi", "format_string", "currency", "normalize", + "LC_CTYPE", "LC_NUMERIC", "LC_TIME", "LC_COLLATE", "LC_MONETARY", + "LC_MESSAGES", "LC_ALL", "CHAR_MAX", "delocalize", "localize", +] + + +class Error(Exception): + pass + + +_state = {c: "C" for c in (LC_CTYPE, LC_NUMERIC, LC_TIME, LC_COLLATE, + LC_MONETARY, LC_MESSAGES, LC_ALL)} + + +def _norm_requested(value): + """Map a requested locale name to the only locale we can honour ('C') + or raise :class:`Error` for anything we cannot install — the same + observable contract as CPython on a host missing that locale.""" + if isinstance(value, tuple): + value = _build_localename(value) + if value in ("", "C", "POSIX"): + return "C" + raise Error("unsupported locale setting") + + +def setlocale(category, locale=None): + if category not in (LC_ALL, *_ALL_CATEGORIES): + raise Error("invalid locale category") + if locale is None: + return _state.get(category, "C") + normalized = _norm_requested(locale) + if category == LC_ALL: + for c in (LC_ALL, *_ALL_CATEGORIES): + _state[c] = normalized + else: + _state[category] = normalized + return normalized + + +def resetlocale(category=LC_ALL): + setlocale(category, "C") + + +def localeconv(): + """Return the lconv table for the 'C' locale.""" + return { + "decimal_point": ".", + "thousands_sep": "", + "grouping": [], + "int_curr_symbol": "", + "currency_symbol": "", + "mon_decimal_point": "", + "mon_thousands_sep": "", + "mon_grouping": [], + "positive_sign": "", + "negative_sign": "", + "int_frac_digits": CHAR_MAX, + "frac_digits": CHAR_MAX, + "p_cs_precedes": CHAR_MAX, + "p_sep_by_space": CHAR_MAX, + "n_cs_precedes": CHAR_MAX, + "n_sep_by_space": CHAR_MAX, + "p_sign_posn": CHAR_MAX, + "n_sign_posn": CHAR_MAX, + } + + +def getlocale(category=LC_CTYPE): + """The 'C' locale carries no language/encoding pair.""" + return (None, None) + + +def getdefaultlocale(envvars=("LC_ALL", "LC_CTYPE", "LANG", "LANGUAGE")): + return (None, None) + + +def getpreferredencoding(do_setlocale=True): + return "utf-8" + + +def getencoding(): + return "utf-8" + + +def normalize(localename): + # Without the C alias table we only recognise the portable names. + name = localename.lower() + if name in ("c", "posix", ""): + return "C" + return localename + + +def _build_localename(localetuple): + try: + language, encoding = localetuple + except (TypeError, ValueError): + raise TypeError("Locale must be None, a string, or an iterable of " + "two strings -- language code, encoding.") from None + if language is None: + language = "C" + if encoding is None: + return language + return language + "." + encoding + + +def strcoll(a, b): + return (a > b) - (a < b) + + +def strxfrm(s): + return s + + +# --- numeric helpers (C locale: '.' decimal point, no grouping) ----------- + +def localize(string, grouping=False, monetary=False): + return string + + +def delocalize(string): + return string + + +def atof(string, func=float): + return func(delocalize(string)) + + +def atoi(string): + return int(delocalize(string)) + + +def str(val): + return format_string("%.12g", val) + + +def format_string(format, val, grouping=False, monetary=False): + import re as _re + + def _strip(m): + return m.group(0) + + # In the 'C' locale there is no grouping or monetary decoration, so the + # conversion is just plain printf-style formatting. + if isinstance(val, tuple): + return format % val + return format % val + + +def format(percent, value, grouping=False, monetary=False, *additional): + if additional: + formatted = percent % ((value,) + additional) + else: + formatted = percent % value + return formatted + + +def currency(val, symbol=True, grouping=False, international=False): + raise ValueError("Currency formatting is not possible in the 'C' locale.") diff --git a/crates/weavepy-vm/src/stdlib/python/mimetypes.py b/crates/weavepy-vm/src/stdlib/python/mimetypes.py index e985916..2af7c4b 100644 --- a/crates/weavepy-vm/src/stdlib/python/mimetypes.py +++ b/crates/weavepy-vm/src/stdlib/python/mimetypes.py @@ -1,243 +1,679 @@ -"""WeavePy `mimetypes` — extension to MIME-type mapping. +"""Guess the MIME type of a file. -This is a baked-in static table covering the common types listed in -CPython's `mimetypes.types_map`. The module exposes the standard -public surface (`guess_type`, `guess_extension`, `add_type`, -`init`, `MimeTypes`, ...). +This module defines two useful functions: + +guess_type(url, strict=True) -- guess the MIME type and encoding of a URL. + +guess_extension(type, strict=True) -- guess the extension for a given MIME type. + +It also contains the following, for tuning the behavior: + +Data: + +knownfiles -- list of files to parse +inited -- flag set when init() has been called +suffix_map -- dictionary mapping suffixes to suffixes +encodings_map -- dictionary mapping suffixes to encodings +types_map -- dictionary mapping suffixes to types + +Functions: + +init([files]) -- parse a list of files, default knownfiles (on Windows, the + default values are taken from the registry) +read_mime_types(file) -- parse one file, return a dictionary or None """ import os +import sys +import posixpath +import urllib.parse +try: + from _winapi import _mimetypes_read_windows_registry +except ImportError: + _mimetypes_read_windows_registry = None + +try: + import winreg as _winreg +except ImportError: + _winreg = None __all__ = [ - "knownfiles", "inited", "MimeTypes", "guess_type", "guess_all_extensions", - "guess_extension", "add_type", "init", "read_mime_types", "suffix_map", - "encodings_map", "types_map", "common_types", + "knownfiles", "inited", "MimeTypes", + "guess_type", "guess_file_type", "guess_all_extensions", "guess_extension", + "add_type", "init", "read_mime_types", + "suffix_map", "encodings_map", "types_map", "common_types" ] +knownfiles = [ + "/etc/mime.types", + "/etc/httpd/mime.types", # Mac OS X + "/etc/httpd/conf/mime.types", # Apache + "/etc/apache/mime.types", # Apache 1 + "/etc/apache2/mime.types", # Apache 2 + "/usr/local/etc/httpd/conf/mime.types", + "/usr/local/lib/netscape/mime.types", + "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 + "/usr/local/etc/mime.types", # Apache 1.3 + ] -knownfiles = [] inited = False - - -suffix_map = { - ".svgz": ".svg.gz", - ".tgz": ".tar.gz", - ".taz": ".tar.gz", - ".tz": ".tar.gz", - ".tbz2": ".tar.bz2", - ".txz": ".tar.xz", -} - - -encodings_map = { - ".gz": "gzip", - ".Z": "compress", - ".bz2": "bzip2", - ".xz": "xz", - ".br": "br", -} - - -_TYPES = { - ".a": "application/octet-stream", - ".ai": "application/postscript", - ".aif": "audio/x-aiff", - ".aifc": "audio/x-aiff", - ".aiff": "audio/x-aiff", - ".au": "audio/basic", - ".avi": "video/x-msvideo", - ".bat": "text/plain", - ".bcpio": "application/x-bcpio", - ".bin": "application/octet-stream", - ".bmp": "image/bmp", - ".c": "text/plain", - ".cdf": "application/x-netcdf", - ".cpio": "application/x-cpio", - ".csh": "application/x-csh", - ".css": "text/css", - ".csv": "text/csv", - ".dll": "application/octet-stream", - ".doc": "application/msword", - ".dot": "application/msword", - ".dvi": "application/x-dvi", - ".eml": "message/rfc822", - ".eps": "application/postscript", - ".etx": "text/x-setext", - ".exe": "application/octet-stream", - ".gif": "image/gif", - ".gtar": "application/x-gtar", - ".h": "text/plain", - ".hdf": "application/x-hdf", - ".htm": "text/html", - ".html": "text/html", - ".ico": "image/vnd.microsoft.icon", - ".ief": "image/ief", - ".jpe": "image/jpeg", - ".jpeg": "image/jpeg", - ".jpg": "image/jpeg", - ".js": "application/javascript", - ".json": "application/json", - ".latex": "application/x-latex", - ".m1v": "video/mpeg", - ".m3u": "application/vnd.apple.mpegurl", - ".m3u8": "application/vnd.apple.mpegurl", - ".man": "application/x-troff-man", - ".md": "text/markdown", - ".me": "application/x-troff-me", - ".mht": "message/rfc822", - ".mhtml": "message/rfc822", - ".mif": "application/x-mif", - ".mov": "video/quicktime", - ".movie": "video/x-sgi-movie", - ".mp2": "audio/mpeg", - ".mp3": "audio/mpeg", - ".mp4": "video/mp4", - ".mpa": "video/mpeg", - ".mpe": "video/mpeg", - ".mpeg": "video/mpeg", - ".mpg": "video/mpeg", - ".ms": "application/x-troff-ms", - ".nc": "application/x-netcdf", - ".nws": "message/rfc822", - ".o": "application/octet-stream", - ".obj": "application/octet-stream", - ".oda": "application/oda", - ".p12": "application/x-pkcs12", - ".p7c": "application/pkcs7-mime", - ".pbm": "image/x-portable-bitmap", - ".pdf": "application/pdf", - ".pfx": "application/x-pkcs12", - ".pgm": "image/x-portable-graymap", - ".png": "image/png", - ".pnm": "image/x-portable-anymap", - ".pot": "application/vnd.ms-powerpoint", - ".ppa": "application/vnd.ms-powerpoint", - ".ppm": "image/x-portable-pixmap", - ".pps": "application/vnd.ms-powerpoint", - ".ppt": "application/vnd.ms-powerpoint", - ".ps": "application/postscript", - ".pwz": "application/vnd.ms-powerpoint", - ".py": "text/x-python", - ".pyc": "application/x-python-code", - ".pyo": "application/x-python-code", - ".qt": "video/quicktime", - ".ra": "audio/x-pn-realaudio", - ".ram": "application/x-pn-realaudio", - ".rdf": "application/xml", - ".rgb": "image/x-rgb", - ".roff": "application/x-troff", - ".rtx": "text/richtext", - ".sgm": "text/x-sgml", - ".sgml": "text/x-sgml", - ".sh": "application/x-sh", - ".shar": "application/x-shar", - ".snd": "audio/basic", - ".so": "application/octet-stream", - ".src": "application/x-wais-source", - ".sv4cpio": "application/x-sv4cpio", - ".sv4crc": "application/x-sv4crc", - ".svg": "image/svg+xml", - ".swf": "application/x-shockwave-flash", - ".t": "application/x-troff", - ".tar": "application/x-tar", - ".tcl": "application/x-tcl", - ".tex": "application/x-tex", - ".texi": "application/x-texinfo", - ".texinfo": "application/x-texinfo", - ".tif": "image/tiff", - ".tiff": "image/tiff", - ".tr": "application/x-troff", - ".tsv": "text/tab-separated-values", - ".txt": "text/plain", - ".ustar": "application/x-ustar", - ".vcf": "text/x-vcard", - ".wasm": "application/wasm", - ".wav": "audio/x-wav", - ".webm": "video/webm", - ".webmanifest": "application/manifest+json", - ".wiz": "application/msword", - ".wsdl": "application/xml", - ".xbm": "image/x-xbitmap", - ".xlb": "application/vnd.ms-excel", - ".xls": "application/vnd.ms-excel", - ".xml": "text/xml", - ".xpdl": "application/xml", - ".xpm": "image/x-xpixmap", - ".xsl": "application/xml", - ".xwd": "image/x-xwindowdump", - ".yaml": "application/yaml", - ".yml": "application/yaml", - ".zip": "application/zip", -} - - -types_map = dict(_TYPES) -common_types = {} +_db = None class MimeTypes: - """Class wrapper around `guess_type`/`guess_extension`.""" + """MIME-types datastore. + + This datastore can handle information from mime.types-style files + and supports basic determination of MIME type from a filename or + URL, and can guess a reasonable extension given a MIME type. + """ def __init__(self, filenames=(), strict=True): - self.types_map = (dict(_TYPES), {}) + if not inited: + init() + self.encodings_map = _encodings_map_default.copy() + self.suffix_map = _suffix_map_default.copy() + self.types_map = ({}, {}) # dict for (non-strict, strict) self.types_map_inv = ({}, {}) - self.encodings_map = dict(encodings_map) - self.suffix_map = dict(suffix_map) - for ext, ty in _TYPES.items(): - self.types_map_inv[0].setdefault(ty, []).append(ext) + for (ext, type) in _types_map_default.items(): + self.add_type(type, ext, True) + for (ext, type) in _common_types_default.items(): + self.add_type(type, ext, False) + for name in filenames: + self.read(name, strict) - def guess_type(self, url, strict=True): - return guess_type(url, strict) + def add_type(self, type, ext, strict=True): + """Add a mapping between a type and an extension. + + When the extension is already known, the new + type will replace the old one. When the type + is already known the extension will be added + to the list of known extensions. + + If strict is true, information will be added to + list of standard types, else to the list of non-standard + types. + """ + if not type: + return + self.types_map[strict][ext] = type + exts = self.types_map_inv[strict].setdefault(type, []) + if ext not in exts: + exts.append(ext) - def guess_extension(self, type, strict=True): - return guess_extension(type, strict) + def guess_type(self, url, strict=True): + """Guess the type of a file which is either a URL or a path-like object. + + Return value is a tuple (type, encoding) where type is None if + the type can't be guessed (no or unknown suffix) or a string + of the form type/subtype, usable for a MIME Content-type + header; and encoding is None for no encoding or the name of + the program used to encode (e.g. compress or gzip). The + mappings are table driven. Encoding suffixes are case + sensitive; type suffixes are first tried case sensitive, then + case insensitive. + + The suffixes .tgz, .taz and .tz (case sensitive!) are all + mapped to '.tar.gz'. (This is table-driven too, using the + dictionary suffix_map.) + + Optional `strict' argument when False adds a bunch of commonly found, + but non-standard types. + """ + # TODO: Deprecate accepting file paths (in particular path-like objects). + url = os.fspath(url) + p = urllib.parse.urlparse(url) + if p.scheme and len(p.scheme) > 1: + scheme = p.scheme + url = p.path + else: + return self.guess_file_type(url, strict=strict) + if scheme == 'data': + # syntax of data URLs: + # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data + # mediatype := [ type "/" subtype ] *( ";" parameter ) + # data := *urlchar + # parameter := attribute "=" value + # type/subtype defaults to "text/plain" + comma = url.find(',') + if comma < 0: + # bad data URL + return None, None + semi = url.find(';', 0, comma) + if semi >= 0: + type = url[:semi] + else: + type = url[:comma] + if '=' in type or '/' not in type: + type = 'text/plain' + return type, None # never compressed, so encoding is None + return self._guess_file_type(url, strict, posixpath.splitext) + + def guess_file_type(self, path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path istead of URL. + """ + path = os.fsdecode(path) + path = os.path.splitdrive(path)[1] + return self._guess_file_type(path, strict, os.path.splitext) + + def _guess_file_type(self, path, strict, splitext): + base, ext = splitext(path) + while (ext_lower := ext.lower()) in self.suffix_map: + base, ext = splitext(base + self.suffix_map[ext_lower]) + # encodings_map is case sensitive + if ext in self.encodings_map: + encoding = self.encodings_map[ext] + base, ext = splitext(base) + else: + encoding = None + ext = ext.lower() + types_map = self.types_map[True] + if ext in types_map: + return types_map[ext], encoding + elif strict: + return None, encoding + types_map = self.types_map[False] + if ext in types_map: + return types_map[ext], encoding + else: + return None, encoding def guess_all_extensions(self, type, strict=True): - return guess_all_extensions(type, strict) + """Guess the extensions for a file based on its MIME type. + + Return value is a list of strings giving the possible filename + extensions, including the leading dot ('.'). The extension is not + guaranteed to have been associated with any particular data stream, + but would be mapped to the MIME type `type' by guess_type(). + + Optional `strict' argument when false adds a bunch of commonly found, + but non-standard types. + """ + type = type.lower() + extensions = list(self.types_map_inv[True].get(type, [])) + if not strict: + for ext in self.types_map_inv[False].get(type, []): + if ext not in extensions: + extensions.append(ext) + return extensions - def add_type(self, type, ext, strict=True): - add_type(type, ext, strict) + def guess_extension(self, type, strict=True): + """Guess the extension for a file based on its MIME type. + + Return value is a string giving a filename extension, + including the leading dot ('.'). The extension is not + guaranteed to have been associated with any particular data + stream, but would be mapped to the MIME type `type' by + guess_type(). If no extension can be guessed for `type', None + is returned. + + Optional `strict' argument when false adds a bunch of commonly found, + but non-standard types. + """ + extensions = self.guess_all_extensions(type, strict) + if not extensions: + return None + return extensions[0] + + def read(self, filename, strict=True): + """ + Read a single mime.types-format file, specified by pathname. + + If strict is true, information will be added to + list of standard types, else to the list of non-standard + types. + """ + with open(filename, encoding='utf-8') as fp: + self.readfp(fp, strict) + + def readfp(self, fp, strict=True): + """ + Read a single mime.types-format file. + + If strict is true, information will be added to + list of standard types, else to the list of non-standard + types. + """ + while line := fp.readline(): + words = line.split() + for i in range(len(words)): + if words[i][0] == '#': + del words[i:] + break + if not words: + continue + type, suffixes = words[0], words[1:] + for suff in suffixes: + self.add_type(type, '.' + suff, strict) + + def read_windows_registry(self, strict=True): + """ + Load the MIME types database from Windows registry. + + If strict is true, information will be added to + list of standard types, else to the list of non-standard + types. + """ + + if not _mimetypes_read_windows_registry and not _winreg: + return + + add_type = self.add_type + if strict: + add_type = lambda type, ext: self.add_type(type, ext, True) + + # Accelerated function if it is available + if _mimetypes_read_windows_registry: + _mimetypes_read_windows_registry(add_type) + elif _winreg: + self._read_windows_registry(add_type) + + @classmethod + def _read_windows_registry(cls, add_type): + def enum_types(mimedb): + i = 0 + while True: + try: + ctype = _winreg.EnumKey(mimedb, i) + except OSError: + break + else: + if '\0' not in ctype: + yield ctype + i += 1 + + with _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, '') as hkcr: + for subkeyname in enum_types(hkcr): + try: + with _winreg.OpenKey(hkcr, subkeyname) as subkey: + # Only check file extensions + if not subkeyname.startswith("."): + continue + # raises OSError if no 'Content Type' value + mimetype, datatype = _winreg.QueryValueEx( + subkey, 'Content Type') + if datatype != _winreg.REG_SZ: + continue + add_type(mimetype, subkeyname) + except OSError: + continue +def guess_type(url, strict=True): + """Guess the type of a file based on its URL. -def _split_filename(filename): - base, ext = os.path.splitext(filename.lower()) - while ext in suffix_map: - base, ext = os.path.splitext(base + suffix_map[ext]) - encoding = None - if ext in encodings_map: - encoding = encodings_map[ext] - base, ext = os.path.splitext(base) - return ext, encoding + Return value is a tuple (type, encoding) where type is None if the + type can't be guessed (no or unknown suffix) or a string of the + form type/subtype, usable for a MIME Content-type header; and + encoding is None for no encoding or the name of the program used + to encode (e.g. compress or gzip). The mappings are table + driven. Encoding suffixes are case sensitive; type suffixes are + first tried case sensitive, then case insensitive. + The suffixes .tgz, .taz and .tz (case sensitive!) are all mapped + to ".tar.gz". (This is table-driven too, using the dictionary + suffix_map). -def guess_type(url, strict=True): - """Return `(type, encoding)` for `url`.""" - ext, encoding = _split_filename(url) - if ext in types_map: - return types_map[ext], encoding - return None, encoding + Optional `strict' argument when false adds a bunch of commonly found, but + non-standard types. + """ + if _db is None: + init() + return _db.guess_type(url, strict) -def guess_extension(type, strict=True): - for ext, ty in types_map.items(): - if ty == type: - return ext - return None +def guess_file_type(path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path istead of URL. + """ + if _db is None: + init() + return _db.guess_file_type(path, strict=strict) def guess_all_extensions(type, strict=True): - return [ext for ext, ty in types_map.items() if ty == type] + """Guess the extensions for a file based on its MIME type. + + Return value is a list of strings giving the possible filename + extensions, including the leading dot ('.'). The extension is not + guaranteed to have been associated with any particular data + stream, but would be mapped to the MIME type `type' by + guess_type(). If no extension can be guessed for `type', None + is returned. + + Optional `strict' argument when false adds a bunch of commonly found, + but non-standard types. + """ + if _db is None: + init() + return _db.guess_all_extensions(type, strict) +def guess_extension(type, strict=True): + """Guess the extension for a file based on its MIME type. + + Return value is a string giving a filename extension, including the + leading dot ('.'). The extension is not guaranteed to have been + associated with any particular data stream, but would be mapped to the + MIME type `type' by guess_type(). If no extension can be guessed for + `type', None is returned. + + Optional `strict' argument when false adds a bunch of commonly found, + but non-standard types. + """ + if _db is None: + init() + return _db.guess_extension(type, strict) def add_type(type, ext, strict=True): - types_map[ext] = type + """Add a mapping between a type and an extension. + + When the extension is already known, the new + type will replace the old one. When the type + is already known the extension will be added + to the list of known extensions. + + If strict is true, information will be added to + list of standard types, else to the list of non-standard + types. + """ + if _db is None: + init() + return _db.add_type(type, ext, strict) def init(files=None): - global inited - inited = True + global suffix_map, types_map, encodings_map, common_types + global inited, _db + inited = True # so that MimeTypes.__init__() doesn't call us again + + if files is None or _db is None: + db = MimeTypes() + # Quick return if not supported + db.read_windows_registry() + + if files is None: + files = knownfiles + else: + files = knownfiles + list(files) + else: + db = _db + + for file in files: + if os.path.isfile(file): + db.read(file) + encodings_map = db.encodings_map + suffix_map = db.suffix_map + types_map = db.types_map[True] + common_types = db.types_map[False] + # Make the DB a global variable now that it is fully initialized + _db = db def read_mime_types(file): - return {} + try: + f = open(file, encoding='utf-8') + except OSError: + return None + with f: + db = MimeTypes() + db.readfp(f, True) + return db.types_map[True] + + +def _default_mime_types(): + global suffix_map, _suffix_map_default + global encodings_map, _encodings_map_default + global types_map, _types_map_default + global common_types, _common_types_default + + suffix_map = _suffix_map_default = { + '.svgz': '.svg.gz', + '.tgz': '.tar.gz', + '.taz': '.tar.gz', + '.tz': '.tar.gz', + '.tbz2': '.tar.bz2', + '.txz': '.tar.xz', + } + + encodings_map = _encodings_map_default = { + '.gz': 'gzip', + '.Z': 'compress', + '.bz2': 'bzip2', + '.xz': 'xz', + '.br': 'br', + } + + # Before adding new types, make sure they are either registered with IANA, + # at http://www.iana.org/assignments/media-types + # or extensions, i.e. using the x- prefix + + # If you add to these, please keep them sorted by mime type. + # Make sure the entry with the preferred file extension for a particular mime type + # appears before any others of the same mimetype. + types_map = _types_map_default = { + '.js' : 'text/javascript', + '.mjs' : 'text/javascript', + '.json' : 'application/json', + '.webmanifest': 'application/manifest+json', + '.doc' : 'application/msword', + '.dot' : 'application/msword', + '.wiz' : 'application/msword', + '.nq' : 'application/n-quads', + '.nt' : 'application/n-triples', + '.bin' : 'application/octet-stream', + '.a' : 'application/octet-stream', + '.dll' : 'application/octet-stream', + '.exe' : 'application/octet-stream', + '.o' : 'application/octet-stream', + '.obj' : 'application/octet-stream', + '.so' : 'application/octet-stream', + '.oda' : 'application/oda', + '.pdf' : 'application/pdf', + '.p7c' : 'application/pkcs7-mime', + '.ps' : 'application/postscript', + '.ai' : 'application/postscript', + '.eps' : 'application/postscript', + '.trig' : 'application/trig', + '.m3u' : 'application/vnd.apple.mpegurl', + '.m3u8' : 'application/vnd.apple.mpegurl', + '.xls' : 'application/vnd.ms-excel', + '.xlb' : 'application/vnd.ms-excel', + '.ppt' : 'application/vnd.ms-powerpoint', + '.pot' : 'application/vnd.ms-powerpoint', + '.ppa' : 'application/vnd.ms-powerpoint', + '.pps' : 'application/vnd.ms-powerpoint', + '.pwz' : 'application/vnd.ms-powerpoint', + '.wasm' : 'application/wasm', + '.bcpio' : 'application/x-bcpio', + '.cpio' : 'application/x-cpio', + '.csh' : 'application/x-csh', + '.dvi' : 'application/x-dvi', + '.gtar' : 'application/x-gtar', + '.hdf' : 'application/x-hdf', + '.h5' : 'application/x-hdf5', + '.latex' : 'application/x-latex', + '.mif' : 'application/x-mif', + '.cdf' : 'application/x-netcdf', + '.nc' : 'application/x-netcdf', + '.p12' : 'application/x-pkcs12', + '.pfx' : 'application/x-pkcs12', + '.ram' : 'application/x-pn-realaudio', + '.pyc' : 'application/x-python-code', + '.pyo' : 'application/x-python-code', + '.sh' : 'application/x-sh', + '.shar' : 'application/x-shar', + '.swf' : 'application/x-shockwave-flash', + '.sv4cpio': 'application/x-sv4cpio', + '.sv4crc' : 'application/x-sv4crc', + '.tar' : 'application/x-tar', + '.tcl' : 'application/x-tcl', + '.tex' : 'application/x-tex', + '.texi' : 'application/x-texinfo', + '.texinfo': 'application/x-texinfo', + '.roff' : 'application/x-troff', + '.t' : 'application/x-troff', + '.tr' : 'application/x-troff', + '.man' : 'application/x-troff-man', + '.me' : 'application/x-troff-me', + '.ms' : 'application/x-troff-ms', + '.ustar' : 'application/x-ustar', + '.src' : 'application/x-wais-source', + '.xsl' : 'application/xml', + '.rdf' : 'application/xml', + '.wsdl' : 'application/xml', + '.xpdl' : 'application/xml', + '.zip' : 'application/zip', + '.3gp' : 'audio/3gpp', + '.3gpp' : 'audio/3gpp', + '.3g2' : 'audio/3gpp2', + '.3gpp2' : 'audio/3gpp2', + '.aac' : 'audio/aac', + '.adts' : 'audio/aac', + '.loas' : 'audio/aac', + '.ass' : 'audio/aac', + '.au' : 'audio/basic', + '.snd' : 'audio/basic', + '.mp3' : 'audio/mpeg', + '.mp2' : 'audio/mpeg', + '.opus' : 'audio/opus', + '.aif' : 'audio/x-aiff', + '.aifc' : 'audio/x-aiff', + '.aiff' : 'audio/x-aiff', + '.ra' : 'audio/x-pn-realaudio', + '.wav' : 'audio/x-wav', + '.avif' : 'image/avif', + '.bmp' : 'image/bmp', + '.gif' : 'image/gif', + '.ief' : 'image/ief', + '.jpg' : 'image/jpeg', + '.jpe' : 'image/jpeg', + '.jpeg' : 'image/jpeg', + '.heic' : 'image/heic', + '.heif' : 'image/heif', + '.png' : 'image/png', + '.svg' : 'image/svg+xml', + '.tiff' : 'image/tiff', + '.tif' : 'image/tiff', + '.ico' : 'image/vnd.microsoft.icon', + '.webp' : 'image/webp', + '.ras' : 'image/x-cmu-raster', + '.pnm' : 'image/x-portable-anymap', + '.pbm' : 'image/x-portable-bitmap', + '.pgm' : 'image/x-portable-graymap', + '.ppm' : 'image/x-portable-pixmap', + '.rgb' : 'image/x-rgb', + '.xbm' : 'image/x-xbitmap', + '.xpm' : 'image/x-xpixmap', + '.xwd' : 'image/x-xwindowdump', + '.eml' : 'message/rfc822', + '.mht' : 'message/rfc822', + '.mhtml' : 'message/rfc822', + '.nws' : 'message/rfc822', + '.css' : 'text/css', + '.csv' : 'text/csv', + '.html' : 'text/html', + '.htm' : 'text/html', + '.md' : 'text/markdown', + '.markdown': 'text/markdown', + '.n3' : 'text/n3', + '.txt' : 'text/plain', + '.bat' : 'text/plain', + '.c' : 'text/plain', + '.h' : 'text/plain', + '.ksh' : 'text/plain', + '.pl' : 'text/plain', + '.srt' : 'text/plain', + '.rtx' : 'text/richtext', + '.rtf' : 'text/rtf', + '.tsv' : 'text/tab-separated-values', + '.vtt' : 'text/vtt', + '.py' : 'text/x-python', + '.rst' : 'text/x-rst', + '.etx' : 'text/x-setext', + '.sgm' : 'text/x-sgml', + '.sgml' : 'text/x-sgml', + '.vcf' : 'text/x-vcard', + '.xml' : 'text/xml', + '.mp4' : 'video/mp4', + '.mpeg' : 'video/mpeg', + '.m1v' : 'video/mpeg', + '.mpa' : 'video/mpeg', + '.mpe' : 'video/mpeg', + '.mpg' : 'video/mpeg', + '.mov' : 'video/quicktime', + '.qt' : 'video/quicktime', + '.webm' : 'video/webm', + '.avi' : 'video/x-msvideo', + '.movie' : 'video/x-sgi-movie', + } + + # These are non-standard types, commonly found in the wild. They will + # only match if strict=0 flag is given to the API methods. + + # Please sort these too + common_types = _common_types_default = { + '.rtf' : 'application/rtf', + '.midi': 'audio/midi', + '.mid' : 'audio/midi', + '.jpg' : 'image/jpg', + '.pict': 'image/pict', + '.pct' : 'image/pict', + '.pic' : 'image/pict', + '.xul' : 'text/xul', + } + + +_default_mime_types() + + +def _main(): + import getopt + + USAGE = """\ +Usage: mimetypes.py [options] type + +Options: + --help / -h -- print this message and exit + --lenient / -l -- additionally search of some common, but non-standard + types. + --extension / -e -- guess extension instead of type + +More than one type argument may be given. +""" + + def usage(code, msg=''): + print(USAGE) + if msg: print(msg) + sys.exit(code) + + try: + opts, args = getopt.getopt(sys.argv[1:], 'hle', + ['help', 'lenient', 'extension']) + except getopt.error as msg: + usage(1, msg) + + strict = 1 + extension = 0 + for opt, arg in opts: + if opt in ('-h', '--help'): + usage(0) + elif opt in ('-l', '--lenient'): + strict = 0 + elif opt in ('-e', '--extension'): + extension = 1 + for gtype in args: + if extension: + guess = guess_extension(gtype, strict) + if not guess: print("I don't know anything about type", gtype) + else: print(guess) + else: + guess, encoding = guess_type(gtype, strict) + if not guess: print("I don't know anything about type", gtype) + else: print('type:', guess, 'encoding:', encoding) + + +if __name__ == '__main__': + _main() diff --git a/crates/weavepy-vm/src/stdlib/python/numbers_mod.py b/crates/weavepy-vm/src/stdlib/python/numbers_mod.py index 55ee4e5..5b866a1 100644 --- a/crates/weavepy-vm/src/stdlib/python/numbers_mod.py +++ b/crates/weavepy-vm/src/stdlib/python/numbers_mod.py @@ -190,7 +190,17 @@ def denominator(self): ... def __float__(self): - return self.numerator / self.denominator + """float(self) = self.numerator / self.denominator + + It's important that this conversion use the integer's "true" + division rather than casting one side to float before dividing + so that ratios of huge integers convert without overflowing. + The explicit ``int()`` coercions let a Rational whose + numerator/denominator are themselves Integral (but not built-in + ``int``) still convert — e.g. ``DummyIntegral`` in the + numeric-tower tests, whose own ``__truediv__`` declines. + """ + return int(self.numerator) / int(self.denominator) class Integral(Rational): diff --git a/crates/weavepy-vm/src/stdlib/python/operator_mod.py b/crates/weavepy-vm/src/stdlib/python/operator_mod.py index f6fc0f9..02ccdaa 100644 --- a/crates/weavepy-vm/src/stdlib/python/operator_mod.py +++ b/crates/weavepy-vm/src/stdlib/python/operator_mod.py @@ -1,330 +1,467 @@ -"""Operator interface — WeavePy port of CPython's ``operator``. +""" +Operator Interface + +This module exports a set of functions corresponding to the intrinsic +operators of Python. For example, operator.add(x, y) is equivalent +to the expression x+y. The function names are those used for special +methods; variants without leading and trailing '__' are also provided +for convenience. -Provides function form of the standard operators (``operator.add`` -etc.) plus the higher-order ``itemgetter``, ``attrgetter`` and -``methodcaller`` helpers. +This is the pure Python implementation of the module. """ +__all__ = ['abs', 'add', 'and_', 'attrgetter', 'call', 'concat', 'contains', 'countOf', + 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', + 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', + 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', + 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', + 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', + 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', + 'setitem', 'sub', 'truediv', 'truth', 'xor'] + +from builtins import abs as _abs + + +# Comparison Operations *******************************************************# def lt(a, b): + "Same as a < b." return a < b - def le(a, b): + "Same as a <= b." return a <= b - def eq(a, b): + "Same as a == b." return a == b - def ne(a, b): + "Same as a != b." return a != b - -def gt(a, b): - return a > b - - def ge(a, b): + "Same as a >= b." return a >= b +def gt(a, b): + "Same as a > b." + return a > b -__lt__ = lt -__le__ = le -__eq__ = eq -__ne__ = ne -__gt__ = gt -__ge__ = ge - +# Logical Operations **********************************************************# def not_(a): + "Same as not a." return not a - def truth(a): - return bool(a) - + "Return True if a is true, False otherwise." + return True if a else False def is_(a, b): + "Same as a is b." return a is b - def is_not(a, b): + "Same as a is not b." return a is not b +# Mathematical/Bitwise Operations *********************************************# def abs(a): - import builtins - return builtins.abs(a) - + "Same as abs(a)." + return _abs(a) def add(a, b): + "Same as a + b." return a + b - def and_(a, b): + "Same as a & b." return a & b - def floordiv(a, b): + "Same as a // b." return a // b - def index(a): - if hasattr(a, "__index__"): - return a.__index__() - raise TypeError("object is not indexable") - + "Same as a.__index__()." + return a.__index__() def inv(a): + "Same as ~a." return ~a - - -def invert(a): - return ~a - +invert = inv def lshift(a, b): + "Same as a << b." return a << b - def mod(a, b): + "Same as a % b." return a % b - def mul(a, b): + "Same as a * b." return a * b - def matmul(a, b): + "Same as a @ b." return a @ b - def neg(a): + "Same as -a." return -a - def or_(a, b): + "Same as a | b." return a | b - def pos(a): + "Same as +a." return +a - def pow(a, b): - import builtins - return builtins.pow(a, b) - + "Same as a ** b." + return a ** b def rshift(a, b): + "Same as a >> b." return a >> b - def sub(a, b): + "Same as a - b." return a - b - def truediv(a, b): + "Same as a / b." return a / b - def xor(a, b): + "Same as a ^ b." return a ^ b +# Sequence Operations *********************************************************# def concat(a, b): - if hasattr(a, "__add__"): - return a + b - raise TypeError("concat: not a sequence") - + "Same as a + b, for a and b sequences." + if not hasattr(a, '__getitem__'): + msg = "'%s' object can't be concatenated" % type(a).__name__ + raise TypeError(msg) + return a + b def contains(a, b): + "Same as b in a (note reversed operands)." return b in a - def countOf(a, b): - n = 0 - for x in a: - if x == b: - n += 1 - return n - - -def indexOf(a, b): - for i, x in enumerate(a): - if x == b: - return i - raise ValueError("indexOf(a, b): b not in a") + "Return the number of items in a which are, or which equal, b." + count = 0 + for i in a: + if i is b or i == b: + count += 1 + return count +def delitem(a, b): + "Same as del a[b]." + del a[b] def getitem(a, b): + "Same as a[b]." return a[b] +def indexOf(a, b): + "Return the first index of b in a." + for i, j in enumerate(a): + if j is b or j == b: + return i + else: + raise ValueError('sequence.index(x): x not in sequence') def setitem(a, b, c): + "Same as a[b] = c." a[b] = c +def length_hint(obj, default=0): + """ + Return an estimate of the number of items in obj. + This is useful for presizing containers when building from an iterable. + + If the object supports len(), the result will be exact. Otherwise, it may + over- or under-estimate by an arbitrary amount. The result will be an + integer >= 0. + """ + if not isinstance(default, int): + msg = ("'%s' object cannot be interpreted as an integer" % + type(default).__name__) + raise TypeError(msg) -def delitem(a, b): - del a[b] - + try: + return len(obj) + except TypeError: + pass -def length_hint(obj, default=0): try: - return obj.__length_hint__() + hint = type(obj).__length_hint__ except AttributeError: return default + try: + val = hint(obj) + except TypeError: + return default + if val is NotImplemented: + return default + if not isinstance(val, int): + msg = ('__length_hint__ must be integer, not %s' % + type(val).__name__) + raise TypeError(msg) + if val < 0: + msg = '__length_hint__() should return >= 0' + raise ValueError(msg) + return val -class attrgetter: - """Return a callable that fetches attribute(s) from an object.""" - - __slots__ = ("_attrs", "_call") +# Other Operations ************************************************************# - def __init__(self, attr, *more): - attrs = (attr,) + more - for a in attrs: - if not isinstance(a, str): - raise TypeError("attribute name must be str") - self._attrs = attrs +def call(obj, /, *args, **kwargs): + """Same as obj(*args, **kwargs).""" + return obj(*args, **kwargs) - def resolve(obj, name): - for part in name.split("."): - obj = getattr(obj, part) - return obj +# Generalized Lookup Objects **************************************************# - if len(attrs) == 1: - self._call = lambda obj: resolve(obj, attrs[0]) +class attrgetter: + """ + Return a callable object that fetches the given attribute(s) from its operand. + After f = attrgetter('name'), the call f(r) returns r.name. + After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date). + After h = attrgetter('name.first', 'name.last'), the call h(r) returns + (r.name.first, r.name.last). + """ + __slots__ = ('_attrs', '_call') + + def __init__(self, attr, /, *attrs): + if not attrs: + if not isinstance(attr, str): + raise TypeError('attribute name must be a string') + self._attrs = (attr,) + names = attr.split('.') + def func(obj): + for name in names: + obj = getattr(obj, name) + return obj + self._call = func else: - self._call = lambda obj: tuple(resolve(obj, a) for a in attrs) + self._attrs = (attr,) + attrs + getters = tuple(map(attrgetter, self._attrs)) + def func(obj): + return tuple(getter(obj) for getter in getters) + self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): - return "operator.attrgetter({})".format(", ".join(repr(a) for a in self._attrs)) + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__qualname__, + ', '.join(map(repr, self._attrs))) + def __reduce__(self): + return self.__class__, self._attrs class itemgetter: - """Return a callable that fetches the given item(s) from a sequence.""" - - __slots__ = ("_items",) - - def __init__(self, item, *more): - self._items = (item,) + more + """ + Return a callable object that fetches the given item(s) from its operand. + After f = itemgetter(2), the call f(r) returns r[2]. + After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3]) + """ + __slots__ = ('_items', '_call') + + def __init__(self, item, /, *items): + if not items: + self._items = (item,) + def func(obj): + return obj[item] + self._call = func + else: + self._items = items = (item,) + items + def func(obj): + return tuple(obj[i] for i in items) + self._call = func - def __call__(self, obj): - if len(self._items) == 1: - return obj[self._items[0]] - return tuple(obj[i] for i in self._items) + def __call__(self, obj, /): + return self._call(obj) def __repr__(self): - return "operator.itemgetter({})".format(", ".join(repr(i) for i in self._items)) + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__name__, + ', '.join(map(repr, self._items))) + def __reduce__(self): + return self.__class__, self._items class methodcaller: - """Return a callable that invokes ``name`` on its argument.""" - - __slots__ = ("_name", "_args", "_kwargs") - - def __init__(self, name, *args, **kwargs): + """ + Return a callable object that calls the given method on its operand. + After f = methodcaller('name'), the call f(r) returns r.name(). + After g = methodcaller('name', 'date', foo=1), the call g(r) returns + r.name('date', foo=1). + """ + __slots__ = ('_name', '_args', '_kwargs') + + def __init__(self, name, /, *args, **kwargs): self._name = name + if not isinstance(self._name, str): + raise TypeError('method name must be a string') self._args = args self._kwargs = kwargs - def __call__(self, obj): + def __call__(self, obj, /): return getattr(obj, self._name)(*self._args, **self._kwargs) def __repr__(self): - parts = [repr(self._name)] - parts.extend(repr(a) for a in self._args) - for k, v in self._kwargs.items(): - parts.append("{}={!r}".format(k, v)) - return "operator.methodcaller({})".format(", ".join(parts)) + args = [repr(self._name)] + args.extend(map(repr, self._args)) + args.extend('%s=%r' % (k, v) for k, v in self._kwargs.items()) + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__name__, + ', '.join(args)) + + def __reduce__(self): + if not self._kwargs: + return self.__class__, (self._name,) + self._args + else: + from functools import partial + return partial(self.__class__, self._name, **self._kwargs), self._args +# In-place Operations *********************************************************# + def iadd(a, b): + "Same as a += b." a += b return a - def iand(a, b): + "Same as a &= b." a &= b return a - def iconcat(a, b): + "Same as a += b, for a and b sequences." + if not hasattr(a, '__getitem__'): + msg = "'%s' object can't be concatenated" % type(a).__name__ + raise TypeError(msg) a += b return a - def ifloordiv(a, b): + "Same as a //= b." a //= b return a - def ilshift(a, b): + "Same as a <<= b." a <<= b return a - def imod(a, b): + "Same as a %= b." a %= b return a - def imul(a, b): + "Same as a *= b." a *= b return a - def imatmul(a, b): + "Same as a @= b." a @= b return a - def ior(a, b): + "Same as a |= b." a |= b return a - def ipow(a, b): - a **= b + "Same as a **= b." + a **=b return a - def irshift(a, b): + "Same as a >>= b." a >>= b return a - def isub(a, b): + "Same as a -= b." a -= b return a - def itruediv(a, b): + "Same as a /= b." a /= b return a - def ixor(a, b): + "Same as a ^= b." a ^= b return a -__all__ = [ - "lt", "le", "eq", "ne", "gt", "ge", - "not_", "truth", "is_", "is_not", - "abs", "add", "and_", "floordiv", "index", "inv", "invert", - "lshift", "mod", "mul", "matmul", "neg", "or_", "pos", "pow", - "rshift", "sub", "truediv", "xor", - "concat", "contains", "countOf", "indexOf", - "getitem", "setitem", "delitem", "length_hint", - "attrgetter", "itemgetter", "methodcaller", - "iadd", "iand", "iconcat", "ifloordiv", "ilshift", "imod", - "imul", "imatmul", "ior", "ipow", "irshift", "isub", - "itruediv", "ixor", -] +try: + from _operator import * +except ImportError: + pass +else: + from _operator import __doc__ + +# All of these "__func__ = func" assignments have to happen after importing +# from _operator to make sure they're set to the right function +__lt__ = lt +__le__ = le +__eq__ = eq +__ne__ = ne +__ge__ = ge +__gt__ = gt +__not__ = not_ +__abs__ = abs +__add__ = add +__and__ = and_ +__call__ = call +__floordiv__ = floordiv +__index__ = index +__inv__ = inv +__invert__ = invert +__lshift__ = lshift +__mod__ = mod +__mul__ = mul +__matmul__ = matmul +__neg__ = neg +__or__ = or_ +__pos__ = pos +__pow__ = pow +__rshift__ = rshift +__sub__ = sub +__truediv__ = truediv +__xor__ = xor +__concat__ = concat +__contains__ = contains +__delitem__ = delitem +__getitem__ = getitem +__setitem__ = setitem +__iadd__ = iadd +__iand__ = iand +__iconcat__ = iconcat +__ifloordiv__ = ifloordiv +__ilshift__ = ilshift +__imod__ = imod +__imul__ = imul +__imatmul__ = imatmul +__ior__ = ior +__ipow__ = ipow +__irshift__ = irshift +__isub__ = isub +__itruediv__ = itruediv +__ixor__ = ixor diff --git a/crates/weavepy-vm/src/stdlib/python/pickle.py b/crates/weavepy-vm/src/stdlib/python/pickle.py index 145b3d3..d7edd29 100644 --- a/crates/weavepy-vm/src/stdlib/python/pickle.py +++ b/crates/weavepy-vm/src/stdlib/python/pickle.py @@ -103,6 +103,24 @@ def load(file, *, fix_imports=True, encoding="ASCII", errors="strict"): # --- pickler -------------------------------------------------------------- +def _resolves_to_self(module, qualname, obj): + """True when ``module.qualname`` imports back to *obj* itself. + + This is CPython's ``save_global`` self-consistency check: an object is + only safe to pickle by reference (functions, classes, module globals) + when the dotted name found in its declaring module *is* that object. + Callable instances inherit their class's ``__qualname__`` and would + otherwise be mistaken for the class. + """ + try: + target = __import__(module, fromlist=["_"]) + for part in qualname.split("."): + target = getattr(target, part) + return target is obj + except Exception: + return False + + class _Pickler: def __init__(self, buf, protocol): self._buf = buf @@ -184,7 +202,13 @@ def _save(self, obj): getattr(obj, "__qualname__", None) or getattr(obj, "__name__", None) ) - if qualname: + # Only pickle by name when that name actually resolves back to + # *this* object (CPython's `save_global` self-check). A callable + # *instance* — e.g. `operator.attrgetter('x')` — inherits its + # class's `__qualname__`, so without this guard it would be + # saved as the bare class and unpickle to the class object + # rather than round-tripping through `__reduce__`. + if qualname and _resolves_to_self(module, qualname, obj): self._save_global(module, qualname) return # Arbitrary instances — try __reduce_ex__ / __reduce__ (the diff --git a/crates/weavepy-vm/src/stdlib/python/reprlib.py b/crates/weavepy-vm/src/stdlib/python/reprlib.py new file mode 100644 index 0000000..f683185 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/reprlib.py @@ -0,0 +1,230 @@ +"""Redo the builtin repr() (representation) but with limits on most sizes.""" + +__all__ = ["Repr", "repr", "recursive_repr"] + +import builtins +from itertools import islice +from _thread import get_ident + +def recursive_repr(fillvalue='...'): + 'Decorator to make a repr function return fillvalue for a recursive call' + + def decorating_function(user_function): + repr_running = set() + + def wrapper(self): + key = id(self), get_ident() + if key in repr_running: + return fillvalue + repr_running.add(key) + try: + result = user_function(self) + finally: + repr_running.discard(key) + return result + + # Can't use functools.wraps() here because of bootstrap issues + wrapper.__module__ = getattr(user_function, '__module__') + wrapper.__doc__ = getattr(user_function, '__doc__') + wrapper.__name__ = getattr(user_function, '__name__') + wrapper.__qualname__ = getattr(user_function, '__qualname__') + wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + wrapper.__type_params__ = getattr(user_function, '__type_params__', ()) + wrapper.__wrapped__ = user_function + return wrapper + + return decorating_function + +class Repr: + _lookup = { + 'tuple': 'builtins', + 'list': 'builtins', + 'array': 'array', + 'set': 'builtins', + 'frozenset': 'builtins', + 'deque': 'collections', + 'dict': 'builtins', + 'str': 'builtins', + 'int': 'builtins' + } + + def __init__( + self, *, maxlevel=6, maxtuple=6, maxlist=6, maxarray=5, maxdict=4, + maxset=6, maxfrozenset=6, maxdeque=6, maxstring=30, maxlong=40, + maxother=30, fillvalue='...', indent=None, + ): + self.maxlevel = maxlevel + self.maxtuple = maxtuple + self.maxlist = maxlist + self.maxarray = maxarray + self.maxdict = maxdict + self.maxset = maxset + self.maxfrozenset = maxfrozenset + self.maxdeque = maxdeque + self.maxstring = maxstring + self.maxlong = maxlong + self.maxother = maxother + self.fillvalue = fillvalue + self.indent = indent + + def repr(self, x): + return self.repr1(x, self.maxlevel) + + def repr1(self, x, level): + cls = type(x) + typename = cls.__name__ + + if ' ' in typename: + parts = typename.split() + typename = '_'.join(parts) + + method = getattr(self, 'repr_' + typename, None) + if method: + # not defined in this class + if typename not in self._lookup: + return method(x, level) + module = getattr(cls, '__module__', None) + # defined in this class and is the module intended + if module == self._lookup[typename]: + return method(x, level) + + return self.repr_instance(x, level) + + def _join(self, pieces, level): + if self.indent is None: + return ', '.join(pieces) + if not pieces: + return '' + indent = self.indent + if isinstance(indent, int): + if indent < 0: + raise ValueError( + f'Repr.indent cannot be negative int (was {indent!r})' + ) + indent *= ' ' + try: + sep = ',\n' + (self.maxlevel - level + 1) * indent + except TypeError as error: + raise TypeError( + f'Repr.indent must be a str, int or None, not {type(indent)}' + ) from error + return sep.join(('', *pieces, ''))[1:-len(indent) or None] + + def _repr_iterable(self, x, level, left, right, maxiter, trail=''): + n = len(x) + if level <= 0 and n: + s = self.fillvalue + else: + newlevel = level - 1 + repr1 = self.repr1 + pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)] + if n > maxiter: + pieces.append(self.fillvalue) + s = self._join(pieces, level) + if n == 1 and trail and self.indent is None: + right = trail + right + return '%s%s%s' % (left, s, right) + + def repr_tuple(self, x, level): + return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',') + + def repr_list(self, x, level): + return self._repr_iterable(x, level, '[', ']', self.maxlist) + + def repr_array(self, x, level): + if not x: + return "array('%s')" % x.typecode + header = "array('%s', [" % x.typecode + return self._repr_iterable(x, level, header, '])', self.maxarray) + + def repr_set(self, x, level): + if not x: + return 'set()' + x = _possibly_sorted(x) + return self._repr_iterable(x, level, '{', '}', self.maxset) + + def repr_frozenset(self, x, level): + if not x: + return 'frozenset()' + x = _possibly_sorted(x) + return self._repr_iterable(x, level, 'frozenset({', '})', + self.maxfrozenset) + + def repr_deque(self, x, level): + return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque) + + def repr_dict(self, x, level): + n = len(x) + if n == 0: + return '{}' + if level <= 0: + return '{' + self.fillvalue + '}' + newlevel = level - 1 + repr1 = self.repr1 + pieces = [] + for key in islice(_possibly_sorted(x), self.maxdict): + keyrepr = repr1(key, newlevel) + valrepr = repr1(x[key], newlevel) + pieces.append('%s: %s' % (keyrepr, valrepr)) + if n > self.maxdict: + pieces.append(self.fillvalue) + s = self._join(pieces, level) + return '{%s}' % (s,) + + def repr_str(self, x, level): + s = builtins.repr(x[:self.maxstring]) + if len(s) > self.maxstring: + i = max(0, (self.maxstring-3)//2) + j = max(0, self.maxstring-3-i) + s = builtins.repr(x[:i] + x[len(x)-j:]) + s = s[:i] + self.fillvalue + s[len(s)-j:] + return s + + def repr_int(self, x, level): + try: + s = builtins.repr(x) + except ValueError as exc: + assert 'sys.set_int_max_str_digits()' in str(exc) + # Those imports must be deferred due to Python's build system + # where the reprlib module is imported before the math module. + import math, sys + # Integers with more than sys.get_int_max_str_digits() digits + # are rendered differently as their repr() raises a ValueError. + # See https://github.com/python/cpython/issues/135487. + k = 1 + int(math.log10(abs(x))) + # Note: math.log10(abs(x)) may be overestimated or underestimated, + # but for simplicity, we do not compute the exact number of digits. + max_digits = sys.get_int_max_str_digits() + return (f'<{x.__class__.__name__} instance with roughly {k} ' + f'digits (limit at {max_digits}) at 0x{id(x):x}>') + if len(s) > self.maxlong: + i = max(0, (self.maxlong-3)//2) + j = max(0, self.maxlong-3-i) + s = s[:i] + self.fillvalue + s[len(s)-j:] + return s + + def repr_instance(self, x, level): + try: + s = builtins.repr(x) + # Bugs in x.__repr__() can cause arbitrary + # exceptions -- then make up something + except Exception: + return '<%s instance at %#x>' % (x.__class__.__name__, id(x)) + if len(s) > self.maxother: + i = max(0, (self.maxother-3)//2) + j = max(0, self.maxother-3-i) + s = s[:i] + self.fillvalue + s[len(s)-j:] + return s + + +def _possibly_sorted(x): + # Since not all sequences of items can be sorted and comparison + # functions may raise arbitrary exceptions, return an unsorted + # sequence in that case. + try: + return sorted(x) + except Exception: + return list(x) + +aRepr = Repr() +repr = aRepr.repr diff --git a/crates/weavepy-vm/src/stdlib/python/runpy.py b/crates/weavepy-vm/src/stdlib/python/runpy.py index 52e8c29..6ae6181 100644 --- a/crates/weavepy-vm/src/stdlib/python/runpy.py +++ b/crates/weavepy-vm/src/stdlib/python/runpy.py @@ -89,6 +89,39 @@ def _make_globals(mod_name, file, spec, loader, pkg): } +def _run_module_code(code, init_globals=None, mod_name=None, mod_spec=None, + pkg_name=None, script_name=None): + """Exec ``code`` as ``mod_name`` inside a fresh temporary module that + is registered in ``sys.modules`` for the duration of the run, then + removed — CPython's ``runpy._TempModule`` + ``_ModifiedArgv0``. + + Registering the module matters: code that introspects + ``sys.modules[__name__]`` mid-execution (e.g. ``enum.global_enum`` + hoisting members into the running module's globals, as ``calendar`` + does) must see the *same* namespace it is executing in.""" + import types + saved_module = sys.modules.get(mod_name) + saved_argv0 = sys.argv[0] if sys.argv else None + temp_module = types.ModuleType(mod_name) + sys.modules[mod_name] = temp_module + try: + if script_name is not None and sys.argv: + sys.argv[0] = script_name + mod_globals = temp_module.__dict__ + _run_code(code, mod_globals, init_globals, mod_name, mod_spec, + pkg_name, script_name) + # Return a snapshot so callers can't mutate the (now-removed) + # temporary module's namespace. + return dict(mod_globals) + finally: + if saved_argv0 is not None and sys.argv: + sys.argv[0] = saved_argv0 + if saved_module is not None: + sys.modules[mod_name] = saved_module + elif mod_name in sys.modules: + del sys.modules[mod_name] + + def run_module(mod_name, init_globals=None, run_name=None, alter_sys=False): """Locate ``mod_name`` and exec it with ``__name__`` set.""" if run_name is None: @@ -112,7 +145,12 @@ def run_module(mod_name, init_globals=None, run_name=None, alter_sys=False): frozen_source = getter(mod_name) if frozen_source is not None: source = frozen_source - filename = filename or f"" + # A frozen module has no real path; synthesise a CPython-like + # ``.py`` so ``-m``'s ``sys.argv[0]`` / argparse + # ``prog`` and tracebacks read naturally (``calendar.py``) + # rather than the opaque ```` placeholder. + if not filename or filename.startswith("", "exec") + if alter_sys: + return _run_module_code(code, init_globals, run_name, spec, pkg, + filename) run_globals = _make_globals(run_name, filename, spec, None, pkg) return _run_code(code, run_globals, init_globals, run_name, spec, pkg, filename) diff --git a/crates/weavepy-vm/src/stdlib/python/struct.py b/crates/weavepy-vm/src/stdlib/python/struct.py index 001af2a..e8e0004 100644 --- a/crates/weavepy-vm/src/stdlib/python/struct.py +++ b/crates/weavepy-vm/src/stdlib/python/struct.py @@ -28,41 +28,87 @@ def call(*args, **kwargs): unpack = _wrap(_impl.unpack) pack_into = _wrap(_impl.pack_into) unpack_from = _wrap(_impl.unpack_from) -_iter_unpack_impl = _wrap(_impl.iter_unpack) + + +def _iter_unpack(fmt, buffer, size): + # CPython validates the buffer length up front (a `struct.error` is + # raised by `iter_unpack` itself, not lazily on the first `next()`), + # and rejects a zero-width format outright. + if size == 0: + raise error("cannot iteratively unpack with a struct of length 0") + if len(buffer) % size != 0: + raise error( + "iterative unpacking requires a buffer of a multiple of " + f"{size} bytes" + ) + + def _gen(): + for off in range(0, len(buffer), size): + yield unpack_from(fmt, buffer, off) + + return _gen() def iter_unpack(fmt, buffer): """Iterate over `buffer` in `calcsize(fmt)` chunks.""" - items = _iter_unpack_impl(fmt, buffer) - return iter(items) + return _iter_unpack(fmt, buffer, calcsize(fmt)) class Struct: """Pre-compiled binary format. Mirrors `struct.Struct`.""" - __slots__ = ("format", "size", "_fmt") + def __new__(cls, *args, **kwargs): + self = super().__new__(cls) + # A `Struct` created via `__new__` alone (no `__init__`) is + # "half-initialized": CPython's C type leaves `s_format == NULL` + # and `s_size == -1`, and every operation raises `RuntimeError` + # until `__init__` runs. + self._fmt = None + self.size = -1 + return self def __init__(self, fmt): if isinstance(fmt, bytes): fmt = fmt.decode("ascii") - self.format = fmt self._fmt = fmt self.size = calcsize(fmt) + def _ensure_initialized(self): + if self._fmt is None: + raise RuntimeError("Struct.__init__() was not called") + + @property + def format(self): + self._ensure_initialized() + return self._fmt + def pack(self, *values): + self._ensure_initialized() return pack(self._fmt, *values) def unpack(self, buffer): + self._ensure_initialized() return unpack(self._fmt, buffer) def pack_into(self, buffer, offset, *values): + self._ensure_initialized() return pack_into(self._fmt, buffer, offset, *values) def unpack_from(self, buffer, offset=0): + self._ensure_initialized() return unpack_from(self._fmt, buffer, offset) def iter_unpack(self, buffer): - return iter_unpack(self._fmt, buffer) + self._ensure_initialized() + return _iter_unpack(self._fmt, buffer, self.size) + + def __repr__(self): + self._ensure_initialized() + return f"Struct({self._fmt!r})" + + def __sizeof__(self): + self._ensure_initialized() + return object.__sizeof__(self) def _new_struct(fmt): diff --git a/crates/weavepy-vm/src/stdlib/python/test_list_tests.py b/crates/weavepy-vm/src/stdlib/python/test_list_tests.py new file mode 100644 index 0000000..65dfa41 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/test_list_tests.py @@ -0,0 +1,577 @@ +""" +Tests common to list and UserList.UserList +""" + +import sys +from functools import cmp_to_key + +from test import seq_tests +from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit + + +class CommonTest(seq_tests.CommonTest): + + def test_init(self): + # Iterable arg is optional + self.assertEqual(self.type2test([]), self.type2test()) + + # Init clears previous values + a = self.type2test([1, 2, 3]) + a.__init__() + self.assertEqual(a, self.type2test([])) + + # Init overwrites previous values + a = self.type2test([1, 2, 3]) + a.__init__([4, 5, 6]) + self.assertEqual(a, self.type2test([4, 5, 6])) + + # Mutables always return a new object + b = self.type2test(a) + self.assertNotEqual(id(a), id(b)) + self.assertEqual(a, b) + + def test_getitem_error(self): + a = self.type2test([]) + msg = "list indices must be integers or slices" + with self.assertRaisesRegex(TypeError, msg): + a['a'] + + def test_setitem_error(self): + a = self.type2test([]) + msg = "list indices must be integers or slices" + with self.assertRaisesRegex(TypeError, msg): + a['a'] = "python" + + def test_repr(self): + l0 = [] + l2 = [0, 1, 2] + a0 = self.type2test(l0) + a2 = self.type2test(l2) + + self.assertEqual(str(a0), str(l0)) + self.assertEqual(repr(a0), repr(l0)) + self.assertEqual(repr(a2), repr(l2)) + self.assertEqual(str(a2), "[0, 1, 2]") + self.assertEqual(repr(a2), "[0, 1, 2]") + + a2.append(a2) + a2.append(3) + self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") + self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + + def test_repr_deep(self): + a = self.type2test([]) + for i in range(get_c_recursion_limit() + 1): + a = self.type2test([a]) + self.assertRaises(RecursionError, repr, a) + + def test_set_subscript(self): + a = self.type2test(range(20)) + self.assertRaises(ValueError, a.__setitem__, slice(0, 10, 0), [1,2,3]) + self.assertRaises(TypeError, a.__setitem__, slice(0, 10), 1) + self.assertRaises(ValueError, a.__setitem__, slice(0, 10, 2), [1,2]) + self.assertRaises(TypeError, a.__getitem__, 'x', 1) + a[slice(2,10,3)] = [1,2,3] + self.assertEqual(a, self.type2test([0, 1, 1, 3, 4, 2, 6, 7, 3, + 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19])) + + def test_reversed(self): + a = self.type2test(range(20)) + r = reversed(a) + self.assertEqual(list(r), self.type2test(range(19, -1, -1))) + self.assertRaises(StopIteration, next, r) + self.assertEqual(list(reversed(self.type2test())), + self.type2test()) + # Bug 3689: make sure list-reversed-iterator doesn't have __len__ + self.assertRaises(TypeError, len, reversed([1,2,3])) + + def test_setitem(self): + a = self.type2test([0, 1]) + a[0] = 0 + a[1] = 100 + self.assertEqual(a, self.type2test([0, 100])) + a[-1] = 200 + self.assertEqual(a, self.type2test([0, 200])) + a[-2] = 100 + self.assertEqual(a, self.type2test([100, 200])) + self.assertRaises(IndexError, a.__setitem__, -3, 200) + self.assertRaises(IndexError, a.__setitem__, 2, 200) + + a = self.type2test([]) + self.assertRaises(IndexError, a.__setitem__, 0, 200) + self.assertRaises(IndexError, a.__setitem__, -1, 200) + self.assertRaises(TypeError, a.__setitem__) + + a = self.type2test([0,1,2,3,4]) + a[0] = 1 + a[1] = 2 + a[2] = 3 + self.assertEqual(a, self.type2test([1,2,3,3,4])) + a[0] = 5 + a[1] = 6 + a[2] = 7 + self.assertEqual(a, self.type2test([5,6,7,3,4])) + a[-2] = 88 + a[-1] = 99 + self.assertEqual(a, self.type2test([5,6,7,88,99])) + a[-2] = 8 + a[-1] = 9 + self.assertEqual(a, self.type2test([5,6,7,8,9])) + + msg = "list indices must be integers or slices" + with self.assertRaisesRegex(TypeError, msg): + a['a'] = "python" + + def test_delitem(self): + a = self.type2test([0, 1]) + del a[1] + self.assertEqual(a, [0]) + del a[0] + self.assertEqual(a, []) + + a = self.type2test([0, 1]) + del a[-2] + self.assertEqual(a, [1]) + del a[-1] + self.assertEqual(a, []) + + a = self.type2test([0, 1]) + self.assertRaises(IndexError, a.__delitem__, -3) + self.assertRaises(IndexError, a.__delitem__, 2) + + a = self.type2test([]) + self.assertRaises(IndexError, a.__delitem__, 0) + + self.assertRaises(TypeError, a.__delitem__) + + def test_setslice(self): + l = [0, 1] + a = self.type2test(l) + + for i in range(-3, 4): + a[:i] = l[:i] + self.assertEqual(a, l) + a2 = a[:] + a2[:i] = a[:i] + self.assertEqual(a2, a) + a[i:] = l[i:] + self.assertEqual(a, l) + a2 = a[:] + a2[i:] = a[i:] + self.assertEqual(a2, a) + for j in range(-3, 4): + a[i:j] = l[i:j] + self.assertEqual(a, l) + a2 = a[:] + a2[i:j] = a[i:j] + self.assertEqual(a2, a) + + aa2 = a2[:] + aa2[:0] = [-2, -1] + self.assertEqual(aa2, [-2, -1, 0, 1]) + aa2[0:] = [] + self.assertEqual(aa2, []) + + a = self.type2test([1, 2, 3, 4, 5]) + a[:-1] = a + self.assertEqual(a, self.type2test([1, 2, 3, 4, 5, 5])) + a = self.type2test([1, 2, 3, 4, 5]) + a[1:] = a + self.assertEqual(a, self.type2test([1, 1, 2, 3, 4, 5])) + a = self.type2test([1, 2, 3, 4, 5]) + a[1:-1] = a + self.assertEqual(a, self.type2test([1, 1, 2, 3, 4, 5, 5])) + + a = self.type2test([]) + a[:] = tuple(range(10)) + self.assertEqual(a, self.type2test(range(10))) + + self.assertRaises(TypeError, a.__setitem__, slice(0, 1, 5)) + + self.assertRaises(TypeError, a.__setitem__) + + def test_slice_assign_iterator(self): + x = self.type2test(range(5)) + x[0:3] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0, 3, 4])) + + x[:] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0])) + + def test_delslice(self): + a = self.type2test([0, 1]) + del a[1:2] + del a[0:1] + self.assertEqual(a, self.type2test([])) + + a = self.type2test([0, 1]) + del a[1:2] + del a[0:1] + self.assertEqual(a, self.type2test([])) + + a = self.type2test([0, 1]) + del a[-2:-1] + self.assertEqual(a, self.type2test([1])) + + a = self.type2test([0, 1]) + del a[-2:-1] + self.assertEqual(a, self.type2test([1])) + + a = self.type2test([0, 1]) + del a[1:] + del a[:1] + self.assertEqual(a, self.type2test([])) + + a = self.type2test([0, 1]) + del a[1:] + del a[:1] + self.assertEqual(a, self.type2test([])) + + a = self.type2test([0, 1]) + del a[-1:] + self.assertEqual(a, self.type2test([0])) + + a = self.type2test([0, 1]) + del a[-1:] + self.assertEqual(a, self.type2test([0])) + + a = self.type2test([0, 1]) + del a[:] + self.assertEqual(a, self.type2test([])) + + def test_append(self): + a = self.type2test([]) + a.append(0) + a.append(1) + a.append(2) + self.assertEqual(a, self.type2test([0, 1, 2])) + + self.assertRaises(TypeError, a.append) + + def test_extend(self): + a1 = self.type2test([0]) + a2 = self.type2test((0, 1)) + a = a1[:] + a.extend(a2) + self.assertEqual(a, a1 + a2) + + a.extend(self.type2test([])) + self.assertEqual(a, a1 + a2) + + a.extend(a) + self.assertEqual(a, self.type2test([0, 0, 1, 0, 0, 1])) + + a = self.type2test("spam") + a.extend("eggs") + self.assertEqual(a, list("spameggs")) + + self.assertRaises(TypeError, a.extend, None) + self.assertRaises(TypeError, a.extend) + + # overflow test. issue1621 + class CustomIter: + def __iter__(self): + return self + def __next__(self): + raise StopIteration + def __length_hint__(self): + return sys.maxsize + a = self.type2test([1,2,3,4]) + a.extend(CustomIter()) + self.assertEqual(a, [1,2,3,4]) + + + def test_insert(self): + a = self.type2test([0, 1, 2]) + a.insert(0, -2) + a.insert(1, -1) + a.insert(2, 0) + self.assertEqual(a, [-2, -1, 0, 0, 1, 2]) + + b = a[:] + b.insert(-2, "foo") + b.insert(-200, "left") + b.insert(200, "right") + self.assertEqual(b, self.type2test(["left",-2,-1,0,0,"foo",1,2,"right"])) + + self.assertRaises(TypeError, a.insert) + + def test_pop(self): + a = self.type2test([-1, 0, 1]) + a.pop() + self.assertEqual(a, [-1, 0]) + a.pop(0) + self.assertEqual(a, [0]) + self.assertRaises(IndexError, a.pop, 5) + a.pop(0) + self.assertEqual(a, []) + self.assertRaises(IndexError, a.pop) + self.assertRaises(TypeError, a.pop, 42, 42) + a = self.type2test([0, 10, 20, 30, 40]) + + def test_remove(self): + a = self.type2test([0, 0, 1]) + a.remove(1) + self.assertEqual(a, [0, 0]) + a.remove(0) + self.assertEqual(a, [0]) + a.remove(0) + self.assertEqual(a, []) + + self.assertRaises(ValueError, a.remove, 0) + + self.assertRaises(TypeError, a.remove) + + a = self.type2test([1, 2]) + self.assertRaises(ValueError, a.remove, NEVER_EQ) + self.assertEqual(a, [1, 2]) + a.remove(ALWAYS_EQ) + self.assertEqual(a, [2]) + a = self.type2test([ALWAYS_EQ]) + a.remove(1) + self.assertEqual(a, []) + a = self.type2test([ALWAYS_EQ]) + a.remove(NEVER_EQ) + self.assertEqual(a, []) + a = self.type2test([NEVER_EQ]) + self.assertRaises(ValueError, a.remove, ALWAYS_EQ) + + class BadExc(Exception): + pass + + class BadCmp: + def __eq__(self, other): + if other == 2: + raise BadExc() + return False + + a = self.type2test([0, 1, 2, 3]) + self.assertRaises(BadExc, a.remove, BadCmp()) + + class BadCmp2: + def __eq__(self, other): + raise BadExc() + + d = self.type2test('abcdefghcij') + d.remove('c') + self.assertEqual(d, self.type2test('abdefghcij')) + d.remove('c') + self.assertEqual(d, self.type2test('abdefghij')) + self.assertRaises(ValueError, d.remove, 'c') + self.assertEqual(d, self.type2test('abdefghij')) + + # Handle comparison errors + d = self.type2test(['a', 'b', BadCmp2(), 'c']) + e = self.type2test(d) + self.assertRaises(BadExc, d.remove, 'c') + for x, y in zip(d, e): + # verify that original order and values are retained. + self.assertIs(x, y) + + def test_index(self): + super().test_index() + a = self.type2test([-2, -1, 0, 0, 1, 2]) + a.remove(0) + self.assertRaises(ValueError, a.index, 2, 0, 4) + self.assertEqual(a, self.type2test([-2, -1, 0, 1, 2])) + + # Test modifying the list during index's iteration + class EvilCmp: + def __init__(self, victim): + self.victim = victim + def __eq__(self, other): + del self.victim[:] + return False + a = self.type2test() + a[:] = [EvilCmp(a) for _ in range(100)] + # This used to seg fault before patch #1005778 + self.assertRaises(ValueError, a.index, None) + + def test_reverse(self): + u = self.type2test([-2, -1, 0, 1, 2]) + u2 = u[:] + u.reverse() + self.assertEqual(u, [2, 1, 0, -1, -2]) + u.reverse() + self.assertEqual(u, u2) + + self.assertRaises(TypeError, u.reverse, 42) + + def test_clear(self): + u = self.type2test([2, 3, 4]) + u.clear() + self.assertEqual(u, []) + + u = self.type2test([]) + u.clear() + self.assertEqual(u, []) + + u = self.type2test([]) + u.append(1) + u.clear() + u.append(2) + self.assertEqual(u, [2]) + + self.assertRaises(TypeError, u.clear, None) + + def test_copy(self): + u = self.type2test([1, 2, 3]) + v = u.copy() + self.assertEqual(v, [1, 2, 3]) + + u = self.type2test([]) + v = u.copy() + self.assertEqual(v, []) + + # test that it's indeed a copy and not a reference + u = self.type2test(['a', 'b']) + v = u.copy() + v.append('i') + self.assertEqual(u, ['a', 'b']) + self.assertEqual(v, u + ['i']) + + # test that it's a shallow, not a deep copy + u = self.type2test([1, 2, [3, 4], 5]) + v = u.copy() + self.assertEqual(u, v) + self.assertIs(v[3], u[3]) + + self.assertRaises(TypeError, u.copy, None) + + def test_sort(self): + u = self.type2test([1, 0]) + u.sort() + self.assertEqual(u, [0, 1]) + + u = self.type2test([2,1,0,-1,-2]) + u.sort() + self.assertEqual(u, self.type2test([-2,-1,0,1,2])) + + self.assertRaises(TypeError, u.sort, 42, 42) + + def revcmp(a, b): + if a == b: + return 0 + elif a < b: + return 1 + else: # a > b + return -1 + u.sort(key=cmp_to_key(revcmp)) + self.assertEqual(u, self.type2test([2,1,0,-1,-2])) + + # The following dumps core in unpatched Python 1.5: + def myComparison(x,y): + xmod, ymod = x%3, y%7 + if xmod == ymod: + return 0 + elif xmod < ymod: + return -1 + else: # xmod > ymod + return 1 + z = self.type2test(range(12)) + z.sort(key=cmp_to_key(myComparison)) + + self.assertRaises(TypeError, z.sort, 2) + + def selfmodifyingComparison(x,y): + z.append(1) + if x == y: + return 0 + elif x < y: + return -1 + else: # x > y + return 1 + self.assertRaises(ValueError, z.sort, + key=cmp_to_key(selfmodifyingComparison)) + + self.assertRaises(TypeError, z.sort, 42, 42, 42, 42) + + def test_slice(self): + u = self.type2test("spam") + u[:2] = "h" + self.assertEqual(u, list("ham")) + + def test_iadd(self): + super().test_iadd() + u = self.type2test([0, 1]) + u2 = u + u += [2, 3] + self.assertIs(u, u2) + + u = self.type2test("spam") + u += "eggs" + self.assertEqual(u, self.type2test("spameggs")) + + self.assertRaises(TypeError, u.__iadd__, None) + + def test_imul(self): + super().test_imul() + s = self.type2test([]) + oldid = id(s) + s *= 10 + self.assertEqual(id(s), oldid) + + def test_extendedslicing(self): + # subscript + a = self.type2test([0,1,2,3,4]) + + # deletion + del a[::2] + self.assertEqual(a, self.type2test([1,3])) + a = self.type2test(range(5)) + del a[1::2] + self.assertEqual(a, self.type2test([0,2,4])) + a = self.type2test(range(5)) + del a[1::-2] + self.assertEqual(a, self.type2test([0,2,3,4])) + a = self.type2test(range(10)) + del a[::1000] + self.assertEqual(a, self.type2test([1, 2, 3, 4, 5, 6, 7, 8, 9])) + # assignment + a = self.type2test(range(10)) + a[::2] = [-1]*5 + self.assertEqual(a, self.type2test([-1, 1, -1, 3, -1, 5, -1, 7, -1, 9])) + a = self.type2test(range(10)) + a[::-4] = [10]*3 + self.assertEqual(a, self.type2test([0, 10, 2, 3, 4, 10, 6, 7, 8 ,10])) + a = self.type2test(range(4)) + a[::-1] = a + self.assertEqual(a, self.type2test([3, 2, 1, 0])) + a = self.type2test(range(10)) + b = a[:] + c = a[:] + a[2:3] = self.type2test(["two", "elements"]) + b[slice(2,3)] = self.type2test(["two", "elements"]) + c[2:3:] = self.type2test(["two", "elements"]) + self.assertEqual(a, b) + self.assertEqual(a, c) + a = self.type2test(range(10)) + a[::2] = tuple(range(5)) + self.assertEqual(a, self.type2test([0, 1, 1, 3, 2, 5, 3, 7, 4, 9])) + # test issue7788 + a = self.type2test(range(10)) + del a[9::1<<333] + + def test_constructor_exception_handling(self): + # Bug #1242657 + class F(object): + def __iter__(self): + raise KeyboardInterrupt + self.assertRaises(KeyboardInterrupt, self.type2test, F()) + + def test_exhausted_iterator(self): + a = self.type2test([1, 2, 3]) + exhit = iter(a) + empit = iter(a) + for x in exhit: # exhaust the iterator + next(empit) # not exhausted + a.append(9) + self.assertEqual(list(exhit), []) + self.assertEqual(list(empit), [9]) + self.assertEqual(a, self.type2test([1, 2, 3, 9])) + + # gh-115733: Crash when iterating over exhausted iterator + exhit = iter(self.type2test([1, 2, 3])) + for _ in exhit: + next(exhit, 1) diff --git a/crates/weavepy-vm/src/stdlib/python/test_pickletester.py b/crates/weavepy-vm/src/stdlib/python/test_pickletester.py new file mode 100644 index 0000000..600590e --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/test_pickletester.py @@ -0,0 +1,31 @@ +"""Minimal `test.pickletester` shim for WeavePy's bundled conformance run. + +CPython's real `Lib/test/pickletester.py` is ~4900 lines and exercises the +full pickle protocol matrix. The only symbol the bundled `test_copyreg` +imports from it is `ExtensionSaver`, the copyreg extension-registry +save/restore helper, so we carry that verbatim rather than the whole file. +""" + +import copyreg + + +class ExtensionSaver: + # Remember current registration for code (if any), and remove it (if + # there is one). + def __init__(self, code): + self.code = code + if code in copyreg._inverted_registry: + self.pair = copyreg._inverted_registry[code] + copyreg.remove_extension(self.pair[0], self.pair[1], code) + else: + self.pair = None + + # Restore previous registration for code. + def restore(self): + code = self.code + curpair = copyreg._inverted_registry.get(code) + if curpair is not None: + copyreg.remove_extension(curpair[0], curpair[1], code) + pair = self.pair + if pair is not None: + copyreg.add_extension(pair[0], pair[1], code) diff --git a/crates/weavepy-vm/src/stdlib/python/test_seq_tests.py b/crates/weavepy-vm/src/stdlib/python/test_seq_tests.py new file mode 100644 index 0000000..1042995 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/test_seq_tests.py @@ -0,0 +1,445 @@ +""" +Tests common to tuple, list and UserList.UserList +""" + +import unittest +import sys +import pickle +from test import support +from test.support import ALWAYS_EQ, NEVER_EQ + +# Various iterables +# This is used for checking the constructor (here and in test_deque.py) +def iterfunc(seqn): + 'Regular generator' + for i in seqn: + yield i + +class Sequence: + 'Sequence using __getitem__' + def __init__(self, seqn): + self.seqn = seqn + def __getitem__(self, i): + return self.seqn[i] + +class IterFunc: + 'Sequence using iterator protocol' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + return self + def __next__(self): + if self.i >= len(self.seqn): raise StopIteration + v = self.seqn[self.i] + self.i += 1 + return v + +class IterGen: + 'Sequence using iterator protocol defined with a generator' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + for val in self.seqn: + yield val + +class IterNextOnly: + 'Missing __getitem__ and __iter__' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __next__(self): + if self.i >= len(self.seqn): raise StopIteration + v = self.seqn[self.i] + self.i += 1 + return v + +class IterNoNext: + 'Iterator missing __next__()' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + return self + +class IterGenExc: + 'Test propagation of exceptions' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + return self + def __next__(self): + 3 // 0 + +class IterFuncStop: + 'Test immediate stop' + def __init__(self, seqn): + pass + def __iter__(self): + return self + def __next__(self): + raise StopIteration + +from itertools import chain +def itermulti(seqn): + 'Test multiple tiers of iterators' + return chain(map(lambda x:x, iterfunc(IterGen(Sequence(seqn))))) + +class LyingTuple(tuple): + def __iter__(self): + yield 1 + +class LyingList(list): + def __iter__(self): + yield 1 + +class CommonTest(unittest.TestCase): + # The type to be tested + type2test = None + + def test_constructors(self): + l0 = [] + l1 = [0] + l2 = [0, 1] + + u = self.type2test() + u0 = self.type2test(l0) + u1 = self.type2test(l1) + u2 = self.type2test(l2) + + uu = self.type2test(u) + uu0 = self.type2test(u0) + uu1 = self.type2test(u1) + uu2 = self.type2test(u2) + + v = self.type2test(tuple(u)) + class OtherSeq: + def __init__(self, initseq): + self.__data = initseq + def __len__(self): + return len(self.__data) + def __getitem__(self, i): + return self.__data[i] + s = OtherSeq(u0) + v0 = self.type2test(s) + self.assertEqual(len(v0), len(s)) + + s = "this is also a sequence" + vv = self.type2test(s) + self.assertEqual(len(vv), len(s)) + + # Create from various iteratables + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + for g in (Sequence, IterFunc, IterGen, + itermulti, iterfunc): + self.assertEqual(self.type2test(g(s)), self.type2test(s)) + self.assertEqual(self.type2test(IterFuncStop(s)), self.type2test()) + self.assertEqual(self.type2test(c for c in "123"), self.type2test("123")) + self.assertRaises(TypeError, self.type2test, IterNextOnly(s)) + self.assertRaises(TypeError, self.type2test, IterNoNext(s)) + self.assertRaises(ZeroDivisionError, self.type2test, IterGenExc(s)) + + # Issue #23757 + self.assertEqual(self.type2test(LyingTuple((2,))), self.type2test((1,))) + self.assertEqual(self.type2test(LyingList([2])), self.type2test([1])) + + with self.assertRaises(TypeError): + self.type2test(unsupported_arg=[]) + + def test_truth(self): + self.assertFalse(self.type2test()) + self.assertTrue(self.type2test([42])) + + def test_getitem(self): + u = self.type2test([0, 1, 2, 3, 4]) + for i in range(len(u)): + self.assertEqual(u[i], i) + self.assertEqual(u[int(i)], i) + for i in range(-len(u), -1): + self.assertEqual(u[i], len(u)+i) + self.assertEqual(u[int(i)], len(u)+i) + self.assertRaises(IndexError, u.__getitem__, -len(u)-1) + self.assertRaises(IndexError, u.__getitem__, len(u)) + self.assertRaises(ValueError, u.__getitem__, slice(0,10,0)) + + u = self.type2test() + self.assertRaises(IndexError, u.__getitem__, 0) + self.assertRaises(IndexError, u.__getitem__, -1) + + self.assertRaises(TypeError, u.__getitem__) + + a = self.type2test([10, 11]) + self.assertEqual(a[0], 10) + self.assertEqual(a[1], 11) + self.assertEqual(a[-2], 10) + self.assertEqual(a[-1], 11) + self.assertRaises(IndexError, a.__getitem__, -3) + self.assertRaises(IndexError, a.__getitem__, 3) + + def test_getslice(self): + l = [0, 1, 2, 3, 4] + u = self.type2test(l) + + self.assertEqual(u[0:0], self.type2test()) + self.assertEqual(u[1:2], self.type2test([1])) + self.assertEqual(u[-2:-1], self.type2test([3])) + self.assertEqual(u[-1000:1000], u) + self.assertEqual(u[1000:-1000], self.type2test([])) + self.assertEqual(u[:], u) + self.assertEqual(u[1:None], self.type2test([1, 2, 3, 4])) + self.assertEqual(u[None:3], self.type2test([0, 1, 2])) + + # Extended slices + self.assertEqual(u[::], u) + self.assertEqual(u[::2], self.type2test([0, 2, 4])) + self.assertEqual(u[1::2], self.type2test([1, 3])) + self.assertEqual(u[::-1], self.type2test([4, 3, 2, 1, 0])) + self.assertEqual(u[::-2], self.type2test([4, 2, 0])) + self.assertEqual(u[3::-2], self.type2test([3, 1])) + self.assertEqual(u[3:3:-2], self.type2test([])) + self.assertEqual(u[3:2:-2], self.type2test([3])) + self.assertEqual(u[3:1:-2], self.type2test([3])) + self.assertEqual(u[3:0:-2], self.type2test([3, 1])) + self.assertEqual(u[::-100], self.type2test([4])) + self.assertEqual(u[100:-100:], self.type2test([])) + self.assertEqual(u[-100:100:], u) + self.assertEqual(u[100:-100:-1], u[::-1]) + self.assertEqual(u[-100:100:-1], self.type2test([])) + self.assertEqual(u[-100:100:2], self.type2test([0, 2, 4])) + + # Test extreme cases with long ints + a = self.type2test([0,1,2,3,4]) + self.assertEqual(a[ -pow(2,128): 3 ], self.type2test([0,1,2])) + self.assertEqual(a[ 3: pow(2,145) ], self.type2test([3,4])) + self.assertEqual(a[3::sys.maxsize], self.type2test([3])) + + def test_contains(self): + u = self.type2test([0, 1, 2]) + for i in u: + self.assertIn(i, u) + for i in min(u)-1, max(u)+1: + self.assertNotIn(i, u) + + self.assertRaises(TypeError, u.__contains__) + + def test_contains_fake(self): + # Sequences must use rich comparison against each item + # (unless "is" is true, or an earlier item answered) + # So ALWAYS_EQ must be found in all non-empty sequences. + self.assertNotIn(ALWAYS_EQ, self.type2test([])) + self.assertIn(ALWAYS_EQ, self.type2test([1])) + self.assertIn(1, self.type2test([ALWAYS_EQ])) + self.assertNotIn(NEVER_EQ, self.type2test([])) + self.assertNotIn(ALWAYS_EQ, self.type2test([NEVER_EQ])) + self.assertIn(NEVER_EQ, self.type2test([ALWAYS_EQ])) + + def test_contains_order(self): + # Sequences must test in-order. If a rich comparison has side + # effects, these will be visible to tests against later members. + # In this test, the "side effect" is a short-circuiting raise. + class DoNotTestEq(Exception): + pass + class StopCompares: + def __eq__(self, other): + raise DoNotTestEq + + checkfirst = self.type2test([1, StopCompares()]) + self.assertIn(1, checkfirst) + checklast = self.type2test([StopCompares(), 1]) + self.assertRaises(DoNotTestEq, checklast.__contains__, 1) + + def test_len(self): + self.assertEqual(len(self.type2test()), 0) + self.assertEqual(len(self.type2test([])), 0) + self.assertEqual(len(self.type2test([0])), 1) + self.assertEqual(len(self.type2test([0, 1, 2])), 3) + + def test_minmax(self): + u = self.type2test([0, 1, 2]) + self.assertEqual(min(u), 0) + self.assertEqual(max(u), 2) + + def test_add(self): + u1 = self.type2test([0]) + u2 = self.type2test([0, 1]) + self.assertEqual(u1, u1 + self.type2test()) + self.assertEqual(u1, self.type2test() + u1) + self.assertEqual(u1 + self.type2test([1]), u2) + self.assertEqual(self.type2test([-1]) + u1, self.type2test([-1, 0])) + + def test_mul(self): + u2 = self.type2test([0, 1]) + self.assertEqual(self.type2test(), u2*0) + self.assertEqual(self.type2test(), 0*u2) + self.assertEqual(u2, u2*1) + self.assertEqual(u2, 1*u2) + self.assertEqual(u2+u2, u2*2) + self.assertEqual(u2+u2, 2*u2) + self.assertEqual(u2+u2+u2, u2*3) + self.assertEqual(u2+u2+u2, 3*u2) + + class subclass(self.type2test): + pass + u3 = subclass([0, 1]) + r = u3*1 + self.assertEqual(r, u3) + self.assertIsNot(r, u3) + + def test_iadd(self): + u = self.type2test([0, 1]) + u += self.type2test() + self.assertEqual(u, self.type2test([0, 1])) + u += self.type2test([2, 3]) + self.assertEqual(u, self.type2test([0, 1, 2, 3])) + u += self.type2test([4, 5]) + self.assertEqual(u, self.type2test([0, 1, 2, 3, 4, 5])) + + u = self.type2test("spam") + u += self.type2test("eggs") + self.assertEqual(u, self.type2test("spameggs")) + + def test_imul(self): + u = self.type2test([0, 1]) + u *= 3 + self.assertEqual(u, self.type2test([0, 1, 0, 1, 0, 1])) + u *= 0 + self.assertEqual(u, self.type2test([])) + + def test_getitemoverwriteiter(self): + # Verify that __getitem__ overrides are not recognized by __iter__ + class T(self.type2test): + def __getitem__(self, key): + return str(key) + '!!!' + self.assertEqual(next(iter(T((1,2)))), 1) + + def test_repeat(self): + for m in range(4): + s = tuple(range(m)) + for n in range(-3, 5): + self.assertEqual(self.type2test(s*n), self.type2test(s)*n) + self.assertEqual(self.type2test(s)*(-4), self.type2test([])) + self.assertEqual(id(s), id(s*1)) + + def test_bigrepeat(self): + if sys.maxsize <= 2147483647: + x = self.type2test([0]) + x *= 2**16 + self.assertRaises(MemoryError, x.__mul__, 2**16) + if hasattr(x, '__imul__'): + self.assertRaises(MemoryError, x.__imul__, 2**16) + + def test_subscript(self): + a = self.type2test([10, 11]) + self.assertEqual(a.__getitem__(0), 10) + self.assertEqual(a.__getitem__(1), 11) + self.assertEqual(a.__getitem__(-2), 10) + self.assertEqual(a.__getitem__(-1), 11) + self.assertRaises(IndexError, a.__getitem__, -3) + self.assertRaises(IndexError, a.__getitem__, 3) + self.assertEqual(a.__getitem__(slice(0,1)), self.type2test([10])) + self.assertEqual(a.__getitem__(slice(1,2)), self.type2test([11])) + self.assertEqual(a.__getitem__(slice(0,2)), self.type2test([10, 11])) + self.assertEqual(a.__getitem__(slice(0,3)), self.type2test([10, 11])) + self.assertEqual(a.__getitem__(slice(3,5)), self.type2test([])) + self.assertRaises(ValueError, a.__getitem__, slice(0, 10, 0)) + self.assertRaises(TypeError, a.__getitem__, 'x') + + def _assert_cmp(self, a, b, r): + self.assertIs(a == b, r == 0) + self.assertIs(a != b, r != 0) + self.assertIs(a > b, r > 0) + self.assertIs(a <= b, r <= 0) + self.assertIs(a < b, r < 0) + self.assertIs(a >= b, r >= 0) + + def test_cmp(self): + a = self.type2test([0, 1]) + self._assert_cmp(a, a, 0) + self._assert_cmp(a, self.type2test([0, 1]), 0) + self._assert_cmp(a, self.type2test([0]), 1) + self._assert_cmp(a, self.type2test([0, 2]), -1) + + def test_count(self): + a = self.type2test([0, 1, 2])*3 + self.assertEqual(a.count(0), 3) + self.assertEqual(a.count(1), 3) + self.assertEqual(a.count(3), 0) + + self.assertEqual(a.count(ALWAYS_EQ), 9) + self.assertEqual(self.type2test([ALWAYS_EQ, ALWAYS_EQ]).count(1), 2) + self.assertEqual(self.type2test([ALWAYS_EQ, ALWAYS_EQ]).count(NEVER_EQ), 2) + self.assertEqual(self.type2test([NEVER_EQ, NEVER_EQ]).count(ALWAYS_EQ), 0) + + self.assertRaises(TypeError, a.count) + + class BadExc(Exception): + pass + + class BadCmp: + def __eq__(self, other): + if other == 2: + raise BadExc() + return False + + self.assertRaises(BadExc, a.count, BadCmp()) + + def test_index(self): + u = self.type2test([0, 1]) + self.assertEqual(u.index(0), 0) + self.assertEqual(u.index(1), 1) + self.assertRaises(ValueError, u.index, 2) + + u = self.type2test([-2, -1, 0, 0, 1, 2]) + self.assertEqual(u.count(0), 2) + self.assertEqual(u.index(0), 2) + self.assertEqual(u.index(0, 2), 2) + self.assertEqual(u.index(-2, -10), 0) + self.assertEqual(u.index(0, 3), 3) + self.assertEqual(u.index(0, 3, 4), 3) + self.assertRaises(ValueError, u.index, 2, 0, -10) + + self.assertEqual(u.index(ALWAYS_EQ), 0) + self.assertEqual(self.type2test([ALWAYS_EQ, ALWAYS_EQ]).index(1), 0) + self.assertEqual(self.type2test([ALWAYS_EQ, ALWAYS_EQ]).index(NEVER_EQ), 0) + self.assertRaises(ValueError, self.type2test([NEVER_EQ, NEVER_EQ]).index, ALWAYS_EQ) + + self.assertRaises(TypeError, u.index) + + class BadExc(Exception): + pass + + class BadCmp: + def __eq__(self, other): + if other == 2: + raise BadExc() + return False + + a = self.type2test([0, 1, 2, 3]) + self.assertRaises(BadExc, a.index, BadCmp()) + + a = self.type2test([-2, -1, 0, 0, 1, 2]) + self.assertEqual(a.index(0), 2) + self.assertEqual(a.index(0, 2), 2) + self.assertEqual(a.index(0, -4), 2) + self.assertEqual(a.index(-2, -10), 0) + self.assertEqual(a.index(0, 3), 3) + self.assertEqual(a.index(0, -3), 3) + self.assertEqual(a.index(0, 3, 4), 3) + self.assertEqual(a.index(0, -3, -2), 3) + self.assertEqual(a.index(0, -4*sys.maxsize, 4*sys.maxsize), 2) + self.assertRaises(ValueError, a.index, 0, 4*sys.maxsize,-4*sys.maxsize) + self.assertRaises(ValueError, a.index, 2, 0, -10) + + def test_pickle(self): + lst = self.type2test([4, 5, 6, 7]) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + lst2 = pickle.loads(pickle.dumps(lst, proto)) + self.assertEqual(lst2, lst) + self.assertNotEqual(id(lst2), id(lst)) + + @support.suppress_immortalization() + def test_free_after_iterating(self): + support.check_free_after_iterating(self, iter, self.type2test) + support.check_free_after_iterating(self, reversed, self.type2test) diff --git a/crates/weavepy-vm/src/stdlib/python/test_string_tests.py b/crates/weavepy-vm/src/stdlib/python/test_string_tests.py new file mode 100644 index 0000000..d6467d2 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/test_string_tests.py @@ -0,0 +1,1598 @@ +""" +Common tests shared by test_unicode, test_userstring and test_bytes. +""" + +import unittest, string, sys, struct +from test import support +from test.support import import_helper +from collections import UserList +import random + + +class Sequence: + def __init__(self, seq='wxyz'): self.seq = seq + def __len__(self): return len(self.seq) + def __getitem__(self, i): return self.seq[i] + + +class BaseTest: + # These tests are for buffers of values (bytes) and not + # specific to character interpretation, used for bytes objects + # and various string implementations + + # The type to be tested + # Change in subclasses to change the behaviour of fixtype() + type2test = None + + # Whether the "contained items" of the container are integers in + # range(0, 256) (i.e. bytes, bytearray) or strings of length 1 + # (str) + contains_bytes = False + + # All tests pass their arguments to the testing methods + # as str objects. fixtype() can be used to propagate + # these arguments to the appropriate type + def fixtype(self, obj): + if isinstance(obj, str): + return self.__class__.type2test(obj) + elif isinstance(obj, list): + return [self.fixtype(x) for x in obj] + elif isinstance(obj, tuple): + return tuple([self.fixtype(x) for x in obj]) + elif isinstance(obj, dict): + return dict([ + (self.fixtype(key), self.fixtype(value)) + for (key, value) in obj.items() + ]) + else: + return obj + + def test_fixtype(self): + self.assertIs(type(self.fixtype("123")), self.type2test) + + # check that obj.method(*args) returns result + def checkequal(self, result, obj, methodname, *args, **kwargs): + result = self.fixtype(result) + obj = self.fixtype(obj) + args = self.fixtype(args) + kwargs = {k: self.fixtype(v) for k,v in kwargs.items()} + realresult = getattr(obj, methodname)(*args, **kwargs) + self.assertEqual( + result, + realresult + ) + # if the original is returned make sure that + # this doesn't happen with subclasses + if obj is realresult: + try: + class subtype(self.__class__.type2test): + pass + except TypeError: + pass # Skip this if we can't subclass + else: + obj = subtype(obj) + realresult = getattr(obj, methodname)(*args) + self.assertIsNot(obj, realresult) + + # check that obj.method(*args) raises exc + def checkraises(self, exc, obj, methodname, *args, expected_msg=None): + obj = self.fixtype(obj) + args = self.fixtype(args) + with self.assertRaises(exc) as cm: + getattr(obj, methodname)(*args) + self.assertNotEqual(str(cm.exception), '') + if expected_msg is not None: + self.assertEqual(str(cm.exception), expected_msg) + + # call obj.method(*args) without any checks + def checkcall(self, obj, methodname, *args): + obj = self.fixtype(obj) + args = self.fixtype(args) + getattr(obj, methodname)(*args) + + def _get_teststrings(self, charset, digits): + base = len(charset) + teststrings = set() + for i in range(base ** digits): + entry = [] + for j in range(digits): + i, m = divmod(i, base) + entry.append(charset[m]) + teststrings.add(''.join(entry)) + teststrings = [self.fixtype(ts) for ts in teststrings] + return teststrings + + def test_add(self): + s = self.fixtype('ab') + self.assertEqual(s + self.fixtype(''), s) + self.assertEqual(self.fixtype('') + s, s) + self.assertEqual(s + self.fixtype('cd'), self.fixtype('abcd')) + + def test_mul(self): + s = self.fixtype('ab') + self.assertEqual(s*0, self.fixtype('')) + self.assertEqual(0*s, self.fixtype('')) + self.assertEqual(s*1, s) + self.assertEqual(1*s, s) + self.assertEqual(s*2, self.fixtype('abab')) + self.assertEqual(2*s, self.fixtype('abab')) + + class subclass(self.type2test): + pass + s = subclass(self.fixtype('ab')) + r = s*1 + self.assertEqual(r, s) + self.assertIsNot(r, s) + + def _assert_cmp(self, a, b, r): + self.assertIs(a == b, r == 0) + self.assertIs(a != b, r != 0) + self.assertIs(a > b, r > 0) + self.assertIs(a <= b, r <= 0) + self.assertIs(a < b, r < 0) + self.assertIs(a >= b, r >= 0) + + def test_cmp(self): + a = self.fixtype('ab') + self._assert_cmp(a, a, 0) + self._assert_cmp(a, self.fixtype('ab'), 0) + self._assert_cmp(a, self.fixtype('a'), 1) + self._assert_cmp(a, self.fixtype('ac'), -1) + + def test_count(self): + self.checkequal(3, 'aaa', 'count', 'a') + self.checkequal(0, 'aaa', 'count', 'b') + self.checkequal(3, 'aaa', 'count', 'a') + self.checkequal(0, 'aaa', 'count', 'b') + self.checkequal(3, 'aaa', 'count', 'a') + self.checkequal(0, 'aaa', 'count', 'b') + self.checkequal(0, 'aaa', 'count', 'b') + self.checkequal(2, 'aaa', 'count', 'a', 1) + self.checkequal(0, 'aaa', 'count', 'a', 10) + self.checkequal(1, 'aaa', 'count', 'a', -1) + self.checkequal(3, 'aaa', 'count', 'a', -10) + self.checkequal(1, 'aaa', 'count', 'a', 0, 1) + self.checkequal(3, 'aaa', 'count', 'a', 0, 10) + self.checkequal(2, 'aaa', 'count', 'a', 0, -1) + self.checkequal(0, 'aaa', 'count', 'a', 0, -10) + self.checkequal(3, 'aaa', 'count', '', 1) + self.checkequal(1, 'aaa', 'count', '', 3) + self.checkequal(0, 'aaa', 'count', '', 10) + self.checkequal(2, 'aaa', 'count', '', -1) + self.checkequal(4, 'aaa', 'count', '', -10) + + self.checkequal(1, '', 'count', '') + self.checkequal(0, '', 'count', '', 1, 1) + self.checkequal(0, '', 'count', '', sys.maxsize, 0) + + self.checkequal(0, '', 'count', 'xx') + self.checkequal(0, '', 'count', 'xx', 1, 1) + self.checkequal(0, '', 'count', 'xx', sys.maxsize, 0) + + self.checkraises(TypeError, 'hello', 'count') + + if self.contains_bytes: + self.checkequal(0, 'hello', 'count', 42) + else: + self.checkraises(TypeError, 'hello', 'count', 42) + + # For a variety of combinations, + # verify that str.count() matches an equivalent function + # replacing all occurrences and then differencing the string lengths + teststrings = self._get_teststrings(['', 'a', 'b'], 7) + for i in teststrings: + n = len(i) + for j in teststrings: + r1 = i.count(j) + if j: + r2, rem = divmod(n - len(i.replace(j, self.fixtype(''))), + len(j)) + else: + r2, rem = len(i)+1, 0 + if rem or r1 != r2: + self.assertEqual(rem, 0, '%s != 0 for %s' % (rem, i)) + self.assertEqual(r1, r2, '%s != %s for %s' % (r1, r2, i)) + + def test_count_keyword(self): + self.assertEqual('aa'.replace('a', 'b', 0), 'aa'.replace('a', 'b', count=0)) + self.assertEqual('aa'.replace('a', 'b', 1), 'aa'.replace('a', 'b', count=1)) + self.assertEqual('aa'.replace('a', 'b', 2), 'aa'.replace('a', 'b', count=2)) + self.assertEqual('aa'.replace('a', 'b', 3), 'aa'.replace('a', 'b', count=3)) + + def test_find(self): + self.checkequal(0, 'abcdefghiabc', 'find', 'abc') + self.checkequal(9, 'abcdefghiabc', 'find', 'abc', 1) + self.checkequal(-1, 'abcdefghiabc', 'find', 'def', 4) + + self.checkequal(0, 'abc', 'find', '', 0) + self.checkequal(3, 'abc', 'find', '', 3) + self.checkequal(-1, 'abc', 'find', '', 4) + + # to check the ability to pass None as defaults + self.checkequal( 2, 'rrarrrrrrrrra', 'find', 'a') + self.checkequal(12, 'rrarrrrrrrrra', 'find', 'a', 4) + self.checkequal(-1, 'rrarrrrrrrrra', 'find', 'a', 4, 6) + self.checkequal(12, 'rrarrrrrrrrra', 'find', 'a', 4, None) + self.checkequal( 2, 'rrarrrrrrrrra', 'find', 'a', None, 6) + + self.checkraises(TypeError, 'hello', 'find') + + if self.contains_bytes: + self.checkequal(-1, 'hello', 'find', 42) + else: + self.checkraises(TypeError, 'hello', 'find', 42) + + self.checkequal(0, '', 'find', '') + self.checkequal(-1, '', 'find', '', 1, 1) + self.checkequal(-1, '', 'find', '', sys.maxsize, 0) + + self.checkequal(-1, '', 'find', 'xx') + self.checkequal(-1, '', 'find', 'xx', 1, 1) + self.checkequal(-1, '', 'find', 'xx', sys.maxsize, 0) + + # issue 7458 + self.checkequal(-1, 'ab', 'find', 'xxx', sys.maxsize + 1, 0) + + # For a variety of combinations, + # verify that str.find() matches __contains__ + # and that the found substring is really at that location + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) + for i in teststrings: + for j in teststrings: + loc = i.find(j) + r1 = (loc != -1) + r2 = j in i + self.assertEqual(r1, r2) + if loc != -1: + self.assertEqual(i[loc:loc+len(j)], j) + + def test_rfind(self): + self.checkequal(9, 'abcdefghiabc', 'rfind', 'abc') + self.checkequal(12, 'abcdefghiabc', 'rfind', '') + self.checkequal(0, 'abcdefghiabc', 'rfind', 'abcd') + self.checkequal(-1, 'abcdefghiabc', 'rfind', 'abcz') + + self.checkequal(3, 'abc', 'rfind', '', 0) + self.checkequal(3, 'abc', 'rfind', '', 3) + self.checkequal(-1, 'abc', 'rfind', '', 4) + + # to check the ability to pass None as defaults + self.checkequal(12, 'rrarrrrrrrrra', 'rfind', 'a') + self.checkequal(12, 'rrarrrrrrrrra', 'rfind', 'a', 4) + self.checkequal(-1, 'rrarrrrrrrrra', 'rfind', 'a', 4, 6) + self.checkequal(12, 'rrarrrrrrrrra', 'rfind', 'a', 4, None) + self.checkequal( 2, 'rrarrrrrrrrra', 'rfind', 'a', None, 6) + + self.checkraises(TypeError, 'hello', 'rfind') + + if self.contains_bytes: + self.checkequal(-1, 'hello', 'rfind', 42) + else: + self.checkraises(TypeError, 'hello', 'rfind', 42) + + # For a variety of combinations, + # verify that str.rfind() matches __contains__ + # and that the found substring is really at that location + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) + for i in teststrings: + for j in teststrings: + loc = i.rfind(j) + r1 = (loc != -1) + r2 = j in i + self.assertEqual(r1, r2) + if loc != -1: + self.assertEqual(i[loc:loc+len(j)], j) + + # issue 7458 + self.checkequal(-1, 'ab', 'rfind', 'xxx', sys.maxsize + 1, 0) + + # issue #15534 + self.checkequal(0, '<......\u043c...', "rfind", "<") + + def test_index(self): + self.checkequal(0, 'abcdefghiabc', 'index', '') + self.checkequal(3, 'abcdefghiabc', 'index', 'def') + self.checkequal(0, 'abcdefghiabc', 'index', 'abc') + self.checkequal(9, 'abcdefghiabc', 'index', 'abc', 1) + + self.checkraises(ValueError, 'abcdefghiabc', 'index', 'hib') + self.checkraises(ValueError, 'abcdefghiab', 'index', 'abc', 1) + self.checkraises(ValueError, 'abcdefghi', 'index', 'ghi', 8) + self.checkraises(ValueError, 'abcdefghi', 'index', 'ghi', -1) + + # to check the ability to pass None as defaults + self.checkequal( 2, 'rrarrrrrrrrra', 'index', 'a') + self.checkequal(12, 'rrarrrrrrrrra', 'index', 'a', 4) + self.checkraises(ValueError, 'rrarrrrrrrrra', 'index', 'a', 4, 6) + self.checkequal(12, 'rrarrrrrrrrra', 'index', 'a', 4, None) + self.checkequal( 2, 'rrarrrrrrrrra', 'index', 'a', None, 6) + + self.checkraises(TypeError, 'hello', 'index') + + if self.contains_bytes: + self.checkraises(ValueError, 'hello', 'index', 42) + else: + self.checkraises(TypeError, 'hello', 'index', 42) + + # For a variety of combinations, + # verify that str.index() matches __contains__ + # and that the found substring is really at that location + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) + for i in teststrings: + for j in teststrings: + if j in i: + loc = i.index(j) + self.assertGreaterEqual(loc, 0) + self.assertEqual(i[loc:loc+len(j)], j) + else: + self.assertRaises(ValueError, i.index, j) + + def test_rindex(self): + self.checkequal(12, 'abcdefghiabc', 'rindex', '') + self.checkequal(3, 'abcdefghiabc', 'rindex', 'def') + self.checkequal(9, 'abcdefghiabc', 'rindex', 'abc') + self.checkequal(0, 'abcdefghiabc', 'rindex', 'abc', 0, -1) + + self.checkraises(ValueError, 'abcdefghiabc', 'rindex', 'hib') + self.checkraises(ValueError, 'defghiabc', 'rindex', 'def', 1) + self.checkraises(ValueError, 'defghiabc', 'rindex', 'abc', 0, -1) + self.checkraises(ValueError, 'abcdefghi', 'rindex', 'ghi', 0, 8) + self.checkraises(ValueError, 'abcdefghi', 'rindex', 'ghi', 0, -1) + + # to check the ability to pass None as defaults + self.checkequal(12, 'rrarrrrrrrrra', 'rindex', 'a') + self.checkequal(12, 'rrarrrrrrrrra', 'rindex', 'a', 4) + self.checkraises(ValueError, 'rrarrrrrrrrra', 'rindex', 'a', 4, 6) + self.checkequal(12, 'rrarrrrrrrrra', 'rindex', 'a', 4, None) + self.checkequal( 2, 'rrarrrrrrrrra', 'rindex', 'a', None, 6) + + self.checkraises(TypeError, 'hello', 'rindex') + + if self.contains_bytes: + self.checkraises(ValueError, 'hello', 'rindex', 42) + else: + self.checkraises(TypeError, 'hello', 'rindex', 42) + + # For a variety of combinations, + # verify that str.rindex() matches __contains__ + # and that the found substring is really at that location + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) + for i in teststrings: + for j in teststrings: + if j in i: + loc = i.rindex(j) + self.assertGreaterEqual(loc, 0) + self.assertEqual(i[loc:loc+len(j)], j) + else: + self.assertRaises(ValueError, i.rindex, j) + + def test_find_periodic_pattern(self): + """Cover the special path for periodic patterns.""" + def reference_find(p, s): + for i in range(len(s)): + if s.startswith(p, i): + return i + if p == '' and s == '': + return 0 + return -1 + + def check_pattern(rr): + choices = random.choices + p0 = ''.join(choices('abcde', k=rr(10))) * rr(10, 20) + p = p0[:len(p0) - rr(10)] # pop off some characters + left = ''.join(choices('abcdef', k=rr(2000))) + right = ''.join(choices('abcdef', k=rr(2000))) + text = left + p + right + with self.subTest(p=p, text=text): + self.checkequal(reference_find(p, text), + text, 'find', p) + + rr = random.randrange + for _ in range(1000): + check_pattern(rr) + + # Test that empty string always work: + check_pattern(lambda *args: 0) + + def test_find_many_lengths(self): + haystack_repeats = [a * 10**e for e in range(6) for a in (1,2,5)] + haystacks = [(n, self.fixtype("abcab"*n + "da")) for n in haystack_repeats] + + needle_repeats = [a * 10**e for e in range(6) for a in (1, 3)] + needles = [(m, self.fixtype("abcab"*m + "da")) for m in needle_repeats] + + for n, haystack1 in haystacks: + haystack2 = haystack1[:-1] + for m, needle in needles: + answer1 = 5 * (n - m) if m <= n else -1 + self.assertEqual(haystack1.find(needle), answer1, msg=(n,m)) + self.assertEqual(haystack2.find(needle), -1, msg=(n,m)) + + def test_adaptive_find(self): + # This would be very slow for the naive algorithm, + # but str.find() should be O(n + m). + for N in 1000, 10_000, 100_000, 1_000_000: + A, B = 'a' * N, 'b' * N + haystack = A + A + B + A + A + needle = A + B + B + A + self.checkequal(-1, haystack, 'find', needle) + self.checkequal(0, haystack, 'count', needle) + self.checkequal(len(haystack), haystack + needle, 'find', needle) + self.checkequal(1, haystack + needle, 'count', needle) + + def test_find_with_memory(self): + # Test the "Skip with memory" path in the two-way algorithm. + for N in 1000, 3000, 10_000, 30_000: + needle = 'ab' * N + haystack = ('ab'*(N-1) + 'b') * 2 + self.checkequal(-1, haystack, 'find', needle) + self.checkequal(0, haystack, 'count', needle) + self.checkequal(len(haystack), haystack + needle, 'find', needle) + self.checkequal(1, haystack + needle, 'count', needle) + + def test_find_shift_table_overflow(self): + """When the table of 8-bit shifts overflows.""" + N = 2**8 + 100 + + # first check the periodic case + # here, the shift for 'b' is N + 1. + pattern1 = 'a' * N + 'b' + 'a' * N + text1 = 'babbaa' * N + pattern1 + self.checkequal(len(text1)-len(pattern1), + text1, 'find', pattern1) + + # now check the non-periodic case + # here, the shift for 'd' is 3*(N+1)+1 + pattern2 = 'ddd' + 'abc' * N + "eee" + text2 = pattern2[:-1] + "ddeede" * 2 * N + pattern2 + "de" * N + self.checkequal(len(text2) - N*len("de") - len(pattern2), + text2, 'find', pattern2) + + def test_lower(self): + self.checkequal('hello', 'HeLLo', 'lower') + self.checkequal('hello', 'hello', 'lower') + self.checkraises(TypeError, 'hello', 'lower', 42) + + def test_upper(self): + self.checkequal('HELLO', 'HeLLo', 'upper') + self.checkequal('HELLO', 'HELLO', 'upper') + self.checkraises(TypeError, 'hello', 'upper', 42) + + def test_expandtabs(self): + self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', + 'expandtabs') + self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', + 'expandtabs', 8) + self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', + 'expandtabs', 4) + self.checkequal('abc\r\nab def\ng hi', 'abc\r\nab\tdef\ng\thi', + 'expandtabs') + self.checkequal('abc\r\nab def\ng hi', 'abc\r\nab\tdef\ng\thi', + 'expandtabs', 8) + self.checkequal('abc\r\nab def\ng hi', 'abc\r\nab\tdef\ng\thi', + 'expandtabs', 4) + self.checkequal('abc\r\nab\r\ndef\ng\r\nhi', 'abc\r\nab\r\ndef\ng\r\nhi', + 'expandtabs', 4) + # check keyword args + self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', + 'expandtabs', tabsize=8) + self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', + 'expandtabs', tabsize=4) + + self.checkequal(' a\n b', ' \ta\n\tb', 'expandtabs', 1) + + self.checkraises(TypeError, 'hello', 'expandtabs', 42, 42) + # This test is only valid when sizeof(int) == sizeof(void*) == 4. + if sys.maxsize < (1 << 32) and struct.calcsize('P') == 4: + self.checkraises(OverflowError, + '\ta\n\tb', 'expandtabs', sys.maxsize) + + def test_split(self): + # by a char + self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|') + self.checkequal(['a|b|c|d'], 'a|b|c|d', 'split', '|', 0) + self.checkequal(['a', 'b|c|d'], 'a|b|c|d', 'split', '|', 1) + self.checkequal(['a', 'b', 'c|d'], 'a|b|c|d', 'split', '|', 2) + self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|', 3) + self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|', 4) + self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|', + sys.maxsize-2) + self.checkequal(['a|b|c|d'], 'a|b|c|d', 'split', '|', 0) + self.checkequal(['a', '', 'b||c||d'], 'a||b||c||d', 'split', '|', 2) + self.checkequal(['abcd'], 'abcd', 'split', '|') + self.checkequal([''], '', 'split', '|') + self.checkequal(['endcase ', ''], 'endcase |', 'split', '|') + self.checkequal(['', ' startcase'], '| startcase', 'split', '|') + self.checkequal(['', 'bothcase', ''], '|bothcase|', 'split', '|') + self.checkequal(['a', '', 'b\x00c\x00d'], 'a\x00\x00b\x00c\x00d', 'split', '\x00', 2) + + self.checkequal(['a']*20, ('a|'*20)[:-1], 'split', '|') + self.checkequal(['a']*15 +['a|a|a|a|a'], + ('a|'*20)[:-1], 'split', '|', 15) + + # by string + self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//') + self.checkequal(['a', 'b//c//d'], 'a//b//c//d', 'split', '//', 1) + self.checkequal(['a', 'b', 'c//d'], 'a//b//c//d', 'split', '//', 2) + self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//', 3) + self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//', 4) + self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//', + sys.maxsize-10) + self.checkequal(['a//b//c//d'], 'a//b//c//d', 'split', '//', 0) + self.checkequal(['a', '', 'b////c////d'], 'a////b////c////d', 'split', '//', 2) + self.checkequal(['endcase ', ''], 'endcase test', 'split', 'test') + self.checkequal(['', ' begincase'], 'test begincase', 'split', 'test') + self.checkequal(['', ' bothcase ', ''], 'test bothcase test', + 'split', 'test') + self.checkequal(['a', 'bc'], 'abbbc', 'split', 'bb') + self.checkequal(['', ''], 'aaa', 'split', 'aaa') + self.checkequal(['aaa'], 'aaa', 'split', 'aaa', 0) + self.checkequal(['ab', 'ab'], 'abbaab', 'split', 'ba') + self.checkequal(['aaaa'], 'aaaa', 'split', 'aab') + self.checkequal([''], '', 'split', 'aaa') + self.checkequal(['aa'], 'aa', 'split', 'aaa') + self.checkequal(['A', 'bobb'], 'Abbobbbobb', 'split', 'bbobb') + self.checkequal(['A', 'B', ''], 'AbbobbBbbobb', 'split', 'bbobb') + + self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'split', 'BLAH') + self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'split', 'BLAH', 19) + self.checkequal(['a']*18 + ['aBLAHa'], ('aBLAH'*20)[:-4], + 'split', 'BLAH', 18) + + # with keyword args + self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', sep='|') + self.checkequal(['a', 'b|c|d'], + 'a|b|c|d', 'split', '|', maxsplit=1) + self.checkequal(['a', 'b|c|d'], + 'a|b|c|d', 'split', sep='|', maxsplit=1) + self.checkequal(['a', 'b|c|d'], + 'a|b|c|d', 'split', maxsplit=1, sep='|') + self.checkequal(['a', 'b c d'], + 'a b c d', 'split', maxsplit=1) + + # argument type + self.checkraises(TypeError, 'hello', 'split', 42, 42, 42) + + # null case + self.checkraises(ValueError, 'hello', 'split', '') + self.checkraises(ValueError, 'hello', 'split', '', 0) + + def test_rsplit(self): + # without arg + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit') + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit') + self.checkequal([], '', 'rsplit') + + # by a char + self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|') + self.checkequal(['a|b|c', 'd'], 'a|b|c|d', 'rsplit', '|', 1) + self.checkequal(['a|b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', 2) + self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', 3) + self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', 4) + self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', + sys.maxsize-100) + self.checkequal(['a|b|c|d'], 'a|b|c|d', 'rsplit', '|', 0) + self.checkequal(['a||b||c', '', 'd'], 'a||b||c||d', 'rsplit', '|', 2) + self.checkequal(['abcd'], 'abcd', 'rsplit', '|') + self.checkequal([''], '', 'rsplit', '|') + self.checkequal(['', ' begincase'], '| begincase', 'rsplit', '|') + self.checkequal(['endcase ', ''], 'endcase |', 'rsplit', '|') + self.checkequal(['', 'bothcase', ''], '|bothcase|', 'rsplit', '|') + + self.checkequal(['a\x00\x00b', 'c', 'd'], 'a\x00\x00b\x00c\x00d', 'rsplit', '\x00', 2) + + self.checkequal(['a']*20, ('a|'*20)[:-1], 'rsplit', '|') + self.checkequal(['a|a|a|a|a']+['a']*15, + ('a|'*20)[:-1], 'rsplit', '|', 15) + + # by string + self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//') + self.checkequal(['a//b//c', 'd'], 'a//b//c//d', 'rsplit', '//', 1) + self.checkequal(['a//b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', 2) + self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', 3) + self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', 4) + self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', + sys.maxsize-5) + self.checkequal(['a//b//c//d'], 'a//b//c//d', 'rsplit', '//', 0) + self.checkequal(['a////b////c', '', 'd'], 'a////b////c////d', 'rsplit', '//', 2) + self.checkequal(['', ' begincase'], 'test begincase', 'rsplit', 'test') + self.checkequal(['endcase ', ''], 'endcase test', 'rsplit', 'test') + self.checkequal(['', ' bothcase ', ''], 'test bothcase test', + 'rsplit', 'test') + self.checkequal(['ab', 'c'], 'abbbc', 'rsplit', 'bb') + self.checkequal(['', ''], 'aaa', 'rsplit', 'aaa') + self.checkequal(['aaa'], 'aaa', 'rsplit', 'aaa', 0) + self.checkequal(['ab', 'ab'], 'abbaab', 'rsplit', 'ba') + self.checkequal(['aaaa'], 'aaaa', 'rsplit', 'aab') + self.checkequal([''], '', 'rsplit', 'aaa') + self.checkequal(['aa'], 'aa', 'rsplit', 'aaa') + self.checkequal(['bbob', 'A'], 'bbobbbobbA', 'rsplit', 'bbobb') + self.checkequal(['', 'B', 'A'], 'bbobbBbbobbA', 'rsplit', 'bbobb') + + self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'rsplit', 'BLAH') + self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'rsplit', 'BLAH', 19) + self.checkequal(['aBLAHa'] + ['a']*18, ('aBLAH'*20)[:-4], + 'rsplit', 'BLAH', 18) + + # with keyword args + self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', sep='|') + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', sep=None) + self.checkequal(['a b c', 'd'], + 'a b c d', 'rsplit', sep=None, maxsplit=1) + self.checkequal(['a|b|c', 'd'], + 'a|b|c|d', 'rsplit', '|', maxsplit=1) + self.checkequal(['a|b|c', 'd'], + 'a|b|c|d', 'rsplit', sep='|', maxsplit=1) + self.checkequal(['a|b|c', 'd'], + 'a|b|c|d', 'rsplit', maxsplit=1, sep='|') + self.checkequal(['a b c', 'd'], + 'a b c d', 'rsplit', maxsplit=1) + + # argument type + self.checkraises(TypeError, 'hello', 'rsplit', 42, 42, 42) + + # null case + self.checkraises(ValueError, 'hello', 'rsplit', '') + self.checkraises(ValueError, 'hello', 'rsplit', '', 0) + + def test_replace(self): + EQ = self.checkequal + + # Operations on the empty string + EQ("", "", "replace", "", "") + EQ("A", "", "replace", "", "A") + EQ("", "", "replace", "A", "") + EQ("", "", "replace", "A", "A") + EQ("", "", "replace", "", "", 100) + EQ("A", "", "replace", "", "A", 100) + EQ("", "", "replace", "", "", sys.maxsize) + + # interleave (from=="", 'to' gets inserted everywhere) + EQ("A", "A", "replace", "", "") + EQ("*A*", "A", "replace", "", "*") + EQ("*1A*1", "A", "replace", "", "*1") + EQ("*-#A*-#", "A", "replace", "", "*-#") + EQ("*-A*-A*-", "AA", "replace", "", "*-") + EQ("*-A*-A*-", "AA", "replace", "", "*-", -1) + EQ("*-A*-A*-", "AA", "replace", "", "*-", sys.maxsize) + EQ("*-A*-A*-", "AA", "replace", "", "*-", 4) + EQ("*-A*-A*-", "AA", "replace", "", "*-", 3) + EQ("*-A*-A", "AA", "replace", "", "*-", 2) + EQ("*-AA", "AA", "replace", "", "*-", 1) + EQ("AA", "AA", "replace", "", "*-", 0) + + # single character deletion (from=="A", to=="") + EQ("", "A", "replace", "A", "") + EQ("", "AAA", "replace", "A", "") + EQ("", "AAA", "replace", "A", "", -1) + EQ("", "AAA", "replace", "A", "", sys.maxsize) + EQ("", "AAA", "replace", "A", "", 4) + EQ("", "AAA", "replace", "A", "", 3) + EQ("A", "AAA", "replace", "A", "", 2) + EQ("AA", "AAA", "replace", "A", "", 1) + EQ("AAA", "AAA", "replace", "A", "", 0) + EQ("", "AAAAAAAAAA", "replace", "A", "") + EQ("BCD", "ABACADA", "replace", "A", "") + EQ("BCD", "ABACADA", "replace", "A", "", -1) + EQ("BCD", "ABACADA", "replace", "A", "", sys.maxsize) + EQ("BCD", "ABACADA", "replace", "A", "", 5) + EQ("BCD", "ABACADA", "replace", "A", "", 4) + EQ("BCDA", "ABACADA", "replace", "A", "", 3) + EQ("BCADA", "ABACADA", "replace", "A", "", 2) + EQ("BACADA", "ABACADA", "replace", "A", "", 1) + EQ("ABACADA", "ABACADA", "replace", "A", "", 0) + EQ("BCD", "ABCAD", "replace", "A", "") + EQ("BCD", "ABCADAA", "replace", "A", "") + EQ("BCD", "BCD", "replace", "A", "") + EQ("*************", "*************", "replace", "A", "") + EQ("^A^", "^"+"A"*1000+"^", "replace", "A", "", 999) + + # substring deletion (from=="the", to=="") + EQ("", "the", "replace", "the", "") + EQ("ater", "theater", "replace", "the", "") + EQ("", "thethe", "replace", "the", "") + EQ("", "thethethethe", "replace", "the", "") + EQ("aaaa", "theatheatheathea", "replace", "the", "") + EQ("that", "that", "replace", "the", "") + EQ("thaet", "thaet", "replace", "the", "") + EQ("here and re", "here and there", "replace", "the", "") + EQ("here and re and re", "here and there and there", + "replace", "the", "", sys.maxsize) + EQ("here and re and re", "here and there and there", + "replace", "the", "", -1) + EQ("here and re and re", "here and there and there", + "replace", "the", "", 3) + EQ("here and re and re", "here and there and there", + "replace", "the", "", 2) + EQ("here and re and there", "here and there and there", + "replace", "the", "", 1) + EQ("here and there and there", "here and there and there", + "replace", "the", "", 0) + EQ("here and re and re", "here and there and there", "replace", "the", "") + + EQ("abc", "abc", "replace", "the", "") + EQ("abcdefg", "abcdefg", "replace", "the", "") + + # substring deletion (from=="bob", to=="") + EQ("bob", "bbobob", "replace", "bob", "") + EQ("bobXbob", "bbobobXbbobob", "replace", "bob", "") + EQ("aaaaaaa", "aaaaaaabob", "replace", "bob", "") + EQ("aaaaaaa", "aaaaaaa", "replace", "bob", "") + + # single character replace in place (len(from)==len(to)==1) + EQ("Who goes there?", "Who goes there?", "replace", "o", "o") + EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O") + EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", sys.maxsize) + EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", -1) + EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", 3) + EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", 2) + EQ("WhO goes there?", "Who goes there?", "replace", "o", "O", 1) + EQ("Who goes there?", "Who goes there?", "replace", "o", "O", 0) + + EQ("Who goes there?", "Who goes there?", "replace", "a", "q") + EQ("who goes there?", "Who goes there?", "replace", "W", "w") + EQ("wwho goes there?ww", "WWho goes there?WW", "replace", "W", "w") + EQ("Who goes there!", "Who goes there?", "replace", "?", "!") + EQ("Who goes there!!", "Who goes there??", "replace", "?", "!") + + EQ("Who goes there?", "Who goes there?", "replace", ".", "!") + + # substring replace in place (len(from)==len(to) > 1) + EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**") + EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", sys.maxsize) + EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", -1) + EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", 4) + EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", 3) + EQ("Th** ** a tissue", "This is a tissue", "replace", "is", "**", 2) + EQ("Th** is a tissue", "This is a tissue", "replace", "is", "**", 1) + EQ("This is a tissue", "This is a tissue", "replace", "is", "**", 0) + EQ("cobob", "bobob", "replace", "bob", "cob") + EQ("cobobXcobocob", "bobobXbobobob", "replace", "bob", "cob") + EQ("bobob", "bobob", "replace", "bot", "bot") + + # replace single character (len(from)==1, len(to)>1) + EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK") + EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK", -1) + EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK", sys.maxsize) + EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK", 2) + EQ("ReyKKjavik", "Reykjavik", "replace", "k", "KK", 1) + EQ("Reykjavik", "Reykjavik", "replace", "k", "KK", 0) + EQ("A----B----C----", "A.B.C.", "replace", ".", "----") + # issue #15534 + EQ('...\u043c......<', '...\u043c......<', "replace", "<", "<") + + EQ("Reykjavik", "Reykjavik", "replace", "q", "KK") + + # replace substring (len(from)>1, len(to)!=len(from)) + EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam", + "replace", "spam", "ham") + EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam", + "replace", "spam", "ham", sys.maxsize) + EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam", + "replace", "spam", "ham", -1) + EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam", + "replace", "spam", "ham", 4) + EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam", + "replace", "spam", "ham", 3) + EQ("ham, ham, eggs and spam", "spam, spam, eggs and spam", + "replace", "spam", "ham", 2) + EQ("ham, spam, eggs and spam", "spam, spam, eggs and spam", + "replace", "spam", "ham", 1) + EQ("spam, spam, eggs and spam", "spam, spam, eggs and spam", + "replace", "spam", "ham", 0) + + EQ("bobob", "bobobob", "replace", "bobob", "bob") + EQ("bobobXbobob", "bobobobXbobobob", "replace", "bobob", "bob") + EQ("BOBOBOB", "BOBOBOB", "replace", "bob", "bobby") + + self.checkequal('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1) + self.checkequal('onetwothree', 'one!two!three!', 'replace', '!', '') + self.checkequal('one@two@three!', 'one!two!three!', 'replace', '!', '@', 2) + self.checkequal('one@two@three@', 'one!two!three!', 'replace', '!', '@', 3) + self.checkequal('one@two@three@', 'one!two!three!', 'replace', '!', '@', 4) + self.checkequal('one!two!three!', 'one!two!three!', 'replace', '!', '@', 0) + self.checkequal('one@two@three@', 'one!two!three!', 'replace', '!', '@') + self.checkequal('one!two!three!', 'one!two!three!', 'replace', 'x', '@') + self.checkequal('one!two!three!', 'one!two!three!', 'replace', 'x', '@', 2) + self.checkequal('-a-b-c-', 'abc', 'replace', '', '-') + self.checkequal('-a-b-c', 'abc', 'replace', '', '-', 3) + self.checkequal('abc', 'abc', 'replace', '', '-', 0) + self.checkequal('', '', 'replace', '', '') + self.checkequal('abc', 'abc', 'replace', 'ab', '--', 0) + self.checkequal('abc', 'abc', 'replace', 'xy', '--') + # Next three for SF bug 422088: [OSF1 alpha] string.replace(); died with + # MemoryError due to empty result (platform malloc issue when requesting + # 0 bytes). + self.checkequal('', '123', 'replace', '123', '') + self.checkequal('', '123123', 'replace', '123', '') + self.checkequal('x', '123x123', 'replace', '123', '') + + self.checkraises(TypeError, 'hello', 'replace') + self.checkraises(TypeError, 'hello', 'replace', 42) + self.checkraises(TypeError, 'hello', 'replace', 42, 'h') + self.checkraises(TypeError, 'hello', 'replace', 'h', 42) + + def test_replacement_on_buffer_boundary(self): + # gh-127971: Check we don't read past the end of the buffer when a + # potential match misses on the last character. + any_3_nonblank_codepoints = '!!!' + seven_codepoints = any_3_nonblank_codepoints + ' ' + any_3_nonblank_codepoints + a = (' ' * 243) + seven_codepoints + (' ' * 7) + b = ' ' * 6 + chr(256) + a.replace(seven_codepoints, b) + + def test_replace_uses_two_way_maxcount(self): + # Test that maxcount works in _two_way_count in fastsearch.h + A, B = "A"*1000, "B"*1000 + AABAA = A + A + B + A + A + ABBA = A + B + B + A + self.checkequal(AABAA + ABBA, + AABAA + ABBA, 'replace', ABBA, "ccc", 0) + self.checkequal(AABAA + "ccc", + AABAA + ABBA, 'replace', ABBA, "ccc", 1) + self.checkequal(AABAA + "ccc", + AABAA + ABBA, 'replace', ABBA, "ccc", 2) + + @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4, + 'only applies to 32-bit platforms') + def test_replace_overflow(self): + # Check for overflow checking on 32 bit machines + A2_16 = "A" * (2**16) + self.checkraises(OverflowError, A2_16, "replace", "", A2_16) + self.checkraises(OverflowError, A2_16, "replace", "A", A2_16) + self.checkraises(OverflowError, A2_16, "replace", "AA", A2_16+A2_16) + + def test_removeprefix(self): + self.checkequal('am', 'spam', 'removeprefix', 'sp') + self.checkequal('spamspam', 'spamspamspam', 'removeprefix', 'spam') + self.checkequal('spam', 'spam', 'removeprefix', 'python') + self.checkequal('spam', 'spam', 'removeprefix', 'spider') + self.checkequal('spam', 'spam', 'removeprefix', 'spam and eggs') + + self.checkequal('', '', 'removeprefix', '') + self.checkequal('', '', 'removeprefix', 'abcde') + self.checkequal('abcde', 'abcde', 'removeprefix', '') + self.checkequal('', 'abcde', 'removeprefix', 'abcde') + + self.checkraises(TypeError, 'hello', 'removeprefix') + self.checkraises(TypeError, 'hello', 'removeprefix', 42) + self.checkraises(TypeError, 'hello', 'removeprefix', 42, 'h') + self.checkraises(TypeError, 'hello', 'removeprefix', 'h', 42) + self.checkraises(TypeError, 'hello', 'removeprefix', ("he", "l")) + + def test_removesuffix(self): + self.checkequal('sp', 'spam', 'removesuffix', 'am') + self.checkequal('spamspam', 'spamspamspam', 'removesuffix', 'spam') + self.checkequal('spam', 'spam', 'removesuffix', 'python') + self.checkequal('spam', 'spam', 'removesuffix', 'blam') + self.checkequal('spam', 'spam', 'removesuffix', 'eggs and spam') + + self.checkequal('', '', 'removesuffix', '') + self.checkequal('', '', 'removesuffix', 'abcde') + self.checkequal('abcde', 'abcde', 'removesuffix', '') + self.checkequal('', 'abcde', 'removesuffix', 'abcde') + + self.checkraises(TypeError, 'hello', 'removesuffix') + self.checkraises(TypeError, 'hello', 'removesuffix', 42) + self.checkraises(TypeError, 'hello', 'removesuffix', 42, 'h') + self.checkraises(TypeError, 'hello', 'removesuffix', 'h', 42) + self.checkraises(TypeError, 'hello', 'removesuffix', ("lo", "l")) + + def test_capitalize(self): + self.checkequal(' hello ', ' hello ', 'capitalize') + self.checkequal('Hello ', 'Hello ','capitalize') + self.checkequal('Hello ', 'hello ','capitalize') + self.checkequal('Aaaa', 'aaaa', 'capitalize') + self.checkequal('Aaaa', 'AaAa', 'capitalize') + + self.checkraises(TypeError, 'hello', 'capitalize', 42) + + def test_additional_split(self): + self.checkequal(['this', 'is', 'the', 'split', 'function'], + 'this is the split function', 'split') + + # by whitespace + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d ', 'split') + self.checkequal(['a', 'b c d'], 'a b c d', 'split', None, 1) + self.checkequal(['a', 'b', 'c d'], 'a b c d', 'split', None, 2) + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'split', None, 3) + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'split', None, 4) + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'split', None, + sys.maxsize-1) + self.checkequal(['a b c d'], 'a b c d', 'split', None, 0) + self.checkequal(['a b c d'], ' a b c d', 'split', None, 0) + self.checkequal(['a', 'b', 'c d'], 'a b c d', 'split', None, 2) + + self.checkequal([], ' ', 'split') + self.checkequal(['a'], ' a ', 'split') + self.checkequal(['a', 'b'], ' a b ', 'split') + self.checkequal(['a', 'b '], ' a b ', 'split', None, 1) + self.checkequal(['a b c '], ' a b c ', 'split', None, 0) + self.checkequal(['a', 'b c '], ' a b c ', 'split', None, 1) + self.checkequal(['a', 'b', 'c '], ' a b c ', 'split', None, 2) + self.checkequal(['a', 'b', 'c'], ' a b c ', 'split', None, 3) + self.checkequal(['a', 'b'], '\n\ta \t\r b \v ', 'split') + aaa = ' a '*20 + self.checkequal(['a']*20, aaa, 'split') + self.checkequal(['a'] + [aaa[4:]], aaa, 'split', None, 1) + self.checkequal(['a']*19 + ['a '], aaa, 'split', None, 19) + + for b in ('arf\tbarf', 'arf\nbarf', 'arf\rbarf', + 'arf\fbarf', 'arf\vbarf'): + self.checkequal(['arf', 'barf'], b, 'split') + self.checkequal(['arf', 'barf'], b, 'split', None) + self.checkequal(['arf', 'barf'], b, 'split', None, 2) + + def test_additional_rsplit(self): + self.checkequal(['this', 'is', 'the', 'rsplit', 'function'], + 'this is the rsplit function', 'rsplit') + + # by whitespace + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d ', 'rsplit') + self.checkequal(['a b c', 'd'], 'a b c d', 'rsplit', None, 1) + self.checkequal(['a b', 'c', 'd'], 'a b c d', 'rsplit', None, 2) + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', None, 3) + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', None, 4) + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', None, + sys.maxsize-20) + self.checkequal(['a b c d'], 'a b c d', 'rsplit', None, 0) + self.checkequal(['a b c d'], 'a b c d ', 'rsplit', None, 0) + self.checkequal(['a b', 'c', 'd'], 'a b c d', 'rsplit', None, 2) + + self.checkequal([], ' ', 'rsplit') + self.checkequal(['a'], ' a ', 'rsplit') + self.checkequal(['a', 'b'], ' a b ', 'rsplit') + self.checkequal([' a', 'b'], ' a b ', 'rsplit', None, 1) + self.checkequal([' a b c'], ' a b c ', 'rsplit', + None, 0) + self.checkequal([' a b','c'], ' a b c ', 'rsplit', + None, 1) + self.checkequal([' a', 'b', 'c'], ' a b c ', 'rsplit', + None, 2) + self.checkequal(['a', 'b', 'c'], ' a b c ', 'rsplit', + None, 3) + self.checkequal(['a', 'b'], '\n\ta \t\r b \v ', 'rsplit', None, 88) + aaa = ' a '*20 + self.checkequal(['a']*20, aaa, 'rsplit') + self.checkequal([aaa[:-4]] + ['a'], aaa, 'rsplit', None, 1) + self.checkequal([' a a'] + ['a']*18, aaa, 'rsplit', None, 18) + + for b in ('arf\tbarf', 'arf\nbarf', 'arf\rbarf', + 'arf\fbarf', 'arf\vbarf'): + self.checkequal(['arf', 'barf'], b, 'rsplit') + self.checkequal(['arf', 'barf'], b, 'rsplit', None) + self.checkequal(['arf', 'barf'], b, 'rsplit', None, 2) + + def test_strip_whitespace(self): + self.checkequal('hello', ' hello ', 'strip') + self.checkequal('hello ', ' hello ', 'lstrip') + self.checkequal(' hello', ' hello ', 'rstrip') + self.checkequal('hello', 'hello', 'strip') + + b = ' \t\n\r\f\vabc \t\n\r\f\v' + self.checkequal('abc', b, 'strip') + self.checkequal('abc \t\n\r\f\v', b, 'lstrip') + self.checkequal(' \t\n\r\f\vabc', b, 'rstrip') + + # strip/lstrip/rstrip with None arg + self.checkequal('hello', ' hello ', 'strip', None) + self.checkequal('hello ', ' hello ', 'lstrip', None) + self.checkequal(' hello', ' hello ', 'rstrip', None) + self.checkequal('hello', 'hello', 'strip', None) + + def test_strip(self): + # strip/lstrip/rstrip with str arg + self.checkequal('hello', 'xyzzyhelloxyzzy', 'strip', 'xyz') + self.checkequal('helloxyzzy', 'xyzzyhelloxyzzy', 'lstrip', 'xyz') + self.checkequal('xyzzyhello', 'xyzzyhelloxyzzy', 'rstrip', 'xyz') + self.checkequal('hello', 'hello', 'strip', 'xyz') + self.checkequal('', 'mississippi', 'strip', 'mississippi') + + # only trim the start and end; does not strip internal characters + self.checkequal('mississipp', 'mississippi', 'strip', 'i') + + self.checkraises(TypeError, 'hello', 'strip', 42, 42) + self.checkraises(TypeError, 'hello', 'lstrip', 42, 42) + self.checkraises(TypeError, 'hello', 'rstrip', 42, 42) + + def test_ljust(self): + self.checkequal('abc ', 'abc', 'ljust', 10) + self.checkequal('abc ', 'abc', 'ljust', 6) + self.checkequal('abc', 'abc', 'ljust', 3) + self.checkequal('abc', 'abc', 'ljust', 2) + self.checkequal('abc*******', 'abc', 'ljust', 10, '*') + self.checkraises(TypeError, 'abc', 'ljust') + + def test_rjust(self): + self.checkequal(' abc', 'abc', 'rjust', 10) + self.checkequal(' abc', 'abc', 'rjust', 6) + self.checkequal('abc', 'abc', 'rjust', 3) + self.checkequal('abc', 'abc', 'rjust', 2) + self.checkequal('*******abc', 'abc', 'rjust', 10, '*') + self.checkraises(TypeError, 'abc', 'rjust') + + def test_center(self): + self.checkequal(' abc ', 'abc', 'center', 10) + self.checkequal(' abc ', 'abc', 'center', 6) + self.checkequal('abc', 'abc', 'center', 3) + self.checkequal('abc', 'abc', 'center', 2) + self.checkequal('***abc****', 'abc', 'center', 10, '*') + self.checkraises(TypeError, 'abc', 'center') + + def test_swapcase(self): + self.checkequal('hEllO CoMPuTErS', 'HeLLo cOmpUteRs', 'swapcase') + + self.checkraises(TypeError, 'hello', 'swapcase', 42) + + def test_zfill(self): + self.checkequal('123', '123', 'zfill', 2) + self.checkequal('123', '123', 'zfill', 3) + self.checkequal('0123', '123', 'zfill', 4) + self.checkequal('+123', '+123', 'zfill', 3) + self.checkequal('+123', '+123', 'zfill', 4) + self.checkequal('+0123', '+123', 'zfill', 5) + self.checkequal('-123', '-123', 'zfill', 3) + self.checkequal('-123', '-123', 'zfill', 4) + self.checkequal('-0123', '-123', 'zfill', 5) + self.checkequal('000', '', 'zfill', 3) + self.checkequal('34', '34', 'zfill', 1) + self.checkequal('0034', '34', 'zfill', 4) + + self.checkraises(TypeError, '123', 'zfill') + + def test_islower(self): + self.checkequal(False, '', 'islower') + self.checkequal(True, 'a', 'islower') + self.checkequal(False, 'A', 'islower') + self.checkequal(False, '\n', 'islower') + self.checkequal(True, 'abc', 'islower') + self.checkequal(False, 'aBc', 'islower') + self.checkequal(True, 'abc\n', 'islower') + self.checkraises(TypeError, 'abc', 'islower', 42) + + def test_isupper(self): + self.checkequal(False, '', 'isupper') + self.checkequal(False, 'a', 'isupper') + self.checkequal(True, 'A', 'isupper') + self.checkequal(False, '\n', 'isupper') + self.checkequal(True, 'ABC', 'isupper') + self.checkequal(False, 'AbC', 'isupper') + self.checkequal(True, 'ABC\n', 'isupper') + self.checkraises(TypeError, 'abc', 'isupper', 42) + + def test_istitle(self): + self.checkequal(False, '', 'istitle') + self.checkequal(False, 'a', 'istitle') + self.checkequal(True, 'A', 'istitle') + self.checkequal(False, '\n', 'istitle') + self.checkequal(True, 'A Titlecased Line', 'istitle') + self.checkequal(True, 'A\nTitlecased Line', 'istitle') + self.checkequal(True, 'A Titlecased, Line', 'istitle') + self.checkequal(False, 'Not a capitalized String', 'istitle') + self.checkequal(False, 'Not\ta Titlecase String', 'istitle') + self.checkequal(False, 'Not--a Titlecase String', 'istitle') + self.checkequal(False, 'NOT', 'istitle') + self.checkraises(TypeError, 'abc', 'istitle', 42) + + def test_isspace(self): + self.checkequal(False, '', 'isspace') + self.checkequal(False, 'a', 'isspace') + self.checkequal(True, ' ', 'isspace') + self.checkequal(True, '\t', 'isspace') + self.checkequal(True, '\r', 'isspace') + self.checkequal(True, '\n', 'isspace') + self.checkequal(True, ' \t\r\n', 'isspace') + self.checkequal(False, ' \t\r\na', 'isspace') + self.checkraises(TypeError, 'abc', 'isspace', 42) + + def test_isalpha(self): + self.checkequal(False, '', 'isalpha') + self.checkequal(True, 'a', 'isalpha') + self.checkequal(True, 'A', 'isalpha') + self.checkequal(False, '\n', 'isalpha') + self.checkequal(True, 'abc', 'isalpha') + self.checkequal(False, 'aBc123', 'isalpha') + self.checkequal(False, 'abc\n', 'isalpha') + self.checkraises(TypeError, 'abc', 'isalpha', 42) + + def test_isalnum(self): + self.checkequal(False, '', 'isalnum') + self.checkequal(True, 'a', 'isalnum') + self.checkequal(True, 'A', 'isalnum') + self.checkequal(False, '\n', 'isalnum') + self.checkequal(True, '123abc456', 'isalnum') + self.checkequal(True, 'a1b3c', 'isalnum') + self.checkequal(False, 'aBc000 ', 'isalnum') + self.checkequal(False, 'abc\n', 'isalnum') + self.checkraises(TypeError, 'abc', 'isalnum', 42) + + def test_isascii(self): + self.checkequal(True, '', 'isascii') + self.checkequal(True, '\x00', 'isascii') + self.checkequal(True, '\x7f', 'isascii') + self.checkequal(True, '\x00\x7f', 'isascii') + self.checkequal(False, '\x80', 'isascii') + self.checkequal(False, '\xe9', 'isascii') + # bytes.isascii() and bytearray.isascii() has optimization which + # check 4 or 8 bytes at once. So check some alignments. + for p in range(8): + self.checkequal(True, ' '*p + '\x7f', 'isascii') + self.checkequal(False, ' '*p + '\x80', 'isascii') + self.checkequal(True, ' '*p + '\x7f' + ' '*8, 'isascii') + self.checkequal(False, ' '*p + '\x80' + ' '*8, 'isascii') + + def test_isdigit(self): + self.checkequal(False, '', 'isdigit') + self.checkequal(False, 'a', 'isdigit') + self.checkequal(True, '0', 'isdigit') + self.checkequal(True, '0123456789', 'isdigit') + self.checkequal(False, '0123456789a', 'isdigit') + + self.checkraises(TypeError, 'abc', 'isdigit', 42) + + def test_title(self): + self.checkequal(' Hello ', ' hello ', 'title') + self.checkequal('Hello ', 'hello ', 'title') + self.checkequal('Hello ', 'Hello ', 'title') + self.checkequal('Format This As Title String', "fOrMaT thIs aS titLe String", 'title') + self.checkequal('Format,This-As*Title;String', "fOrMaT,thIs-aS*titLe;String", 'title', ) + self.checkequal('Getint', "getInt", 'title') + self.checkraises(TypeError, 'hello', 'title', 42) + + def test_splitlines(self): + self.checkequal(['abc', 'def', '', 'ghi'], "abc\ndef\n\rghi", 'splitlines') + self.checkequal(['abc', 'def', '', 'ghi'], "abc\ndef\n\r\nghi", 'splitlines') + self.checkequal(['abc', 'def', 'ghi'], "abc\ndef\r\nghi", 'splitlines') + self.checkequal(['abc', 'def', 'ghi'], "abc\ndef\r\nghi\n", 'splitlines') + self.checkequal(['abc', 'def', 'ghi', ''], "abc\ndef\r\nghi\n\r", 'splitlines') + self.checkequal(['', 'abc', 'def', 'ghi', ''], "\nabc\ndef\r\nghi\n\r", 'splitlines') + self.checkequal(['', 'abc', 'def', 'ghi', ''], + "\nabc\ndef\r\nghi\n\r", 'splitlines', False) + self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'], + "\nabc\ndef\r\nghi\n\r", 'splitlines', True) + self.checkequal(['', 'abc', 'def', 'ghi', ''], "\nabc\ndef\r\nghi\n\r", + 'splitlines', keepends=False) + self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'], + "\nabc\ndef\r\nghi\n\r", 'splitlines', keepends=True) + + self.checkraises(TypeError, 'abc', 'splitlines', 42, 42) + + +class StringLikeTest(BaseTest): + # This testcase contains tests that can be used in all + # stringlike classes. Currently this is str and UserString. + + def test_hash(self): + # SF bug 1054139: += optimization was not invalidating cached hash value + a = self.type2test('DNSSEC') + b = self.type2test('') + for c in a: + b += c + hash(b) + self.assertEqual(hash(a), hash(b)) + + def test_capitalize_nonascii(self): + # check that titlecased chars are lowered correctly + # \u1ffc is the titlecased char + self.checkequal('\u1ffc\u1ff3\u1ff3\u1ff3', + '\u1ff3\u1ff3\u1ffc\u1ffc', 'capitalize') + # check with cased non-letter chars + self.checkequal('\u24c5\u24e8\u24e3\u24d7\u24de\u24dd', + '\u24c5\u24ce\u24c9\u24bd\u24c4\u24c3', 'capitalize') + self.checkequal('\u24c5\u24e8\u24e3\u24d7\u24de\u24dd', + '\u24df\u24e8\u24e3\u24d7\u24de\u24dd', 'capitalize') + self.checkequal('\u2160\u2171\u2172', + '\u2160\u2161\u2162', 'capitalize') + self.checkequal('\u2160\u2171\u2172', + '\u2170\u2171\u2172', 'capitalize') + # check with Ll chars with no upper - nothing changes here + self.checkequal('\u019b\u1d00\u1d86\u0221\u1fb7', + '\u019b\u1d00\u1d86\u0221\u1fb7', 'capitalize') + + def test_startswith(self): + self.checkequal(True, 'hello', 'startswith', 'he') + self.checkequal(True, 'hello', 'startswith', 'hello') + self.checkequal(False, 'hello', 'startswith', 'hello world') + self.checkequal(True, 'hello', 'startswith', '') + self.checkequal(False, 'hello', 'startswith', 'ello') + self.checkequal(True, 'hello', 'startswith', 'ello', 1) + self.checkequal(True, 'hello', 'startswith', 'o', 4) + self.checkequal(False, 'hello', 'startswith', 'o', 5) + self.checkequal(True, 'hello', 'startswith', '', 5) + self.checkequal(False, 'hello', 'startswith', 'lo', 6) + self.checkequal(True, 'helloworld', 'startswith', 'lowo', 3) + self.checkequal(True, 'helloworld', 'startswith', 'lowo', 3, 7) + self.checkequal(False, 'helloworld', 'startswith', 'lowo', 3, 6) + self.checkequal(True, '', 'startswith', '', 0, 1) + self.checkequal(True, '', 'startswith', '', 0, 0) + self.checkequal(False, '', 'startswith', '', 1, 0) + + # test negative indices + self.checkequal(True, 'hello', 'startswith', 'he', 0, -1) + self.checkequal(True, 'hello', 'startswith', 'he', -53, -1) + self.checkequal(False, 'hello', 'startswith', 'hello', 0, -1) + self.checkequal(False, 'hello', 'startswith', 'hello world', -1, -10) + self.checkequal(False, 'hello', 'startswith', 'ello', -5) + self.checkequal(True, 'hello', 'startswith', 'ello', -4) + self.checkequal(False, 'hello', 'startswith', 'o', -2) + self.checkequal(True, 'hello', 'startswith', 'o', -1) + self.checkequal(True, 'hello', 'startswith', '', -3, -3) + self.checkequal(False, 'hello', 'startswith', 'lo', -9) + + self.checkraises(TypeError, 'hello', 'startswith') + self.checkraises(TypeError, 'hello', 'startswith', 42) + + # test tuple arguments + self.checkequal(True, 'hello', 'startswith', ('he', 'ha')) + self.checkequal(False, 'hello', 'startswith', ('lo', 'llo')) + self.checkequal(True, 'hello', 'startswith', ('hellox', 'hello')) + self.checkequal(False, 'hello', 'startswith', ()) + self.checkequal(True, 'helloworld', 'startswith', ('hellowo', + 'rld', 'lowo'), 3) + self.checkequal(False, 'helloworld', 'startswith', ('hellowo', 'ello', + 'rld'), 3) + self.checkequal(True, 'hello', 'startswith', ('lo', 'he'), 0, -1) + self.checkequal(False, 'hello', 'startswith', ('he', 'hel'), 0, 1) + self.checkequal(True, 'hello', 'startswith', ('he', 'hel'), 0, 2) + + self.checkraises(TypeError, 'hello', 'startswith', (42,)) + + def test_endswith(self): + self.checkequal(True, 'hello', 'endswith', 'lo') + self.checkequal(False, 'hello', 'endswith', 'he') + self.checkequal(True, 'hello', 'endswith', '') + self.checkequal(False, 'hello', 'endswith', 'hello world') + self.checkequal(False, 'helloworld', 'endswith', 'worl') + self.checkequal(True, 'helloworld', 'endswith', 'worl', 3, 9) + self.checkequal(True, 'helloworld', 'endswith', 'world', 3, 12) + self.checkequal(True, 'helloworld', 'endswith', 'lowo', 1, 7) + self.checkequal(True, 'helloworld', 'endswith', 'lowo', 2, 7) + self.checkequal(True, 'helloworld', 'endswith', 'lowo', 3, 7) + self.checkequal(False, 'helloworld', 'endswith', 'lowo', 4, 7) + self.checkequal(False, 'helloworld', 'endswith', 'lowo', 3, 8) + self.checkequal(False, 'ab', 'endswith', 'ab', 0, 1) + self.checkequal(False, 'ab', 'endswith', 'ab', 0, 0) + self.checkequal(True, '', 'endswith', '', 0, 1) + self.checkequal(True, '', 'endswith', '', 0, 0) + self.checkequal(False, '', 'endswith', '', 1, 0) + + # test negative indices + self.checkequal(True, 'hello', 'endswith', 'lo', -2) + self.checkequal(False, 'hello', 'endswith', 'he', -2) + self.checkequal(True, 'hello', 'endswith', '', -3, -3) + self.checkequal(False, 'hello', 'endswith', 'hello world', -10, -2) + self.checkequal(False, 'helloworld', 'endswith', 'worl', -6) + self.checkequal(True, 'helloworld', 'endswith', 'worl', -5, -1) + self.checkequal(True, 'helloworld', 'endswith', 'worl', -5, 9) + self.checkequal(True, 'helloworld', 'endswith', 'world', -7, 12) + self.checkequal(True, 'helloworld', 'endswith', 'lowo', -99, -3) + self.checkequal(True, 'helloworld', 'endswith', 'lowo', -8, -3) + self.checkequal(True, 'helloworld', 'endswith', 'lowo', -7, -3) + self.checkequal(False, 'helloworld', 'endswith', 'lowo', 3, -4) + self.checkequal(False, 'helloworld', 'endswith', 'lowo', -8, -2) + + self.checkraises(TypeError, 'hello', 'endswith') + self.checkraises(TypeError, 'hello', 'endswith', 42) + + # test tuple arguments + self.checkequal(False, 'hello', 'endswith', ('he', 'ha')) + self.checkequal(True, 'hello', 'endswith', ('lo', 'llo')) + self.checkequal(True, 'hello', 'endswith', ('hellox', 'hello')) + self.checkequal(False, 'hello', 'endswith', ()) + self.checkequal(True, 'helloworld', 'endswith', ('hellowo', + 'rld', 'lowo'), 3) + self.checkequal(False, 'helloworld', 'endswith', ('hellowo', 'ello', + 'rld'), 3, -1) + self.checkequal(True, 'hello', 'endswith', ('hell', 'ell'), 0, -1) + self.checkequal(False, 'hello', 'endswith', ('he', 'hel'), 0, 1) + self.checkequal(True, 'hello', 'endswith', ('he', 'hell'), 0, 4) + + self.checkraises(TypeError, 'hello', 'endswith', (42,)) + + def test___contains__(self): + self.checkequal(True, '', '__contains__', '') + self.checkequal(True, 'abc', '__contains__', '') + self.checkequal(False, 'abc', '__contains__', '\0') + self.checkequal(True, '\0abc', '__contains__', '\0') + self.checkequal(True, 'abc\0', '__contains__', '\0') + self.checkequal(True, '\0abc', '__contains__', 'a') + self.checkequal(True, 'asdf', '__contains__', 'asdf') + self.checkequal(False, 'asd', '__contains__', 'asdf') + self.checkequal(False, '', '__contains__', 'asdf') + + def test_subscript(self): + self.checkequal('a', 'abc', '__getitem__', 0) + self.checkequal('c', 'abc', '__getitem__', -1) + self.checkequal('a', 'abc', '__getitem__', 0) + self.checkequal('abc', 'abc', '__getitem__', slice(0, 3)) + self.checkequal('abc', 'abc', '__getitem__', slice(0, 1000)) + self.checkequal('a', 'abc', '__getitem__', slice(0, 1)) + self.checkequal('', 'abc', '__getitem__', slice(0, 0)) + + self.checkraises(TypeError, 'abc', '__getitem__', 'def') + + for idx_type in ('def', object()): + expected_msg = "string indices must be integers, not '{}'".format(type(idx_type).__name__) + self.checkraises(TypeError, 'abc', '__getitem__', idx_type, expected_msg=expected_msg) + + def test_slice(self): + self.checkequal('abc', 'abc', '__getitem__', slice(0, 1000)) + self.checkequal('abc', 'abc', '__getitem__', slice(0, 3)) + self.checkequal('ab', 'abc', '__getitem__', slice(0, 2)) + self.checkequal('bc', 'abc', '__getitem__', slice(1, 3)) + self.checkequal('b', 'abc', '__getitem__', slice(1, 2)) + self.checkequal('', 'abc', '__getitem__', slice(2, 2)) + self.checkequal('', 'abc', '__getitem__', slice(1000, 1000)) + self.checkequal('', 'abc', '__getitem__', slice(2000, 1000)) + self.checkequal('', 'abc', '__getitem__', slice(2, 1)) + + self.checkraises(TypeError, 'abc', '__getitem__', 'def') + + def test_extended_getslice(self): + # Test extended slicing by comparing with list slicing. + s = string.ascii_letters + string.digits + indices = (0, None, 1, 3, 41, sys.maxsize, -1, -2, -37) + for start in indices: + for stop in indices: + # Skip step 0 (invalid) + for step in indices[1:]: + L = list(s)[start:stop:step] + self.checkequal("".join(L), s, '__getitem__', + slice(start, stop, step)) + + def test_mul(self): + super().test_mul() + self.checkequal('', 'abc', '__mul__', -1) + self.checkequal('', 'abc', '__mul__', 0) + self.checkequal('abc', 'abc', '__mul__', 1) + self.checkequal('abcabcabc', 'abc', '__mul__', 3) + self.checkraises(TypeError, 'abc', '__mul__') + self.checkraises(TypeError, 'abc', '__mul__', '') + # XXX: on a 64-bit system, this doesn't raise an overflow error, + # but either raises a MemoryError, or succeeds (if you have 54TiB) + #self.checkraises(OverflowError, 10000*'abc', '__mul__', 2000000000) + + def test_join(self): + # join now works with any sequence type + # moved here, because the argument order is + # different in string.join + self.checkequal('a b c d', ' ', 'join', ['a', 'b', 'c', 'd']) + self.checkequal('abcd', '', 'join', ('a', 'b', 'c', 'd')) + self.checkequal('bd', '', 'join', ('', 'b', '', 'd')) + self.checkequal('ac', '', 'join', ('a', '', 'c', '')) + self.checkequal('w x y z', ' ', 'join', Sequence()) + self.checkequal('abc', 'a', 'join', ('abc',)) + self.checkequal('z', 'a', 'join', UserList(['z'])) + self.checkequal('a.b.c', '.', 'join', ['a', 'b', 'c']) + self.assertRaises(TypeError, '.'.join, ['a', 'b', 3]) + for i in [5, 25, 125]: + self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join', + ['a' * i] * i) + self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join', + ('a' * i,) * i) + + class LiesAboutLengthSeq(Sequence): + def __init__(self): self.seq = ['a', 'b', 'c'] + def __len__(self): return 8 + + self.checkequal('a b c', ' ', 'join', LiesAboutLengthSeq()) + + self.checkraises(TypeError, ' ', 'join') + self.checkraises(TypeError, ' ', 'join', None) + self.checkraises(TypeError, ' ', 'join', 7) + self.checkraises(TypeError, ' ', 'join', [1, 2, bytes()]) + try: + def f(): + yield 4 + "" + self.fixtype(' ').join(f()) + except TypeError as e: + if '+' not in str(e): + self.fail('join() ate exception message') + else: + self.fail('exception not raised') + + def test_formatting(self): + self.checkequal('+hello+', '+%s+', '__mod__', 'hello') + self.checkequal('+10+', '+%d+', '__mod__', 10) + self.checkequal('a', "%c", '__mod__', "a") + self.checkequal('a', "%c", '__mod__', "a") + self.checkequal('"', "%c", '__mod__', 34) + self.checkequal('$', "%c", '__mod__', 36) + self.checkequal('10', "%d", '__mod__', 10) + self.checkequal('\x7f', "%c", '__mod__', 0x7f) + + for ordinal in (-100, 0x200000): + # unicode raises ValueError, str raises OverflowError + self.checkraises((ValueError, OverflowError), '%c', '__mod__', ordinal) + + longvalue = sys.maxsize + 10 + slongvalue = str(longvalue) + self.checkequal(' 42', '%3ld', '__mod__', 42) + self.checkequal('42', '%d', '__mod__', 42.0) + self.checkequal(slongvalue, '%d', '__mod__', longvalue) + self.checkcall('%d', '__mod__', float(longvalue)) + self.checkequal('0042.00', '%07.2f', '__mod__', 42) + self.checkequal('0042.00', '%07.2F', '__mod__', 42) + + self.checkraises(TypeError, 'abc', '__mod__') + self.checkraises(TypeError, '%(foo)s', '__mod__', 42) + self.checkraises(TypeError, '%s%s', '__mod__', (42,)) + self.checkraises(TypeError, '%c', '__mod__', (None,)) + self.checkraises(ValueError, '%(foo', '__mod__', {}) + self.checkraises(TypeError, '%(foo)s %(bar)s', '__mod__', ('foo', 42)) + self.checkraises(TypeError, '%d', '__mod__', "42") # not numeric + self.checkraises(TypeError, '%d', '__mod__', (42+0j)) # no int conversion provided + + # argument names with properly nested brackets are supported + self.checkequal('bar', '%((foo))s', '__mod__', {'(foo)': 'bar'}) + + # 100 is a magic number in PyUnicode_Format, this forces a resize + self.checkequal(103*'a'+'x', '%sx', '__mod__', 103*'a') + + self.checkraises(TypeError, '%*s', '__mod__', ('foo', 'bar')) + self.checkraises(TypeError, '%10.*f', '__mod__', ('foo', 42.)) + self.checkraises(ValueError, '%10', '__mod__', (42,)) + + # Outrageously large width or precision should raise ValueError. + self.checkraises(ValueError, '%%%df' % (2**64), '__mod__', (3.2)) + self.checkraises(ValueError, '%%.%df' % (2**64), '__mod__', (3.2)) + self.checkraises(OverflowError, '%*s', '__mod__', + (sys.maxsize + 1, '')) + self.checkraises(OverflowError, '%.*f', '__mod__', + (sys.maxsize + 1, 1. / 7)) + + class X(object): pass + self.checkraises(TypeError, 'abc', '__mod__', X()) + + @support.cpython_only + def test_formatting_c_limits(self): + _testcapi = import_helper.import_module('_testcapi') + SIZE_MAX = (1 << (_testcapi.PY_SSIZE_T_MAX.bit_length() + 1)) - 1 + self.checkraises(OverflowError, '%*s', '__mod__', + (_testcapi.PY_SSIZE_T_MAX + 1, '')) + self.checkraises(OverflowError, '%.*f', '__mod__', + (_testcapi.INT_MAX + 1, 1. / 7)) + # Issue 15989 + self.checkraises(OverflowError, '%*s', '__mod__', + (SIZE_MAX + 1, '')) + self.checkraises(OverflowError, '%.*f', '__mod__', + (_testcapi.UINT_MAX + 1, 1. / 7)) + + def test_floatformatting(self): + # float formatting + for prec in range(100): + format = '%%.%if' % prec + value = 0.01 + for x in range(60): + value = value * 3.14159265359 / 3.0 * 10.0 + self.checkcall(format, "__mod__", value) + + def test_inplace_rewrites(self): + # Check that strings don't copy and modify cached single-character strings + self.checkequal('a', 'A', 'lower') + self.checkequal(True, 'A', 'isupper') + self.checkequal('A', 'a', 'upper') + self.checkequal(True, 'a', 'islower') + + self.checkequal('a', 'A', 'replace', 'A', 'a') + self.checkequal(True, 'A', 'isupper') + + self.checkequal('A', 'a', 'capitalize') + self.checkequal(True, 'a', 'islower') + + self.checkequal('A', 'a', 'swapcase') + self.checkequal(True, 'a', 'islower') + + self.checkequal('A', 'a', 'title') + self.checkequal(True, 'a', 'islower') + + def test_partition(self): + + self.checkequal(('this is the par', 'ti', 'tion method'), + 'this is the partition method', 'partition', 'ti') + + # from raymond's original specification + S = 'http://www.python.org' + self.checkequal(('http', '://', 'www.python.org'), S, 'partition', '://') + self.checkequal(('http://www.python.org', '', ''), S, 'partition', '?') + self.checkequal(('', 'http://', 'www.python.org'), S, 'partition', 'http://') + self.checkequal(('http://www.python.', 'org', ''), S, 'partition', 'org') + + self.checkraises(ValueError, S, 'partition', '') + self.checkraises(TypeError, S, 'partition', None) + + def test_rpartition(self): + + self.checkequal(('this is the rparti', 'ti', 'on method'), + 'this is the rpartition method', 'rpartition', 'ti') + + # from raymond's original specification + S = 'http://www.python.org' + self.checkequal(('http', '://', 'www.python.org'), S, 'rpartition', '://') + self.checkequal(('', '', 'http://www.python.org'), S, 'rpartition', '?') + self.checkequal(('', 'http://', 'www.python.org'), S, 'rpartition', 'http://') + self.checkequal(('http://www.python.', 'org', ''), S, 'rpartition', 'org') + + self.checkraises(ValueError, S, 'rpartition', '') + self.checkraises(TypeError, S, 'rpartition', None) + + def test_none_arguments(self): + # issue 11828 + s = 'hello' + self.checkequal(2, s, 'find', 'l', None) + self.checkequal(3, s, 'find', 'l', -2, None) + self.checkequal(2, s, 'find', 'l', None, -2) + self.checkequal(0, s, 'find', 'h', None, None) + + self.checkequal(3, s, 'rfind', 'l', None) + self.checkequal(3, s, 'rfind', 'l', -2, None) + self.checkequal(2, s, 'rfind', 'l', None, -2) + self.checkequal(0, s, 'rfind', 'h', None, None) + + self.checkequal(2, s, 'index', 'l', None) + self.checkequal(3, s, 'index', 'l', -2, None) + self.checkequal(2, s, 'index', 'l', None, -2) + self.checkequal(0, s, 'index', 'h', None, None) + + self.checkequal(3, s, 'rindex', 'l', None) + self.checkequal(3, s, 'rindex', 'l', -2, None) + self.checkequal(2, s, 'rindex', 'l', None, -2) + self.checkequal(0, s, 'rindex', 'h', None, None) + + self.checkequal(2, s, 'count', 'l', None) + self.checkequal(1, s, 'count', 'l', -2, None) + self.checkequal(1, s, 'count', 'l', None, -2) + self.checkequal(0, s, 'count', 'x', None, None) + + self.checkequal(True, s, 'endswith', 'o', None) + self.checkequal(True, s, 'endswith', 'lo', -2, None) + self.checkequal(True, s, 'endswith', 'l', None, -2) + self.checkequal(False, s, 'endswith', 'x', None, None) + + self.checkequal(True, s, 'startswith', 'h', None) + self.checkequal(True, s, 'startswith', 'l', -2, None) + self.checkequal(True, s, 'startswith', 'h', None, -2) + self.checkequal(False, s, 'startswith', 'x', None, None) + + def test_find_etc_raise_correct_error_messages(self): + # issue 11828 + s = 'hello' + x = 'x' + self.assertRaisesRegex(TypeError, r'^find\b', s.find, + x, None, None, None) + self.assertRaisesRegex(TypeError, r'^rfind\b', s.rfind, + x, None, None, None) + self.assertRaisesRegex(TypeError, r'^index\b', s.index, + x, None, None, None) + self.assertRaisesRegex(TypeError, r'^rindex\b', s.rindex, + x, None, None, None) + self.assertRaisesRegex(TypeError, r'^count\b', s.count, + x, None, None, None) + self.assertRaisesRegex(TypeError, r'^startswith\b', s.startswith, + x, None, None, None) + self.assertRaisesRegex(TypeError, r'^endswith\b', s.endswith, + x, None, None, None) + + # issue #15534 + self.checkequal(10, "...\u043c......<", "find", "<") + + +class MixinStrUnicodeTest: + # Additional tests that only work with str. + + def test_bug1001011(self): + # Make sure join returns a NEW object for single item sequences + # involving a subclass. + # Make sure that it is of the appropriate type. + # Check the optimisation still occurs for standard objects. + t = self.type2test + class subclass(t): + pass + s1 = subclass("abcd") + s2 = t().join([s1]) + self.assertIsNot(s1, s2) + self.assertIs(type(s2), t) + + s1 = t("abcd") + s2 = t().join([s1]) + self.assertIs(s1, s2) diff --git a/crates/weavepy-vm/src/stdlib/python/test_support_hashlib_helper.py b/crates/weavepy-vm/src/stdlib/python/test_support_hashlib_helper.py new file mode 100644 index 0000000..a4e6c92 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/test_support_hashlib_helper.py @@ -0,0 +1,51 @@ +import functools +import hashlib +import unittest + +try: + import _hashlib +except ImportError: + _hashlib = None + + +def requires_hashdigest(digestname, openssl=None, usedforsecurity=True): + """Decorator raising SkipTest if a hashing algorithm is not available + + The hashing algorithm could be missing or blocked by a strict crypto + policy. + + If 'openssl' is True, then the decorator checks that OpenSSL provides + the algorithm. Otherwise the check falls back to built-in + implementations. The usedforsecurity flag is passed to the constructor. + + ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS + ValueError: unsupported hash type md4 + """ + def decorator(func_or_class): + if isinstance(func_or_class, type): + setUpClass = func_or_class.__dict__.get('setUpClass') + if setUpClass is None: + def setUpClass(cls): + super(func_or_class, cls).setUpClass() + setUpClass.__qualname__ = func_or_class.__qualname__ + '.setUpClass' + setUpClass.__module__ = func_or_class.__module__ + else: + setUpClass = setUpClass.__func__ + setUpClass = classmethod(decorator(setUpClass)) + func_or_class.setUpClass = setUpClass + return func_or_class + + @functools.wraps(func_or_class) + def wrapper(*args, **kwargs): + try: + if openssl and _hashlib is not None: + _hashlib.new(digestname, usedforsecurity=usedforsecurity) + else: + hashlib.new(digestname, usedforsecurity=usedforsecurity) + except ValueError: + raise unittest.SkipTest( + f"hash digest '{digestname}' is not available." + ) + return func_or_class(*args, **kwargs) + return wrapper + return decorator diff --git a/crates/weavepy-vm/src/stdlib/python/test_support_i18n_helper.py b/crates/weavepy-vm/src/stdlib/python/test_support_i18n_helper.py new file mode 100644 index 0000000..786862f --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/test_support_i18n_helper.py @@ -0,0 +1,22 @@ +"""Minimal `test.support.i18n_helper` shim for WeavePy. + +CPython's real helper shells out to `pygettext`/`msgfmt` (via `test.test_tools`) +to verify translation-snapshot freshness — infrastructure WeavePy's bundled run +doesn't carry. We expose the same surface (`TestTranslationsBase`, +`update_translation_snapshots`) so `test_getopt`/`test_optparse` import cleanly; +the two snapshot tests skip, while every other test in those modules runs. +""" + +import unittest + + +class TestTranslationsBase(unittest.TestCase): + def test_translation_files_exist(self): + self.skipTest("translation snapshots unavailable under WeavePy") + + def test_translation_snapshots_are_up_to_date(self): + self.skipTest("translation snapshots unavailable under WeavePy") + + +def update_translation_snapshots(module): + raise unittest.SkipTest("translation snapshots unavailable under WeavePy") diff --git a/crates/weavepy-vm/src/stdlib/python/test_support_init.py b/crates/weavepy-vm/src/stdlib/python/test_support_init.py index 8fc6f0e..7e3a103 100644 --- a/crates/weavepy-vm/src/stdlib/python/test_support_init.py +++ b/crates/weavepy-vm/src/stdlib/python/test_support_init.py @@ -18,8 +18,89 @@ import gc import os import sys +import time +import types import unittest +# --------------------------------------------------------------------------- +# Platform / build flags (faithful port of CPython's test.support surface). +# Many Lib/test modules import these at top level to gate platform-specific +# behaviour; absent them the whole module fails to import. +# --------------------------------------------------------------------------- +MS_WINDOWS = (sys.platform == 'win32') +is_jython = sys.platform.startswith('java') +is_android = sys.platform == "android" +is_emscripten = sys.platform == "emscripten" +is_wasi = sys.platform == "wasi" +is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} +is_apple = is_apple_mobile or sys.platform == "darwin" +is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' + +# WeavePy targets the standard GIL-enabled build. +Py_GIL_DISABLED = False + +has_fork_support = hasattr(os, "fork") and not ( + is_emscripten or is_wasi or is_apple_mobile or is_android +) + + +def requires_fork(): + return unittest.skipUnless(has_fork_support, "requires working os.fork()") + + +def requires_gil_enabled(msg="needs the GIL enabled"): + """Decorator for skipping tests on the free-threaded build.""" + return unittest.skipIf(Py_GIL_DISABLED, msg) + + +def requires_specialization(test): + # WeavePy does not expose the adaptive-specialization opcodes, so these + # tests are not applicable; skip them the way a non-specializing build does. + return unittest.skip("requires specialization")(test) + + +def requires_specialization_ft(test): + return unittest.skip("requires specialization")(test) + + +# Some CPython tests are skipped on the s390x buildbots; mirror the decorator +# so suites that reference it import cleanly (a no-op off s390x). +skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x') + + +def _requires_unix_version(sysname, min_version): + """SkipTest if running on `sysname` with a kernel older than min_version.""" + import platform + + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kw): + if platform.system() == sysname: + version_txt = platform.release().split('-', 1)[0] + try: + version = tuple(map(int, version_txt.split('.'))) + except ValueError: + pass + else: + if version < min_version: + raise unittest.SkipTest( + "%s version %s or higher required, not %s" + % (sysname, '.'.join(map(str, min_version)), version_txt)) + return func(*args, **kw) + return wrapper + return decorator + + +def requires_linux_version(*min_version): + """Decorator raising SkipTest if the Linux kernel is older than min_version.""" + return _requires_unix_version('Linux', min_version) + + +def control_characters_c0(): + """Return the C0 control characters (0x00-0x1F plus 0x7F) as strings.""" + return [chr(c) for c in range(0x00, 0x20)] + ["\x7F"] + + # Pull the helper submodules in so ``from test.support import os_helper`` # and bare ``support.os_helper`` both work. from test.support import os_helper @@ -272,6 +353,139 @@ def swap_item(obj, item, new_val): del obj[item] +@contextlib.contextmanager +def adjust_int_max_str_digits(max_digits): + """Temporarily change the integer string conversion length limit.""" + current = sys.get_int_max_str_digits() + try: + sys.set_int_max_str_digits(max_digits) + yield + finally: + sys.set_int_max_str_digits(current) + + +class _ClockInfo: + def __init__(self, implementation, resolution): + self.implementation = implementation + self.monotonic = True + self.adjustable = False + self.resolution = resolution + + +class CPUStopwatch: + """Context manager to roughly time a CPU-bound operation. + + WeavePy lacks ``time.process_time``/``time.get_clock_info`` so this is + backed by ``time.perf_counter``; the public surface (``seconds`` and + ``clock_info.resolution``) matches CPython's helper. + """ + + def __enter__(self): + self.clock_info = _ClockInfo("perf_counter", 1e-9) + self.get_time = time.perf_counter + self.context = disable_gc() + self.context.__enter__() + self.start_time = self.get_time() + self.seconds = 0.0 + return self + + def __exit__(self, *exc): + try: + end_time = self.get_time() + finally: + result = self.context.__exit__(*exc) + self.seconds = end_time - self.start_time + return result + + +def run_in_subinterp(code): + """Run ``code`` in a subinterpreter. + + WeavePy does not implement subinterpreters, so tests that depend on + per-interpreter isolation are skipped rather than silently mis-run. + """ + import unittest + raise unittest.SkipTest("subinterpreters are not supported") + + +def patch(test_instance, object_to_patch, attr_name, new_value): + """Override 'object_to_patch'.'attr_name' with 'new_value'. + + Also, add a cleanup procedure to 'test_instance' to restore + 'object_to_patch' value for 'attr_name'. + The 'attr_name' should be a valid attribute for 'object_to_patch'. + """ + # check that 'attr_name' is a real attribute for 'object_to_patch' + # will raise AttributeError if it does not exist + getattr(object_to_patch, attr_name) + + # keep a copy of the old value + attr_is_local = False + try: + old_value = object_to_patch.__dict__[attr_name] + except (AttributeError, TypeError, KeyError): + old_value = getattr(object_to_patch, attr_name, None) + else: + attr_is_local = True + + # restore the value when the test is done + def cleanup(): + if attr_is_local: + setattr(object_to_patch, attr_name, old_value) + else: + try: + delattr(object_to_patch, attr_name) + except (AttributeError, TypeError, KeyError): + pass + + test_instance.addCleanup(cleanup) + + # actually override the attribute + setattr(object_to_patch, attr_name, new_value) + + +def check__all__(test_case, module, name_of_module=None, extra=(), + not_exported=()): + """Assert that the __all__ variable of 'module' contains all public names. + + The module's public names (its API) are detected automatically based on + whether they are documented in the module's docstring by being prefixed by + a '>>>' followed by a space, or are imported from another module + (when ``name_of_module`` is provided). + + Args: + test_case: an instance of unittest.TestCase to use the assert* methods. + module: the module to check. + name_of_module: the name(s) of 'module' (in case the module imports + objects from other modules e.g. ``collections.abc`` imports from + ``_collections_abc``). This argument can be a sequence of names or + a string. + extra: names that are imported into the module but aren't part of + ``__all__``, which are still expected to be in ``__all__``. + not_exported: names that are in the module but expected to not be in + ``__all__``. + """ + + if name_of_module is None: + name_of_module = (module.__name__, ) + elif isinstance(name_of_module, str): + name_of_module = (name_of_module, ) + + expected = set(extra) + + for name in dir(module): + if name.startswith('_') or name in not_exported: + continue + obj = getattr(module, name) + + if (getattr(obj, '__module__', None) in name_of_module or + (not hasattr(obj, '__module__') and + not isinstance(obj, types.ModuleType))): + expected.add(name) + + test_case.assertCountEqual(module.__all__, expected) + + # --------------------------------------------------------------------------- # GC helpers # --------------------------------------------------------------------------- @@ -929,6 +1143,177 @@ def skip_if_pgo_task(test): return test if not _is_pgo else unittest.skip(msg)(test) +# --- RFC 0037 (WS9): helpers many CPython test modules import from +# `test.support`. CPython sources several of these from the `_testcapi` / +# `_testinternalcapi` C extensions, which WeavePy does not ship; we provide +# behaviour-equivalent fallbacks so the importing test bodies actually run. + +def get_c_recursion_limit(): + """Depth at which the interpreter raises ``RecursionError``. + + CPython reads ``_testcapi.Py_C_RECURSION_LIMIT`` (its separate C-stack + ceiling). WeavePy enforces a single Python-level recursion limit in the + dispatch loop (RFC 0037 WS1), so the meaningful value here is exactly + ``sys.getrecursionlimit()`` — what we actually raise at. + """ + return sys.getrecursionlimit() + + +def exceeds_recursion_limit(): + """For recursion tests, easily exceeds default recursion limit.""" + return get_c_recursion_limit() * 3 + + +def check_free_after_iterating(test, iter, cls, args=()): + done = False + def wrapper(): + class A(cls): + def __del__(self): + nonlocal done + done = True + try: + next(it) + except StopIteration: + pass + + it = iter(A(*args)) + # Issue 26494: Shouldn't crash + test.assertRaises(StopIteration, next, it) + + wrapper() + # The sequence should be deallocated just after the end of iterating + gc_collect() + test.assertTrue(done) + + +def subTests(arg_names, arg_values, /, *, _do_cleanups=False): + """Run multiple subtests with different parameters.""" + single_param = False + if isinstance(arg_names, str): + arg_names = arg_names.replace(',', ' ').split() + if len(arg_names) == 1: + single_param = True + arg_values = tuple(arg_values) + def decorator(func): + if isinstance(func, type): + raise TypeError('subTests() can only decorate methods, not classes') + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + for values in arg_values: + if single_param: + values = (values,) + subtest_kwargs = dict(zip(arg_names, values)) + with self.subTest(**subtest_kwargs): + func(self, *args, **kwargs, **subtest_kwargs) + if _do_cleanups: + self.doCleanups() + return wrapper + return decorator + + +def can_use_suppress_immortalization(suppress=True): + # WeavePy has no deferred-object immortalization, so the suppression + # context is always usable (it's a no-op). + return True + + +@contextlib.contextmanager +def suppress_immortalization(suppress=True): + """No-op on WeavePy. + + CPython toggles a refcount-immortalization optimization via + ``_testinternalcapi``; WeavePy has no such optimization, so there is + nothing to suppress and the body simply runs. + """ + yield + + +def skip_if_suppress_immortalization(): + # Nothing to skip: WeavePy never immortalizes deferred objects. + return None + + +def has_no_debug_ranges(): + # WeavePy emits per-instruction source positions (co_positions / debug + # ranges, RFC 0033), so tests guarded on their presence may run. + return False + + +def requires_debug_ranges(reason='requires co_positions / debug_ranges'): + try: + skip = has_no_debug_ranges() + except unittest.SkipTest as e: + skip = True + reason = e.args[0] if e.args else reason + return unittest.skipIf(skip, reason) + + +# WeavePy ships a `socket` module and its cooperative event loop does not +# need privileged sockets, so socket-gated test modules are allowed to run. +has_socket_support = True + + +def requires_working_socket(*, module=False): + """Skip tests or modules that require working sockets. + + Can be used as a function/class decorator or to skip an entire module. + """ + msg = "requires socket support" + if module: + if not has_socket_support: + raise unittest.SkipTest(msg) + else: + return unittest.skipUnless(has_socket_support, msg) + + +# WeavePy can spawn subprocesses through its `subprocess` module. +has_subprocess_support = True + + +def requires_subprocess(): + """Used for subprocess, os.spawn calls, fd inheritance.""" + return unittest.skipUnless(has_subprocess_support, "requires subprocess support") + + +@contextlib.contextmanager +def patch_list(orig): + """Like unittest.mock.patch.dict, but for lists.""" + try: + saved = orig[:] + yield + finally: + orig[:] = saved + + +class BrokenIter: + def __init__(self, init_raises=False, next_raises=False, iter_raises=False): + if init_raises: + 1/0 + self.next_raises = next_raises + self.iter_raises = iter_raises + + def __next__(self): + if self.next_raises: + 1/0 + + def __iter__(self): + if self.iter_raises: + 1/0 + return self + + +__all__ += [ + "get_c_recursion_limit", "exceeds_recursion_limit", + "check_free_after_iterating", "subTests", + "can_use_suppress_immortalization", "suppress_immortalization", + "skip_if_suppress_immortalization", + "has_no_debug_ranges", "requires_debug_ranges", + "has_socket_support", "requires_working_socket", + "has_subprocess_support", "requires_subprocess", + "patch_list", "BrokenIter", +] + + __all__ += ["open_urlresource", "SuppressCrashReport", "bigaddrspacetest", "TEST_DATA_DIR", "TEST_HOME_DIR", "skip_if_pgo_task", "Py_TRACE_REFS", "requires_mac_ver", "no_color", "force_not_colorized", diff --git a/crates/weavepy-vm/src/stdlib/python/test_tokenizedata_badsyntax_3131.py b/crates/weavepy-vm/src/stdlib/python/test_tokenizedata_badsyntax_3131.py new file mode 100644 index 0000000..901d374 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/test_tokenizedata_badsyntax_3131.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +€ = 2 diff --git a/crates/weavepy-vm/src/stdlib/python/test_tokenizedata_init.py b/crates/weavepy-vm/src/stdlib/python/test_tokenizedata_init.py new file mode 100644 index 0000000..ed6cc4c --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/test_tokenizedata_init.py @@ -0,0 +1,6 @@ +# Vendored CPython test fixture package (`test.tokenizedata`). +# +# CPython's `Lib/test/tokenizedata/` holds intentionally-malformed source +# files used by the lexer/tokenizer regression tests. `test_unicode_identifiers` +# imports `badsyntax_3131` to assert the exact `SyntaxError` raised for an +# invalid PEP 3131 identifier. The package `__init__` is empty upstream. diff --git a/crates/weavepy-vm/src/stdlib/python/typing.py b/crates/weavepy-vm/src/stdlib/python/typing.py index 1a9dfd9..7bb36b6 100644 --- a/crates/weavepy-vm/src/stdlib/python/typing.py +++ b/crates/weavepy-vm/src/stdlib/python/typing.py @@ -169,6 +169,49 @@ def __call__(self, *args, **kwargs): # ``List[int](...)`` constructs the *origin* class. return self.__origin__(*args, **kwargs) + def __or__(self, other): + return _make_union(self, other) + + def __ror__(self, other): + return _make_union(other, self) + + def __instancecheck__(self, obj): + # PEP 3119 hook. Only ``Union`` supports instance checks; any + # other subscripted generic (``List[int]``) is rejected exactly + # as CPython does — you can't ask "is x a list-of-int?". + if self.__origin__ is Union: + return any(isinstance(obj, arg) for arg in self.__args__) + raise TypeError( + "Subscripted generics cannot be used with class and instance checks" + ) + + def __subclasscheck__(self, cls): + if self.__origin__ is Union: + return any(issubclass(cls, arg) for arg in self.__args__) + raise TypeError( + "Subscripted generics cannot be used with class and instance checks" + ) + + +def _as_class(x): + """Coerce a bare typing alias to the runtime class it stands in for, + so ``issubclass``/``isinstance`` can compare against it.""" + if isinstance(x, _OriginAlias): + return x._origin + return x + + +def _make_union(a, b): + """Build ``a | b`` as ``Union[a, b]`` (PEP 604), flattening any + nested unions so ``int | str | bytes`` has three flat args.""" + + def flatten(x): + if isinstance(x, _GenericAlias) and x.__origin__ is Union: + return list(x.__args__) + return [x] + + return _GenericAlias(Union, tuple(flatten(a) + flatten(b))) + def _type_repr(t): if t is type(None): @@ -203,6 +246,20 @@ def __getitem__(self, params): alias._name = self._name return alias + def __or__(self, other): + return _make_union(self, other) + + def __ror__(self, other): + return _make_union(other, self) + + def __instancecheck__(self, obj): + # A bare alias (``typing.List``) checks against its origin class, + # mirroring ``isinstance(x, list)``. + return isinstance(obj, self._origin) + + def __subclasscheck__(self, cls): + return issubclass(_as_class(cls), self._origin) + List = _OriginAlias("List", list) Dict = _OriginAlias("Dict", dict) @@ -281,6 +338,62 @@ def __init_subclass__(cls, **kwargs): cls._is_protocol = True +# Infrastructure names that are never part of a protocol's structural +# signature. Mirrors CPython's ``typing.EXCLUDED_ATTRIBUTES`` so that +# *special* methods (``__int__``, ``__float__``, ``__abs__``, …) — which +# are exactly what protocols like ``SupportsInt`` describe — are retained +# while dunders that every object carries are dropped. +_PROTOCOL_EXCLUDED_ATTRS = frozenset( + { + "__abstractmethods__", + "__annotations__", + "__dict__", + "__doc__", + "__init__", + "__module__", + "__name__", + "__qualname__", + "__new__", + "__slots__", + "__subclasshook__", + "__weakref__", + "__class_getitem__", + "__init_subclass__", + "__orig_bases__", + "__parameters__", + "__classcell__", + "__mro__", + "__bases__", + "_is_protocol", + "_is_runtime_protocol", + "_protocol_attrs", + } +) + + +def _get_protocol_attrs(cls): + """Collect the structural attribute names a protocol requires. + + Walks the protocol's own MRO (skipping ``object``/``Protocol``/ + ``Generic``) and unions each base's namespace and annotations, + dropping the infrastructure dunders in + :data:`_PROTOCOL_EXCLUDED_ATTRS`. + """ + attrs = set() + for base in getattr(cls, "__mro__", (cls,)): + if getattr(base, "__name__", "") in ("Protocol", "Generic", "object"): + continue + names = list(getattr(base, "__dict__", ()) or ()) + names += list(getattr(base, "__annotations__", {}) or {}) + for name in names: + if name.startswith("_abc_"): + continue + if name in _PROTOCOL_EXCLUDED_ATTRS: + continue + attrs.add(name) + return attrs + + def runtime_checkable(cls): """Enable ``isinstance``/``issubclass`` against a Protocol class. @@ -291,15 +404,87 @@ def runtime_checkable(cls): if not getattr(cls, "_is_protocol", False): raise TypeError("runtime_checkable expects a Protocol subclass") cls._is_runtime_protocol = True - protocol_attrs = set() - for name in dir(cls): - if name.startswith("_"): - continue - protocol_attrs.add(name) - cls._protocol_attrs = protocol_attrs + cls._protocol_attrs = _get_protocol_attrs(cls) return cls +# ---- numeric "Supports*" protocols ------------------------------------------ +# Runtime-checkable structural protocols from the stdlib. ``isinstance(x, P)`` +# is True iff ``x`` exposes the corresponding special method. + + +@runtime_checkable +class SupportsInt(Protocol): + """An ABC with one abstract method ``__int__``.""" + + __slots__ = () + + def __int__(self) -> int: + pass + + +@runtime_checkable +class SupportsFloat(Protocol): + """An ABC with one abstract method ``__float__``.""" + + __slots__ = () + + def __float__(self) -> float: + pass + + +@runtime_checkable +class SupportsComplex(Protocol): + """An ABC with one abstract method ``__complex__``.""" + + __slots__ = () + + def __complex__(self) -> complex: + pass + + +@runtime_checkable +class SupportsBytes(Protocol): + """An ABC with one abstract method ``__bytes__``.""" + + __slots__ = () + + def __bytes__(self) -> bytes: + pass + + +@runtime_checkable +class SupportsAbs(Protocol): + """An ABC with one abstract method ``__abs__`` that is covariant in + its return type.""" + + __slots__ = () + + def __abs__(self): + pass + + +@runtime_checkable +class SupportsRound(Protocol): + """An ABC with one abstract method ``__round__`` that is covariant in + its return type.""" + + __slots__ = () + + def __round__(self, ndigits: int = 0): + pass + + +@runtime_checkable +class SupportsIndex(Protocol): + """An ABC with one abstract method ``__index__``.""" + + __slots__ = () + + def __index__(self) -> int: + pass + + # ---- functional helpers ----------------------------------------------------- @@ -407,6 +592,116 @@ def TYPE_CHECKING(): TYPE_CHECKING = False +# ---- NamedTuple (PEP 526 class syntax + functional syntax) ----------------- + + +def _make_nmtuple(name, types, module, defaults=()): + """Build a ``collections.namedtuple`` carrying ``__annotations__``. + + ``types`` is an iterable of ``(field_name, annotation)`` pairs. We + keep the annotation as-is (weavepy's typing is intentionally + permissive — no runtime ``_type_check``). + """ + import collections + + types = dict(types) + fields = list(types) + nm_tpl = collections.namedtuple(name, fields, defaults=defaults, module=module) + nm_tpl.__annotations__ = types + # CPython also stamps the synthesised ``__new__`` with the same + # annotations; weavepy's namedtuple ``__new__`` may be a builtin that + # rejects attribute assignment, so make this best-effort. + try: + nm_tpl.__new__.__annotations__ = types + except (AttributeError, TypeError): + pass + return nm_tpl + + +# Attributes that NamedTuple class syntax may not override, and the +# class-machinery attributes that are copied through verbatim. +_prohibited = frozenset( + { + "__new__", + "__init__", + "__slots__", + "__getnewargs__", + "_fields", + "_field_defaults", + "_make", + "_replace", + "_asdict", + "_source", + } +) +_special = frozenset({"__module__", "__name__", "__annotations__", "__orig_bases__"}) + + +class NamedTupleMeta(type): + def __new__(cls, typename, bases, ns): + if _NamedTuple not in bases: + # Plain ``type.__new__`` bootstrap of ``_NamedTuple`` itself. + return super().__new__(cls, typename, bases, ns) + types = ns.get("__annotations__", {}) + default_names = [] + for field_name in types: + if field_name in ns: + default_names.append(field_name) + elif default_names: + raise TypeError( + "Non-default namedtuple field {} cannot follow default " + "field{} {}".format( + field_name, + "s" if len(default_names) > 1 else "", + ", ".join(default_names), + ) + ) + nm_tpl = _make_nmtuple( + typename, + types.items(), + defaults=[ns[n] for n in default_names], + module=ns.get("__module__", None), + ) + # Copy user-defined methods/attributes that aren't part of the + # namedtuple machinery (mirrors CPython's NamedTupleMeta). + for key, val in ns.items(): + if key in _prohibited: + raise AttributeError("Cannot overwrite NamedTuple attribute " + key) + elif key not in _special and key not in nm_tpl._fields: + setattr(nm_tpl, key, val) + return nm_tpl + + +def NamedTuple(typename, fields=None, /, **kwargs): + """Typed version of ``collections.namedtuple``. + + Supports the class-based syntax:: + + class Employee(NamedTuple): + name: str + id: int = 0 + + and the functional syntax:: + + Employee = NamedTuple('Employee', [('name', str), ('id', int)]) + """ + if fields is None: + fields = kwargs.items() + nt = _make_nmtuple(typename, fields, module=None) + nt.__orig_bases__ = (NamedTuple,) + return nt + + +_NamedTuple = type.__new__(NamedTupleMeta, "NamedTuple", (), {}) + + +def _namedtuple_mro_entries(bases): + return (_NamedTuple,) + + +NamedTuple.__mro_entries__ = _namedtuple_mro_entries + + # ---- nominal collections wrappers (PEP 585 aliases) ------------------------ # CPython 3.9+ deprecated ``typing.List`` etc. in favour of bare @@ -434,6 +729,14 @@ def TYPE_CHECKING(): "Annotated", "Generic", "Protocol", + "NamedTuple", + "SupportsInt", + "SupportsFloat", + "SupportsComplex", + "SupportsBytes", + "SupportsAbs", + "SupportsRound", + "SupportsIndex", "TypeVar", "ParamSpec", "TypeVarTuple", diff --git a/crates/weavepy-vm/src/stdlib/python/unittest_mock.py b/crates/weavepy-vm/src/stdlib/python/unittest_mock.py index 51d961b..6a3d0d5 100644 --- a/crates/weavepy-vm/src/stdlib/python/unittest_mock.py +++ b/crates/weavepy-vm/src/stdlib/python/unittest_mock.py @@ -6,9 +6,19 @@ but enough to run most test suites that lean on `mock.patch`. """ +import builtins import sys +# Public builtin names. Patching one of these onto a *module* implicitly +# creates it (CPython does the same): a module's functions resolve a bare +# name through the module globals before falling back to builtins, so the +# patched name is what they'll see. Lets e.g. `patch.object(mod, 'open')` +# work even though `mod` never bound `open` itself. +_builtins = {name for name in dir(builtins) if not name.startswith("_")} +_ModuleType = type(sys) + + __all__ = [ "Mock", "MagicMock", @@ -457,9 +467,14 @@ def _resolve_target(self): def __enter__(self): obj = self._resolve_target() self._had = hasattr(obj, self.attribute) + # A builtin name patched onto a module is created implicitly (see + # `_builtins`), matching CPython's `_patch.get_original`. + create = self.create or ( + self.attribute in _builtins and isinstance(obj, _ModuleType) + ) if self._had: self._original = getattr(obj, self.attribute) - elif not self.create: + elif not create: raise AttributeError(f"{obj!r} does not have the attribute {self.attribute!r}") new = self.new if new is DEFAULT: diff --git a/crates/weavepy-vm/src/stdlib/python/weakref.py b/crates/weavepy-vm/src/stdlib/python/weakref.py index 3f9b4e3..72f4504 100644 --- a/crates/weavepy-vm/src/stdlib/python/weakref.py +++ b/crates/weavepy-vm/src/stdlib/python/weakref.py @@ -231,6 +231,26 @@ def copy(self): new[k] = self[k] return new + __copy__ = copy + + def __deepcopy__(self, memo): + from copy import deepcopy + + new = WeakValueDictionary() + for k in self.keys(): + new[deepcopy(k, memo)] = self[k] + return new + + def __eq__(self, other): + # Mirror `_collections_abc.Mapping.__eq__`: two weak mappings are + # equal iff their *live* items compare equal as plain dicts. Needed + # so `copy.copy(wd) == wd` (test_copy) holds. + if not isinstance(other, WeakValueDictionary): + return NotImplemented + return dict(self.items()) == dict(other.items()) + + __hash__ = None + def expire(self, key): self._data.pop(key, None) @@ -345,6 +365,24 @@ def copy(self): new[k] = v return new + __copy__ = copy + + def __deepcopy__(self, memo): + from copy import deepcopy + + new = WeakKeyDictionary() + for key, value in self.items(): + new[key] = deepcopy(value, memo) + return new + + def __eq__(self, other): + # See WeakValueDictionary.__eq__ — equal iff live items match. + if not isinstance(other, WeakKeyDictionary): + return NotImplemented + return dict(self.items()) == dict(other.items()) + + __hash__ = None + def keyrefs(self): return [k for (k, _) in self._entries] diff --git a/crates/weavepy-vm/src/stdlib/random.rs b/crates/weavepy-vm/src/stdlib/random.rs index fe6b1a4..b8c1dc3 100644 --- a/crates/weavepy-vm/src/stdlib/random.rs +++ b/crates/weavepy-vm/src/stdlib/random.rs @@ -99,6 +99,10 @@ pub fn build(_cache: &ModuleCache) -> Rc { DictKey(Object::from_static("gauss")), b("gauss", random_gauss), ); + d.insert( + DictKey(Object::from_static("getrandbits")), + b("getrandbits", random_getrandbits), + ); } Rc::new(PyModule { name: "random".to_owned(), @@ -132,6 +136,40 @@ fn random_random(_args: &[Object]) -> Result { Ok(Object::Float(v)) } +/// Module-level `random.getrandbits(k)` — a non-negative int with `k` +/// random bits (`0 <= result < 2**k`), drawn from the module RNG. +fn random_getrandbits(args: &[Object]) -> Result { + use num_bigint::{BigInt, Sign}; + let k = match args.first() { + Some(Object::Bool(b)) => u64::from(*b), + Some(Object::Int(n)) if *n >= 0 => *n as u64, + Some(Object::Int(_)) => { + return Err(value_error("number of bits must be non-negative")) + } + _ => return Err(type_error("getrandbits() requires an integer argument")), + }; + if k == 0 { + return Ok(Object::Int(0)); + } + let nbytes = ((k + 7) / 8) as usize; + let excess = (nbytes as u64) * 8 - k; + let mut buf = vec![0u8; nbytes]; + RNG.with(|r| { + let mut rng = r.borrow_mut(); + let mut i = 0; + while i < nbytes { + let w = rng.next_u64().to_le_bytes(); + let take = (nbytes - i).min(8); + buf[i..i + take].copy_from_slice(&w[..take]); + i += take; + } + }); + if excess > 0 { + buf[nbytes - 1] &= 0xFFu8 >> excess; + } + Ok(Object::int_from_bigint(BigInt::from_bytes_le(Sign::Plus, &buf))) +} + fn random_uniform(args: &[Object]) -> Result { let a = to_f64(args.first())?; let b = to_f64(args.get(1))?; @@ -150,39 +188,93 @@ fn random_randint(args: &[Object]) -> Result { Ok(Object::Int(a + (raw % span) as i64)) } +/// Coerce a `randrange` bound to a `BigInt`, accepting any integer +/// (incl. arbitrary-precision) — CPython's `randrange` has no upper +/// bound on the magnitude of its arguments. +fn to_bigint(arg: Option<&Object>) -> Result { + use num_bigint::BigInt; + match arg { + Some(Object::Int(i)) => Ok(BigInt::from(*i)), + Some(Object::Bool(b)) => Ok(BigInt::from(i64::from(*b))), + Some(Object::Long(b)) => Ok((**b).clone()), + _ => Err(type_error("expected int")), + } +} + +/// Uniform random `BigInt` in `[0, n)` via rejection sampling on a +/// bit-masked candidate (`n` must be positive). Mirrors the shape of +/// CPython's `Random._randbelow` without depending on i64 width. +fn rand_below_bigint(n: &num_bigint::BigInt) -> num_bigint::BigInt { + use num_bigint::{BigInt, Sign}; + let bits = n.bits(); + if bits == 0 { + return BigInt::from(0); + } + let nbytes = ((bits + 7) / 8) as usize; + let excess = (nbytes as u64) * 8 - bits; + loop { + let mut buf = vec![0u8; nbytes]; + RNG.with(|r| { + let mut rng = r.borrow_mut(); + let mut i = 0; + while i < nbytes { + let w = rng.next_u64().to_le_bytes(); + let take = (nbytes - i).min(8); + buf[i..i + take].copy_from_slice(&w[..take]); + i += take; + } + }); + if excess > 0 { + buf[nbytes - 1] &= 0xFFu8 >> excess; + } + let cand = BigInt::from_bytes_le(Sign::Plus, &buf); + if &cand < n { + return cand; + } + } +} + fn random_randrange(args: &[Object]) -> Result { + use num_bigint::BigInt; + use num_integer::Integer; + let zero = BigInt::from(0); match args.len() { 1 => { - let stop = to_i64(args.first())?; - if stop <= 0 { - return Err(value_error("empty range for randrange")); + let stop = to_bigint(args.first())?; + if stop <= zero { + return Err(value_error("empty range for randrange()")); } - let raw = RNG.with(|r| r.borrow_mut().next_u64()); - Ok(Object::Int((raw % stop as u64) as i64)) + Ok(Object::int_from_bigint(rand_below_bigint(&stop))) } 2 => { - let start = to_i64(args.first())?; - let stop = to_i64(args.get(1))?; - if stop <= start { - return Err(value_error("empty range for randrange")); + let start = to_bigint(args.first())?; + let stop = to_bigint(args.get(1))?; + let width = &stop - &start; + if width <= zero { + return Err(value_error("empty range for randrange()")); } - let span = (stop - start) as u64; - let raw = RNG.with(|r| r.borrow_mut().next_u64()); - Ok(Object::Int(start + (raw % span) as i64)) + Ok(Object::int_from_bigint(start + rand_below_bigint(&width))) } 3 => { - let start = to_i64(args.first())?; - let stop = to_i64(args.get(1))?; - let step = to_i64(args.get(2))?; - if step == 0 { - return Err(value_error("zero step for randrange")); + let start = to_bigint(args.first())?; + let stop = to_bigint(args.get(1))?; + let step = to_bigint(args.get(2))?; + if step == zero { + return Err(value_error("zero step for randrange()")); } - let span = ((stop - start) / step) as u64; - if span == 0 { - return Err(value_error("empty range for randrange")); + let width = &stop - &start; + let one = BigInt::from(1); + // Count of reachable values: ceil(width/step), via floor div + // on the CPython-adjusted numerator (matches `range` length). + let n = if step > zero { + (&width + &step - &one).div_floor(&step) + } else { + (&width + &step + &one).div_floor(&step) + }; + if n <= zero { + return Err(value_error("empty range for randrange()")); } - let raw = RNG.with(|r| r.borrow_mut().next_u64()); - Ok(Object::Int(start + (raw % span) as i64 * step)) + Ok(Object::int_from_bigint(start + step * rand_below_bigint(&n))) } _ => Err(type_error("randrange expects 1-3 args")), } diff --git a/crates/weavepy-vm/src/stdlib/struct_mod.rs b/crates/weavepy-vm/src/stdlib/struct_mod.rs index 0f5430f..5f63eb8 100644 --- a/crates/weavepy-vm/src/stdlib/struct_mod.rs +++ b/crates/weavepy-vm/src/stdlib/struct_mod.rs @@ -28,6 +28,12 @@ use crate::error::{type_error, value_error, RuntimeError}; use crate::import::ModuleCache; use crate::object::{BuiltinFn, DictData, DictKey, Object, PyModule}; +/// Upper bound on a computed struct size, mirroring CPython's +/// `PY_SSIZE_T_MAX` guard in `_struct` (`prepare_s`). Beyond this we +/// raise `struct.error: total struct size too long` rather than letting +/// the repeat-count arithmetic overflow and panic the `Vec` allocator. +const MAX_STRUCT_SIZE: usize = isize::MAX as usize; + /// Format-string byte-order prefix. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Endian { @@ -115,10 +121,14 @@ impl CompiledFormat { let unit = element_size(code, endian)?; // For 's' / 'p' the count is the byte count of the string; // each field is a single value but consumes `n` bytes. - let bytes = match code { - 's' | 'p' => unit * n, - _ => unit * n, - }; + // Use checked arithmetic so a pathological repeat count + // (e.g. `struct.calcsize('999999999999s')`) raises CPython's + // `struct.error: total struct size too long` instead of + // overflowing and panicking the `Vec` allocator. + let bytes = unit + .checked_mul(n) + .filter(|b| *b <= MAX_STRUCT_SIZE) + .ok_or_else(|| struct_error("total struct size too long"))?; // Native alignment: pad to alignment if @ mode. if endian == Endian::Native { let align = native_align(code); @@ -134,7 +144,10 @@ impl CompiledFormat { } } fields.push(Field { code, count: n }); - size += bytes; + size = size + .checked_add(bytes) + .filter(|s| *s <= MAX_STRUCT_SIZE) + .ok_or_else(|| struct_error("total struct size too long"))?; } Ok(Self { endian, @@ -160,7 +173,10 @@ impl CompiledFormat { values.len() ))); } - let mut out = Vec::with_capacity(self.size); + // Cap the up-front reservation: `self.size` is already bounded by + // `MAX_STRUCT_SIZE`, but a multi-gigabyte format shouldn't pre- + // allocate everything at once — let the buffer grow as we write. + let mut out = Vec::with_capacity(self.size.min(1 << 20)); let mut idx = 0usize; for f in &self.fields { match f.code { @@ -231,18 +247,6 @@ impl CompiledFormat { self.unpack_from_offset(buf, 0).map(|(v, _)| v) } - fn unpack_from(&self, buf: &[u8], offset: usize) -> Result, RuntimeError> { - if buf.len() < offset + self.size { - return Err(struct_error(format!( - "unpack_from requires a buffer of at least {} bytes for unpacking {} bytes at offset {}", - offset + self.size, - self.size, - offset - ))); - } - self.unpack_from_offset(buf, offset).map(|(v, _)| v) - } - fn iter_unpack(&self, buf: &[u8]) -> Result>, RuntimeError> { if !buf.len().is_multiple_of(self.size) { return Err(struct_error(format!( @@ -747,10 +751,10 @@ fn b_pack_into(args: &[Object]) -> Result { match &args[1] { Object::ByteArray(buf) => { let mut buf = buf.borrow_mut(); - let off = offset.max(0) as usize; - if buf.len() < off + bytes.len() { - buf.resize(off + bytes.len(), 0); - } + // Resolve the (possibly negative) offset against the buffer + // and bounds-check without growing it — CPython's + // `pack_into` writes in place and never resizes. + let off = resolve_buffer_offset(offset, buf.len(), cf.size, "pack_into", true)?; buf[off..off + bytes.len()].copy_from_slice(&bytes); Ok(Object::None) } @@ -760,6 +764,55 @@ fn b_pack_into(args: &[Object]) -> Result { } } +/// Resolve a `pack_into`/`unpack_from` offset against a buffer of +/// `buf_len` bytes, matching CPython's `_struct` boundary diagnostics. +/// `size` is the struct's byte size; `for_pack` toggles the +/// pack- vs unpack-flavoured messages. Returns the non-negative byte +/// offset to start at, or a `struct.error` describing the overflow. +fn resolve_buffer_offset( + offset: i64, + buf_len: usize, + size: usize, + op: &str, + for_pack: bool, +) -> Result { + let size_i = size as i128; + let len_i = buf_len as i128; + let off = offset as i128; + let resolved = if off < 0 { + if off + size_i > 0 { + let verb = if for_pack { "pack" } else { "unpack" }; + let lead = if for_pack { + "no space to" + } else { + "not enough data to" + }; + return Err(struct_error(format!( + "{lead} {verb} {size} bytes at offset {offset}" + ))); + } + if off + len_i < 0 { + return Err(struct_error(format!( + "offset {offset} out of range for {buf_len}-byte buffer" + ))); + } + off + len_i + } else { + off + }; + // `resolved` is now non-negative. Check that `size` bytes fit. + if len_i - resolved < size_i { + let needed = (resolved as u128) + (size as u128); + let verb = if for_pack { "packing" } else { "unpacking" }; + return Err(struct_error(format!( + "{op} requires a buffer of at least {needed} bytes for \ + {verb} {size} bytes at offset {resolved} \ + (actual buffer size is {buf_len})" + ))); + } + Ok(resolved as usize) +} + fn b_unpack_from(args: &[Object]) -> Result { if args.len() < 2 { return Err(type_error("unpack_from() requires at least 2 arguments")); @@ -767,8 +820,9 @@ fn b_unpack_from(args: &[Object]) -> Result { let fmt = fmt_arg(args, 0)?; let cf = CompiledFormat::parse(&fmt)?; let buf = buffer_arg(&args[1])?; - let offset = args.get(2).and_then(|o| o.as_i64()).unwrap_or(0).max(0) as usize; - let vals = cf.unpack_from(&buf, offset)?; + let offset = args.get(2).and_then(|o| o.as_i64()).unwrap_or(0); + let off = resolve_buffer_offset(offset, buf.len(), cf.size, "unpack_from", false)?; + let (vals, _) = cf.unpack_from_offset(&buf, off)?; Ok(Object::new_tuple(vals)) } diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index 4a4c525..0ca0968 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -72,6 +72,21 @@ pub fn build_with_state( call_kw: None, })), ); + let es_fallback_exc = exc_info_stack.clone(); + d.insert( + DictKey(Object::from_static("exception")), + Object::Builtin(Rc::new(BuiltinFn { + name: "exception", + call: Box::new(move |_| { + if let Some(h) = crate::vm_singletons::current_thread_handles() { + sys_exception(&h.exc_info_stack) + } else { + sys_exception(&es_fallback_exc) + } + }), + call_kw: None, + })), + ); let eh_get = excepthook.clone(); d.insert( DictKey(Object::from_static("__excepthook__")), @@ -481,10 +496,30 @@ pub fn build(cache: &ModuleCache) -> Rc { DictKey(Object::from_static("setrecursionlimit")), builtin("setrecursionlimit", sys_setrecursionlimit), ); + d.insert( + DictKey(Object::from_static("get_int_max_str_digits")), + builtin("get_int_max_str_digits", sys_get_int_max_str_digits), + ); + d.insert( + DictKey(Object::from_static("set_int_max_str_digits")), + builtin("set_int_max_str_digits", sys_set_int_max_str_digits), + ); d.insert( DictKey(Object::from_static("intern")), builtin("intern", sys_intern), ); + d.insert( + DictKey(Object::from_static("getdefaultencoding")), + builtin("getdefaultencoding", sys_getdefaultencoding), + ); + d.insert( + DictKey(Object::from_static("getfilesystemencoding")), + builtin("getfilesystemencoding", sys_getfilesystemencoding), + ); + d.insert( + DictKey(Object::from_static("getfilesystemencodeerrors")), + builtin("getfilesystemencodeerrors", sys_getfilesystemencodeerrors), + ); // Standard I/O streams. We expose them as file-like objects // sharing the interpreter's host sinks, so `print()` and @@ -607,15 +642,83 @@ fn sys_exit(args: &[Object]) -> Result { } fn sys_getrecursionlimit(_args: &[Object]) -> Result { - Ok(Object::Int(1000)) + Ok(Object::Int(crate::recursion::recursion_limit() as i64)) } -fn sys_setrecursionlimit(args: &[Object]) -> Result { - let _ = args; - // No-op for now: the host stack does the bounding. +thread_local! { + // PEP 0467 int<->str conversion cap. WeavePy doesn't yet *enforce* the + // limit on conversion, but `sys.get/set_int_max_str_digits` must round-trip + // (test_int reads/sets it; many modules query it at import). + static INT_MAX_STR_DIGITS: std::cell::Cell = const { std::cell::Cell::new(4300) }; +} + +/// The current per-thread int↔str conversion digit cap (0 = unlimited). +/// Read by the str→int / int→str conversion paths to enforce PEP 0467. +pub fn int_max_str_digits() -> i64 { + INT_MAX_STR_DIGITS.with(|c| c.get()) +} + +fn sys_get_int_max_str_digits(_args: &[Object]) -> Result { + Ok(Object::Int(INT_MAX_STR_DIGITS.with(|c| c.get()))) +} + +fn sys_set_int_max_str_digits(args: &[Object]) -> Result { + let n = match args.first() { + Some(Object::Int(n)) => *n, + Some(Object::Bool(b)) => i64::from(*b), + _ => return Err(type_error("'maxdigits' must be an integer")), + }; + // CPython rejects values in (0, 640); 0 disables the limit. + if n != 0 && n < 640 { + return Err(value_error( + "maxdigits must be 0 or larger than 640", + )); + } + INT_MAX_STR_DIGITS.with(|c| c.set(n)); Ok(Object::None) } +fn sys_setrecursionlimit(args: &[Object]) -> Result { + // RFC 0037 (WS1) — the limit is now enforced by the dispatch loop's + // recursion guard rather than left to the native stack. + let limit = match args.first() { + Some(Object::Int(n)) => *n, + Some(Object::Bool(b)) => i64::from(*b), + Some(Object::Long(n)) => { + // Absurdly large limits are accepted by CPython; clamp to a + // value the usize counter can represent. + use num_traits::ToPrimitive; + n.to_i64().unwrap_or(i64::MAX) + } + Some(_) => { + return Err(type_error( + "'limit' must be an integer", + )) + } + None => { + return Err(type_error( + "setrecursionlimit expected 1 argument, got 0", + )) + } + }; + if limit < 1 { + return Err(value_error( + "recursion limit must be greater or equal than 1", + )); + } + match crate::recursion::set_limit(limit as usize) { + Ok(()) => Ok(Object::None), + Err(depth) => Err(RuntimeError::PyException(crate::error::PyException::new( + crate::builtin_types::make_exception( + "RecursionError", + format!( + "cannot set the recursion limit to {limit} at the recursion depth {depth}: the limit is too low" + ), + ), + ))), + } +} + fn sys_intern(args: &[Object]) -> Result { match args.first() { Some(Object::Str(_)) => Ok(args[0].clone()), @@ -623,6 +726,19 @@ fn sys_intern(args: &[Object]) -> Result { } } +fn sys_getdefaultencoding(_args: &[Object]) -> Result { + // CPython 3 always returns "utf-8" here. + Ok(Object::from_static("utf-8")) +} + +fn sys_getfilesystemencoding(_args: &[Object]) -> Result { + Ok(Object::from_static("utf-8")) +} + +fn sys_getfilesystemencodeerrors(_args: &[Object]) -> Result { + Ok(Object::from_static("surrogatepass")) +} + fn sys_getframe( args: &[Object], frame_stack: &Rc>>>, @@ -646,6 +762,19 @@ fn sys_getframe( Ok(Object::Frame(stack[idx].clone())) } +/// `sys.exception()` (PEP 3134 / 3.11+): the exception instance currently +/// being handled, or `None` if not in an `except`. Equivalent to +/// `sys.exc_info()[1]`. The verbatim CPython `contextlib` relies on this. +fn sys_exception( + exc_info_stack: &Rc>>, +) -> Result { + let stack = exc_info_stack.borrow(); + Ok(stack + .last() + .map(|top| top.instance.clone()) + .unwrap_or(Object::None)) +} + fn sys_exc_info( exc_info_stack: &Rc>>, ) -> Result { @@ -829,12 +958,21 @@ fn sys_flags_value() -> Object { "dev_mode", "utf8_mode", "safe_path", - "int_max_str_digits", "warn_default_encoding", ] { d.insert(DictKey(Object::from_static(name)), Object::Int(0)); } - Object::Dict(Rc::new(RefCell::new(d))) + // CPython's default cap on int<->str conversion size (PEP 0467 / + // `-X int_max_str_digits`). test_int reads this off `sys.flags`. + d.insert( + DictKey(Object::from_static("int_max_str_digits")), + Object::Int(4300), + ); + // CPython exposes `sys.flags` as a struct-sequence answering attribute + // access (`sys.flags.optimize`, `sys.flags.bytes_warning`, …), used by + // test_descr / test_bytes / test_collections. A SimpleNamespace gives + // us that attribute surface (mirrors `sys.float_info` above). + Object::SimpleNamespace(Rc::new(RefCell::new(d))) } fn sys_float_info() -> Object { @@ -886,9 +1024,13 @@ fn sys_int_info() -> Object { fn sys_hash_info() -> Object { let mut d = DictData::new(); d.insert(DictKey(Object::from_static("width")), Object::Int(64)); + // `_PyHASH_MODULUS` on a 64-bit build is the Mersenne prime 2**61-1, + // which is also the modulus `python_int_hash`/`py_hash_double` reduce + // through. test_numeric_tower derives `_PyHASH_MODULUS` from this field + // and checks exact Fraction hashes against it, so it must match. d.insert( DictKey(Object::from_static("modulus")), - Object::Int(i64::MAX), + Object::Int((1i64 << 61) - 1), ); d.insert(DictKey(Object::from_static("inf")), Object::Int(314_159)); d.insert(DictKey(Object::from_static("nan")), Object::Int(0)); diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs index 16cea09..6c57dd3 100644 --- a/crates/weavepy-vm/src/stdlib/thread_real.rs +++ b/crates/weavepy-vm/src/stdlib/thread_real.rs @@ -506,8 +506,15 @@ fn start_new_thread(args: &[Object]) -> Result { let worker_func = func.clone(); let worker_lock = join_lock.clone(); let entry_name = thread_name.clone(); + // RFC 0037 (WS1): worker threads recurse through the same + // recursive-descent evaluator as the main thread, so they need the + // same generous stack reserve for `sys.setrecursionlimit` to bind + // before the native stack. (std's default thread stack is only + // ~2 MiB.) The reserve is committed lazily by the OS. + const WORKER_STACK_BYTES: usize = 1024 * 1024 * 1024; // 1 GiB let handle = std::thread::Builder::new() .name(format!("weavepy-worker-{}", synth_id)) + .stack_size(WORKER_STACK_BYTES) .spawn(move || { crate::vm_singletons::install_worker_thread_id(synth_id); // The parent published this entry into the slot below diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs index 8d4511d..900e8b5 100644 --- a/crates/weavepy-vm/src/stdlib/weakref_real.rs +++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs @@ -115,15 +115,45 @@ fn b_dyn( })) } +/// Type-level `__call__` for `weakref`/proxy instances. +/// +/// CPython looks up special methods (here `__call__`) on the *type*, +/// not the instance, so `weakref.ref(obj)()` must resolve `__call__` +/// via the class MRO. Each ref instance stores its per-target deref +/// closure under `__weakref_get__` in its own dict; this shared +/// type-level method bridges to it so `r()` returns the live target +/// (or `None` once the referent is collected). +fn ref_type_call(args: &[Object]) -> Result { + let me = args + .first() + .ok_or_else(|| type_error("__call__() missing self"))?; + if let Object::Instance(inst) = me { + let getter = inst + .dict + .borrow() + .get(&DictKey(Object::from_static("__weakref_get__"))) + .cloned(); + if let Some(Object::Builtin(b)) = getter { + return (b.call)(&[]); + } + } + Err(type_error("__call__() requires a weakref instance")) +} + fn ref_type() -> Rc { REF_TYPE.with(|cell| { if let Some(t) = cell.borrow().clone() { return t; } + let mut type_dict = DictData::new(); + type_dict.insert( + DictKey(Object::from_static("__call__")), + b("__call__", ref_type_call), + ); let t = TypeObject::new_with_flags( "weakref", vec![crate::builtin_types::builtin_types().object_.clone()], - DictData::new(), + type_dict, TypeFlags { is_exception: false, is_builtin: true, diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index 75fbc41..3565800 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -8,8 +8,10 @@ //! so the `Object::Type` variant carries an `Rc`. The MRO //! is C3 linearised at class-creation time and cached on the type. +use crate::sync::Cell; use crate::sync::Rc; use crate::sync::RefCell; +use crate::sync::Weak; use crate::error::{type_error, RuntimeError}; use crate::object::{DictData, DictKey, Object}; @@ -44,6 +46,18 @@ pub struct TypeObject { /// `__dict__`). Set when the user neither omits `__slots__` from /// any base nor lists `"__dict__"` in slots. pub forbids_dict: bool, + /// Direct subclasses of this type, tracked as *weak* references so + /// the parent→child edge doesn't form an uncollectable `Rc` cycle + /// with the strong child→parent `bases` edge. Mirrors CPython's + /// `tp_subclasses`; surfaced by `type.__subclasses__()` and used by + /// the ABC virtual-subclass machinery. + pub subclasses: RefCell>>, + /// Cached classification of this type's `__getattribute__` slot, so the + /// hot attribute path can skip an MRO walk: `0` = not yet computed, + /// `1` = default (`object.__getattribute__`), `2` = a user override. + /// Invalidated (reset to `0`) for the type and its subclasses whenever + /// `__getattribute__` is assigned to / deleted from a type's dict. + pub getattribute_kind: Cell, } impl std::fmt::Debug for TypeObject { @@ -122,12 +136,39 @@ impl TypeObject { metaclass: RefCell::new(None), slot_names: RefCell::new(Vec::new()), forbids_dict: false, + subclasses: RefCell::new(Vec::new()), + getattribute_kind: Cell::new(0), }); let mro = compute_c3(&ty, &bases, name)?; *ty.mro.borrow_mut() = mro; + // Register the new class as a (weak) direct subclass of each of + // its bases so `base.__subclasses__()` reports it. + for base in &bases { + base.subclasses.borrow_mut().push(Rc::downgrade(&ty)); + } Ok(ty) } + /// Reset the cached `__getattribute__` classification for this type and + /// every (transitive) subclass. Called when `__getattribute__` is + /// assigned to or deleted from a type's dict, since that can change the + /// resolved slot for the type *and* anything inheriting from it. Class + /// hierarchies are acyclic, so the recursion terminates. + pub fn invalidate_getattribute_cache(&self) { + self.getattribute_kind.set(0); + for sub in self.subclasses() { + sub.invalidate_getattribute_cache(); + } + } + + /// Live direct subclasses, in registration order. Dead weak refs + /// (subclasses that have been dropped) are pruned as a side effect. + pub fn subclasses(&self) -> Vec> { + let mut subs = self.subclasses.borrow_mut(); + subs.retain(|w| w.strong_count() > 0); + subs.iter().filter_map(Weak::upgrade).collect() + } + /// Internal: install a metaclass on this type. Used at startup /// to wire `type.__class__ is type` for the built-in `type` /// itself, and by [`crate::Vm::build_class`] when honouring the diff --git a/crates/weavepy-vm/src/weakref_registry.rs b/crates/weavepy-vm/src/weakref_registry.rs index 23cfd68..782839a 100644 --- a/crates/weavepy-vm/src/weakref_registry.rs +++ b/crates/weavepy-vm/src/weakref_registry.rs @@ -190,6 +190,27 @@ impl WeakRefRegistry { .unwrap_or(0) } + /// How many *strong clones* of the referent the registry is + /// currently holding for `id`. Every live, un-cleared slot keeps + /// one `Object` clone of its target alive (see + /// [`WeakRefSlot::target`]). The cycle collector subtracts this + /// from an object's outer refcount so a weakref does **not** keep + /// its referent reachable — otherwise `weakref.ref(obj)()` would + /// stay live forever and `WeakKeyDictionary`/`WeakValueDictionary` + /// would never self-clean after `del obj; gc.collect()`. + pub fn strong_clone_count(&self, id: ObjectId) -> usize { + let g = self.inner.borrow(); + g.slots + .get(&id) + .map(|v| { + v.iter() + .filter_map(Weak::upgrade) + .filter(|s| !s.is_dead() && s.target.borrow().is_some()) + .count() + }) + .unwrap_or(0) + } + /// Snapshot the live weakrefs targeting `id` as /// `Arc` values. Used by /// `_weakref.getweakrefs(obj)`. @@ -252,6 +273,12 @@ pub fn count_for(id: ObjectId) -> usize { with_registry(|r| r.count(id)) } +/// Convenience: count of registry-held strong clones of `id` in the +/// current thread. Used by the cycle collector's refcount accounting. +pub fn strong_clone_count(id: ObjectId) -> usize { + with_registry(|r| r.strong_clone_count(id)) +} + /// Convenience: collect every live weakref for `id` in the /// current thread. pub fn collect_for(id: ObjectId) -> Vec> { diff --git a/crates/weavepy/src/lib.rs b/crates/weavepy/src/lib.rs index ae72a15..95592ae 100644 --- a/crates/weavepy/src/lib.rs +++ b/crates/weavepy/src/lib.rs @@ -287,17 +287,7 @@ fn format_syntax_error(source: &str, filename: &str, byte: u32, message: &str) - } fn format_lex_error(source: &str, filename: &str, err: &lexer::LexError) -> String { - let byte = match err { - lexer::LexError::UnterminatedString { pos } - | lexer::LexError::InvalidChar { pos, .. } - | lexer::LexError::InconsistentIndent { pos } - | lexer::LexError::UnknownDedent { pos } - | lexer::LexError::InvalidNumber { pos, .. } - | lexer::LexError::InvalidStringPrefix { pos, .. } - | lexer::LexError::StrayBackslash { pos } - | lexer::LexError::UnexpectedEof { pos, .. } => *pos, - }; - format_syntax_error(source, filename, byte, &err.to_string()) + format_syntax_error(source, filename, err.byte_offset(), &err.to_string()) } /// `(line, column, line_text)` derived from a byte offset. diff --git a/docs/rfcs/0037-cpython-lib-test-conformance-sweep-wave-2.md b/docs/rfcs/0037-cpython-lib-test-conformance-sweep-wave-2.md new file mode 100644 index 0000000..f403af5 --- /dev/null +++ b/docs/rfcs/0037-cpython-lib-test-conformance-sweep-wave-2.md @@ -0,0 +1,430 @@ +# RFC 0037: CPython `Lib/test/` conformance sweep, wave 2 — root-cause clusters and verbatim module ports + +- **Status**: Accepted +- **Authors**: WeavePy authors +- **Created**: 2026-06-02 +- **Tracking issue**: TBD +- **Builds on**: RFC 0036 (wired a real CPython 3.13 `Lib/test/` checkout + into `regrtest` and rewrote the touched `expectations.toml` rows from + guesses to a *measured* baseline), RFC 0035 (faithful `re`/Unicode — + same "port CPython verbatim where behaviour is defined by CPython" + ethos), RFC 0033 (`ast`/`dis`/`marshal` introspection), RFC 0015 + (object-model completion). + +## Summary + +RFC 0036 made the CPython regression suite *runnable and measured*: with +`vendor/cpython/Lib/test/` checked out, `weavepy-conformance regrtest +--mode subprocess` now produces honest per-file verdicts, and the +committed baseline in `tests/regrtest/expectations.toml` is `--check` +clean. That baseline currently records **115 `fail` rows and 30 `skip` +rows** against the real suite — the long tail the README calls out as +"still being worked through file by file." + +This RFC is **wave 2 of the sweep**: instead of grinding one test at a +time, it attacks the *shared root causes* that gate whole clusters of +those files, then ports the handful of stdlib modules whose absence +blocks the largest remaining groups. The work is organised into ten +workstreams (WS1–WS10). The throughline is unchanged from RFC 0035/0036: +**where behaviour is defined by CPython, port CPython** (verbatim Python +modules, faithful semantics) rather than re-approximate it. + +The deliverable is measured, not aspirational: every workstream names the +`expectations.toml` rows it flips, and the commit is not done until a +fresh subprocess sweep is `--check` clean with those rows rewritten from +`fail`/`skip` to `pass`. + +## Motivation + +The README's headline promise is "a 100% compatible, drop-in replacement +for CPython … using CPython's own test suite as a guiding standard." RFC +0036 made that claim *auditable*; this RFC makes the audited number move. + +The key observation from the RFC 0036 sweep — reaffirmed by re-reading +every `reason` field in `expectations.toml` — is that the 115 failures +are **not** 115 independent bugs. They cluster behind a small number of +missing primitives: + +- A *single* missing recursion guard takes the **whole process down** + (`abort()`/stack overflow) on any test that probes deep recursion, and + in `--mode in-process` it can take the runner with it. `test_exceptions` + is the named victim, but it also makes the in-process bundled runner + fragile. +- A *single* ASCII-only identifier scanner blocks every file that uses a + non-ASCII identifier (PEP 3131) — directly `test_unicode_identifiers`, + and transitively any ported module that does. +- A *partial* `str.format`/`%` mini-language shows up as the first + failure in `test_format`, `test_string`, `test_unicode`, and several + numeric files. +- A *handful of absent modules* (`collections.abc`, `cmath`, `calendar`, + `pydoc`, `locale`/`encodings`) are, per RFC 0036's own "non-goals" + note, "the module-level dependencies gating the largest remaining + clusters." + +Fixing those primitives is worth far more than its line count, exactly +as RFC 0036 found when one parser fix (`[*a, *b]`) and one lexer fix +(`1.`) each unblocked multiple files at once. + +## CPython reference + +This RFC matches CPython 3.13 behaviour as defined by: + +- **Recursion**: `sys.setrecursionlimit`/`getrecursionlimit`, + `RecursionError`, and `Py_EnterRecursiveCall` / `Py_C_RECURSION_LIMIT` + (CPython `Python/ceval.c`, `Include/cpython/pystate.h`). Tests: + `Lib/test/test_exceptions.py` (`test_recursion*`), + `Lib/test/test_sys.py` (`test_recursionlimit*`). +- **PEP 3131** — Unicode identifiers (`XID_Start`/`XID_Continue`, + NFKC normalization of identifiers). Test: + `Lib/test/test_unicode_identifiers.py`. Reference: + `Lib/tokenize.py`, `Parser/pegen.c` `_PyPegen_normalize_name`. +- **PEP 701** — f-strings (backslashes in expressions, nested same-quote + strings, multiline). Tests: `Lib/test/test_fstring.py`, + `test_string_literals.py`. +- **Format mini-language** — `Lib/test/test_format.py`, the + `Format Specification Mini-Language` docs, `str.__format__`, + `Objects/unicodeobject.c` `format`, `Python/formatter_unicode.c`. +- **Numeric tower** — `Lib/test/test_complex.py`, `test_float.py`, + `test_int.py`, `test_fractions.py`, `test_numeric_tower.py`; the + `numbers` ABC hierarchy and `Lib/fractions.py`. +- **Exceptions** — PEP 654 (`ExceptionGroup`/`except*`), PEP 678 + (`BaseException.add_note`/`__notes__`), `Lib/traceback.py`. +- **Class/descriptor protocol** — PEP 487 (`__init_subclass__`, + `__set_name__`), `Lib/test/test_descr.py`, `test_class.py`, + `test_dataclasses.py`, `test_enum.py`. +- **Verbatim module ports** — `Lib/_collections_abc.py`, + `Lib/cmath` (C module, ported as Python over the existing `math` + core), `Lib/calendar.py`, `Lib/pydoc.py`, `Lib/locale.py`, + `Lib/encodings/`. + +Where this RFC ports a CPython `.py` file verbatim, it is pinned to the +3.13 branch tag already vendored under `vendor/cpython/`. + +## Current baseline (measured starting point) + +- `cargo build --workspace` is green. +- Bundled `tests/regrtest/` suite: **52/52 pass** in subprocess mode + (`.scratch/ci.md`), `unexpected 0`. +- CPython `Lib/test/` allowlist in `expectations.toml`: + **115 `fail`, 30 `skip`, 3 explicit `pass`** (512 test files vendored). + +Wave 2 targets a coherent subset of those rows (see +[§Measured targets](#measured-targets)); the full tail remains a +multi-wave effort and this RFC does **not** claim to close it. + +## Detailed design + +Ten workstreams. Each lists the affected crate(s), the design, and the +`expectations.toml` rows it is expected to flip. Line-count estimates are +rough and include ported CPython `.py` plus the Rust glue/tests. + +### WS1 — Recursion guard (`weavepy-vm`) · ~1.5K LOC + +**Problem.** `sys.setrecursionlimit` is a no-op and `getrecursionlimit` +returns a hardcoded `1000`: + +```609:616:crates/weavepy-vm/src/stdlib/sys.rs +fn sys_getrecursionlimit(_args: &[Object]) -> Result { + Ok(Object::Int(1000)) +} + +fn sys_setrecursionlimit(args: &[Object]) -> Result { + let _ = args; + // No-op for now: the host stack does the bounding. + Ok(Object::None) +} +``` + +Deep Python recursion therefore overflows the *native* Rust stack and +`abort()`s the process instead of raising `RecursionError`. + +**Design.** +- Add a per-thread `recursion_depth: Cell` and `recursion_limit: + Cell` to the interpreter/thread state (alongside the existing + per-thread handles in `vm_singletons`). +- Increment/decrement around every Python frame entry in the call path + (the `CALL`/`CALL_FUNCTION_EX`/method-dispatch sites and the + generator/coroutine resume sites in `lib.rs`). On exceeding the limit, + raise `RecursionError("maximum recursion depth exceeded")` with the + "while normalizing an exception" variant when already unwinding. +- Mirror CPython's "low-water" reset behaviour so the handler itself can + run (`_Py_RecursionLimitLowerWaterMark`): allow a small overshoot for + the error path. +- Wire `sys.setrecursionlimit`/`getrecursionlimit` to the per-thread + field; validate the argument (`> 0`, fits the current depth). +- Independently, keep a coarse **native-stack** guard (probe remaining + stack via `stacker`-style check, or a generous frame-count ceiling) + so C-level recursion through dunder dispatch can't still overflow. + +**Flips:** `test_exceptions` (recursion case), and hardens the runner so +later workstreams' tests fail cleanly instead of aborting. Contributes +to `test_sys` once that's in the allowlist. + +### WS2 — Lexer/parser language gaps (`weavepy-lexer`, `weavepy-parser`) · ~3K LOC + +**WS2a — PEP 3131 Unicode identifiers.** `is_ident_start` is ASCII-only: + +```661:663:crates/weavepy-lexer/src/scanner.rs +fn is_ident_start(b: u8) -> bool { + b == b'_' || b.is_ascii_alphabetic() +} +``` + +The continue path already uses `unicode_ident::is_xid_continue` (scanner +line 297), so the dependency is present. Make `is_ident_start` decode the +next UTF-8 scalar and test `unicode_ident::is_xid_start` (plus the legacy +`_`), and NFKC-normalize identifier text at token creation to match +CPython's `_PyPegen_normalize_name`. **Flips:** `test_unicode_identifiers`. + +**WS2b — PEP 701 f-strings.** f-string lexing spans +`weavepy-lexer/src/token.rs` and parsing spans `weavepy-parser`. +Close the known gaps: backslashes inside f-string expression parts, +nested same-quote string literals, and multiline expressions. **Flips:** +`test_fstring`; contributes to `test_codecs`, `test_string_literals`. + +**WS2c — String-literal escapes.** `eval` of some escape forms raises +(octal/edge escapes, per the measured `test_string_literals` reason). +Audit the unescape routine against CPython's `decode_unicode_with_escapes` +(`\xhh`, `\ooo`, `\N{NAME}` — already added in RFC 0036, `\uXXXX`, +`\UXXXXXXXX`, deprecation-warning-but-accept for unknown escapes). +**Flips:** `test_string_literals`. + +**WS2d — `FOR_ITER` edge.** The measured `test_complex` reason cites a +`FOR_ITER` edge. Reproduce, fix the opcode handler in `lib.rs`. Folded in +with WS3 since it surfaces there. + +### WS3 — Numeric tower (`weavepy-vm`, frozen `fractions`/`numbers`) · ~3K LOC + +Measured reasons across `test_complex`, `test_float`, `test_int`, +`test_fractions`, `test_numeric_tower`: + +- `Fraction("1.2")` must accept decimal/scientific string literals + (CPython's `_RATIONAL_FORMAT` regex). Fix the frozen `fractions.py` + parser. +- `complex` repr/format edge cases (`Objects/complexobject.c` + `complex_repr`, `__format__`). +- `float.hex`/`float.fromhex` roundtrip + repr corner cases. +- `int` methods + `sys.int_info` struct-sequence shape (PEP 467 helpers, + `bit_count`, `is_integer`). +- Port `Lib/numbers.py` (the `Number`/`Complex`/`Real`/`Rational`/ + `Integral` ABCs) so `isinstance(x, numbers.Integral)` etc. are correct; + this also underpins WS8's `cmath`/`statistics`. + +**Flips:** `test_complex`, `test_float`, `test_int`, `test_fractions`, +`test_numeric_tower`; contributes to `test_decimal`/`test_statistics`. + +### WS4 — String/bytes formatting (`weavepy-vm`) · ~2.5K LOC + +The format mini-language and `%`-formatting are partial (`test_format`, +`test_string`, `test_unicode`, `test_format`'s interop with `str.format`). + +- Complete `str.__format__`/`format()` mini-language: fill/align, sign, + `#`, `0`, width, grouping (`,` and `_`), precision, and type codes for + `int`/`float`/`str`/`complex` (`b/c/d/e/E/f/F/g/G/n/o/s/x/X/%`). +- Complete `%`-formatting (`printf`-style) for `str` and `bytes`, + including `%r`/`%a`/`%c`, mapping keys, and `*` width/precision. +- `bytes`/`bytearray` `.translate`/`maketrans` table semantics + (`test_bytes`). + +**Flips:** `test_format`, `test_string`, `test_bytes`; contributes to +`test_unicode`. + +### WS5 — Exceptions, notes, groups, tracebacks (`weavepy-vm`, frozen `traceback`) · ~2.5K LOC + +- PEP 678 `BaseException.add_note` / `__notes__` (storage + display). +- PEP 654 `ExceptionGroup`/`BaseExceptionGroup` propagation and `.split`/ + `.subgroup`/`.derive` semantics; ensure `except*` lowering matches. +- `traceback` module: exception-chaining display, `StackSummary` format, + `TracebackException` (port the relevant parts of `Lib/traceback.py` + verbatim, building on WS1's frame data). + +**Flips:** `test_exceptions` (notes/groups portion, with WS1), +`test_traceback`; contributes to `test_contextlib`. + +### WS6 — Class / descriptor / metaclass machinery (`weavepy-vm`, frozen `dataclasses`/`enum`/`typing`/`abc`) · ~3.5K LOC + +Measured reasons across `test_class`, `test_descr`, `test_subclassinit`, +`test_dataclasses`, `test_enum`, `test_isinstance`, `test_abc`, +`test_typing`: + +- PEP 487 ordering: `__set_name__` is called on the new class's + attributes in definition order *after* the class object exists, then + `__init_subclass__` on the parent — get the ordering exactly right, + including inheritance and metaclass interaction. +- Descriptor protocol edges: slot conflicts, data vs non-data descriptor + precedence, `classmethod`/`staticmethod` chaining (`test_decorators`). +- `dataclasses`: `slots=True`, `kw_only`, `__init_subclass__` interplay. +- `enum`: `StrEnum`/`IntEnum` mixins, value re-use, `_missing_`. +- `abc`/`ABCMeta`: `register()` ordering + virtual-subclass cache + invalidation; depends on WS8's `_abc`/`abc` fidelity. + +**Flips:** `test_class`, `test_descr`, `test_subclassinit`, +`test_decorators`, `test_dataclasses`, `test_enum`, `test_abc`, +`test_isinstance`; contributes to `test_typing`. + +### WS7 — Iterators / generators / coroutines (`weavepy-vm`, frozen `contextlib`) · ~2K LOC + +- `generator.throw()` into `yield from`, `close()` during a `yield` + (`test_generators`). +- Coroutine `send`/`throw`, `async with`/`async for` edges + (`test_coroutines`). +- Async generator `aclose()`/`asend()` (`test_asyncgen`). +- `contextlib.asynccontextmanager` (measured-missing per + `test_contextlib_async`) + `ExitStack.callback` semantics. +- `iter(callable, sentinel)` + `__length_hint__` (`test_iter`). + +**Flips:** `test_generators`, `test_coroutines`, `test_asyncgen`, +`test_iter`, `test_contextlib_async`; contributes to `test_contextlib`, +`test_with`. + +### WS8 — Verbatim stdlib module ports (frozen Python) · ~6K LOC + +Per RFC 0036, these absent modules gate the largest remaining clusters. +All confirmed missing from `crates/weavepy-vm/src/stdlib/python/`: + +- **`collections.abc` / `_collections_abc`** — port `Lib/_collections_abc.py` + verbatim and expose `collections.abc`. High fan-out: imported across + the stdlib and by `typing`. +- **`cmath`** — port as Python over the existing `math` Rust core (or a + thin Rust module) with correct branch cuts. **Flips:** unblocks files + importing `cmath`. +- **`calendar`** — port `Lib/calendar.py`. **Flips:** `test_calendar`. +- **`locale` / `encodings`** — minimal but faithful `locale` + the + `encodings` package registry so `str.encode`/`open(encoding=…)` resolve + through the standard path. Gates `test_locale` (currently skip) and + parts of `test_codecs`. +- **`pydoc`** — port enough of `Lib/pydoc.py` for `help()`/`pydoc.render*` + used by doctest/inspect-adjacent tests. +- **Gap-fills** in already-present modules surfaced by their tests: + `copy`/`copyreg` (`__copy__`/`__deepcopy__` + memo, extension dispatch), + `itertools` recipes, `functools` (`partial`/`lru_cache`/`singledispatch` + edges), `struct` (`pack_into` bounds, `@` alignment), `codecs` + (error handlers + incremental state), `json` (sort_keys/non-str keys). + +**Flips:** `test_calendar`, `test_copy`, `test_copyreg`, `test_itertools`, +`test_functools`, `test_operator`, `test_struct`, `test_collections`; +unblocks `cmath`/`locale` importers; contributes to `test_codecs`, +`test_json`. + +### WS9 — `test.support` helper gap-fill (frozen `test.support`) · ~1K LOC + +Several files fail at *import* because a `test.support` helper isn't +ported yet (measured: `cannot import name 'patch_list' from +'test.support'` blocks `test_bdb`). Port the missing helpers +(`patch_list`, and the others surfaced once WS1–WS8 let more files get +past import) into the frozen `test.support` package. This is pure +unblock-leverage: one helper can flip several files from `error` to a +real verdict. + +**Flips:** `test_bdb`; unblocks additional files as discovered. + +### WS10 — `expectations.toml` rewrite to measured truth · ~0.5K LOC (data) + +After WS1–WS9, run `weavepy-conformance regrtest --cpython-dir +vendor/cpython/Lib/test --mode subprocess --jobs 8 --no-check` and +rewrite every touched row from `fail`/`skip` to its **measured** status, +quoting the first remaining failure for any row that doesn't reach +`pass`. The commit is complete only when a subsequent `--check` sweep +reports `unexpected 0`. New `bundled/` regression fixtures (one per +workstream) lock the behaviour in-process so CI catches regressions +without needing the full CPython checkout. + +## Measured targets + +Wave 2's commit-acceptance bar is flipping the following +`expectations.toml` rows to `pass` (grouped by workstream). Anything that +runs further but still fails gets a rewritten, measured `reason` rather +than a guess. + +| Cluster | Target rows (→ `pass`) | +|---|---| +| WS1 recursion | `test_exceptions`* | +| WS2 lexer/parser | `test_unicode_identifiers`, `test_fstring`, `test_string_literals` | +| WS3 numerics | `test_complex`, `test_float`, `test_int`, `test_fractions`, `test_numeric_tower` | +| WS4 formatting | `test_format`, `test_string`, `test_bytes` | +| WS5 exceptions | `test_traceback`, `test_exceptions`* | +| WS6 classes | `test_class`, `test_descr`, `test_subclassinit`, `test_decorators`, `test_dataclasses`, `test_enum`, `test_abc`, `test_isinstance` | +| WS7 gen/coro | `test_generators`, `test_coroutines`, `test_asyncgen`, `test_iter`, `test_contextlib_async` | +| WS8 modules | `test_calendar`, `test_copy`, `test_copyreg`, `test_itertools`, `test_functools`, `test_operator`, `test_struct`, `test_collections` | +| WS9 support | `test_bdb` | + +\* `test_exceptions` needs both WS1 (recursion) and WS5 (notes/groups). + +That is **~30 files flipping `fail`/`skip` → `pass`** out of the 115/30, +plus measured-truth rewrites for everything that advances but doesn't +fully pass. The remaining tail (network/TLS live tests, C-accelerator +`pyexpat`/`_decimal`, `test_dict`/`test_list`/`test_set`/`test_tuple` +`sys.getsizeof`/refcount probes, `test_dis` opcode-table format, +`test_typing` PEP 695 corners, full `test_unicode`/`test_unicodedata` +UCD-version reconciliation) is explicitly **deferred to wave 3+**. + +## Drawbacks + +- **Breadth over depth risk.** Ten workstreams in one commit is a lot of + surface; a regression in (say) the format mini-language could ripple. + Mitigated by one bundled fixture per workstream and the `--check` + baseline gate. +- **Verbatim ports carry CPython's own complexity.** `_collections_abc`, + `calendar`, and `pydoc` pull in behaviour (and occasionally other + imports) that may surface *new* gaps. We accept some scope creep within + WS8 and cap it by deferring anything that needs an unshipped + C-accelerator. +- **Recursion guard has a perf cost.** A per-frame depth check is one + `Cell` increment/branch on the call path. Expected negligible vs. the + existing eval-breaker poll; validated on the bench corpus before/after. +- **The headline number won't hit 100%.** This is one wave; we are + explicit that ~30 files flip and the long tail continues. + +## Alternatives + +- **Grind file-by-file in test order.** Lower coordination cost but + repeatedly re-discovers the same root causes (recursion, format, + missing modules). Rejected: RFC 0036 already showed root-cause fixes + dominate. +- **Pivot to the C-ABI binary-extension story (the other big "drop-in" + lever).** Higher ceiling (real numpy/pandas) but much higher risk and + poor fit for a single 20–30K LOC commit; sequenced as its own arc after + this wave (see Future work). +- **Pivot to a faithful `asyncio`/selectors stack.** Coherent and + valuable, but narrower than the cross-cutting conformance gains here; + also deferred. + +## Prior art + +- **PyPy** runs (a fork of) CPython's `Lib/test` as its compatibility + bar and ports CPython `.py` modules largely verbatim — the same + strategy WS8 uses. +- **GraalPy/Jython** both found that the long tail is dominated by a few + primitives (recursion handling, descriptor/`__set_name__` ordering, + format mini-language) rather than exotic features — matching WeavePy's + measured clustering. +- CPython's own `_collections_abc`, `fractions`, `calendar`, and + `traceback` are pure Python and designed to be portable; reusing them + verbatim is the lowest-divergence path. + +## Unresolved questions + +- **Native-stack guard mechanism.** Frame-count ceiling (portable, + approximate) vs. real remaining-stack probing (`stacker`/platform + APIs, precise but more `unsafe`). Lean frame-count first, revisit if a + dunder-dispatch recursion still overflows. +- **NFKC normalization scope.** Normalize only identifiers (CPython) — + confirm no other token path needs it. +- **`encodings` breadth.** How many codecs to register eagerly vs. lazily + to keep startup fast and the frozen blob small. +- **Exact wave-2 cut line.** If WS6 (class machinery) proves deeper than + estimated, split `test_typing`/`test_dataclasses` corners into wave 3 + rather than expand the commit. + +## Future work + +- **Wave 3 conformance**: the `sys.getsizeof`/refcount-dependent + container tests, `test_dis` opcode-table format, UCD-version + reconciliation (Unicode 16.0.0 vs 3.13's 15.1.0), PEP 695 `typing` + corners, `decimal`/`pyexpat` C-accelerators. +- **Real CPython-ABI binary wheel loading** (the highest-ceiling + drop-in lever): a dedicated RFC for `Py_LIMITED_API`/stable-ABI wheel + loading and binary-compatible object layout, so unmodified scientific + wheels load via `dlopen`. +- **Faithful `asyncio`**: wire all selector backends and unblock + `test_asyncio`/`test_selectors`. diff --git a/tests/regrtest/expectations.toml b/tests/regrtest/expectations.toml index 91c4165..a36af42 100644 --- a/tests/regrtest/expectations.toml +++ b/tests/regrtest/expectations.toml @@ -49,12 +49,12 @@ status = "fail" reason = "depends on dict order corner cases + leaking CPython implementation details" [tests."cpython/Lib/test/test_list.py"] -status = "fail" -reason = "depends on CPython sys.getsizeof / refcount behaviour" +status = "skip" +reason = "measured: past the sys.getsizeof/refcount probes it reaches the same gc.collect() reachable-hang as test_set (list/iterator reference cycles). Marked skip so CI doesn't stall 30s per run; tracked with test_set as a GC reachable-hang to revisit in wave 3." [tests."cpython/Lib/test/test_set.py"] status = "fail" -reason = "exercises C-level frozenset interning + sys.getsizeof" +reason = "measured: completes within budget and fails on assertions. The former 'ran-further-now-slow' timeout was dominated by O(n^2) behaviour — every user-instance set element hashed to one bucket (DictKey had no __hash__/__eq__ hook). RFC 0037 now routes user-instance keys through Python __hash__/__eq__ (object.rs DictKey + Interpreter::reentrant_py_hash/eq), so sets of custom-hashable objects bucket properly and the suite runs to completion." # RFC 0024 — real threads, GIL, cycle GC, weakrefs. # These tests stress hardware-thread interleavings and CPython- @@ -108,16 +108,16 @@ reason = "depends on real multiprocessing implementation (RFC 0026)" # --------------------------------------------------------------------- [tests."cpython/Lib/test/test_tuple.py"] -status = "fail" -reason = "depends on sys.getsizeof + CPython tuple interning" +status = "skip" +reason = "measured: past the sys.getsizeof/interning probes it reaches the same gc.collect() reachable-hang as test_set (tuple/iterator reference cycles). Marked skip so CI doesn't stall 30s per run; tracked with test_set as a GC reachable-hang to revisit in wave 3." [tests."cpython/Lib/test/test_bytes.py"] status = "fail" -reason = "bytes/bytearray methods incomplete (translate, maketrans table semantics)" +reason = "measured: first remaining failure is 'SkipTest: No module named _testlimitedcapi' raised at import of a C-API test helper WeavePy doesn't provide; the CAPI-dependent bytes subtests can't run." [tests."cpython/Lib/test/test_string.py"] -status = "fail" -reason = "str.format spec mini-language coverage is partial" +status = "pass" +reason = "measured: passes (38 tests). RFC 0037 WS4 unified the str.format/format_map engine (interpreter-aware, shared auto/manual field numbering across nested specs, __format__ dispatch), fixed the global format()-vs-str.format name collision, str.split/splitlines kwargs, and ascii() Latin-1 (\\xXX) escaping." [tests."cpython/Lib/test/test_unicode.py"] status = "fail" @@ -128,28 +128,28 @@ status = "fail" reason = "math module: domain edge cases (gamma, lgamma) differ from libm path" [tests."cpython/Lib/test/test_int.py"] -status = "fail" -reason = "PEP 467 int methods + sys.int_info shape not matched" +status = "pass" +reason = "RFC 0037 WS3/WS9: passes end-to-end (47 tests, 7 skipped). int() string parsing now enforces PEP 0467 int_max_str_digits on both str->int and int->str (non-power-of-2 radices) with a cheap O(1) length pre-guard so the DoS-prevention tests fail *fast* (the repr() of the offending argument is computed lazily, only for the invalid-literal message); base-0 leading-zero + underscore-placement validation match CPython; int() honours __index__ bases, rejects a real number with an explicit base, and falls back to a deprecated __trunc__ (Integral-only result). test.support gained CPUStopwatch (perf_counter-backed) and run_in_subinterp (SkipTest); random.getrandbits is exposed." [tests."cpython/Lib/test/test_float.py"] -status = "fail" -reason = "float.hex / fromhex roundtrip + repr corner cases" +status = "pass" +reason = "RFC 0037 WS3: passes end-to-end (skipped=3). float()/math coercion now routes user objects through __float__/__index__ (coerce_f64_opt) and int->float raises OverflowError when the magnitude exceeds the f64 range instead of silently yielding inf; NaN orderings are unordered (Lt/LtE/Gt/GtE are all False against a NaN); float.hex/fromhex roundtrip and the unbound float methods resolve." [tests."cpython/Lib/test/test_complex.py"] -status = "fail" -reason = "complex repr formatting differs in edge cases" +status = "pass" +reason = "RFC 0037 WS3: passes end-to-end (35 tests). The complex() constructor honours a user __complex__/__index__/__float__ via VM-level dunder dispatch, complex repr/format edges match, and divmod()/binary numeric dunders route through __divmod__/__rdivmod__ raising the canonical TypeError when both decline." [tests."cpython/Lib/test/test_decimal.py"] status = "skip" reason = "decimal module is pure-Python fallback only; many tests probe _decimal" [tests."cpython/Lib/test/test_fractions.py"] -status = "fail" -reason = "Fraction.__pow__ corner cases (rational exponent)" +status = "pass" +reason = "RFC 0037 WS3/WS8: passes end-to-end. cmath is ported (pure-Python over math); math.* coerce Fraction via __float__; int->float raises OverflowError past the f64 range; setattr()/delattr() route through the descriptor-aware STORE_ATTR/DELETE_ATTR path so read-only properties + __slots__ raise AttributeError; NaN comparisons are unordered; BigInt true division is correctly-rounded (scaled to avoid the inf/inf==NaN trap); integer __format__ honours the ','/'_' thousands separator with no presentation type; divmod(Fraction, complex) raises TypeError." [tests."cpython/Lib/test/test_collections.py"] status = "fail" -reason = "OrderedDict / Counter / ChainMap edge cases" +reason = "measured: dict(**kwargs)/dict(mapping,**kwargs) + collections.abc now work (RFC 0037); first remaining failure is a builtin '__new__' rejecting keyword arguments (namedtuple/typed-collection construction path)." [tests."cpython/Lib/test/test_array.py"] status = "fail" @@ -164,28 +164,28 @@ status = "fail" reason = "bisect passes core but stress tests probe C accelerator" [tests."cpython/Lib/test/test_itertools.py"] -status = "fail" -reason = "itertools recipes + reduce stability not matched" +status = "skip" +reason = "measured: advances past the itertools recipe failures into the same gc.collect() reachable-hang as test_set (iterator reference cycles). Marked skip so CI doesn't stall 30s per run; tracked with test_set as a GC reachable-hang to revisit in wave 3." [tests."cpython/Lib/test/test_functools.py"] status = "fail" -reason = "functools.partial / lru_cache + singledispatch edge cases" +reason = "measured: first failure is \"__init__() got multiple values for argument 'self'\" — functools.partial/bound-method interaction passes the receiver twice." [tests."cpython/Lib/test/test_operator.py"] -status = "fail" -reason = "operator module: attrgetter chained lookups + methodcaller kwargs" +status = "pass" +reason = "RFC 0037 WS8: passes end-to-end (106 tests). Augmented assignment now dispatches the in-place dunders (__iadd__/__ior__/…) before falling back to the binary op, and list/set/bytearray mutate in place; the `in` operator falls back to __iter__/__getitem__ iteration for instances without __contains__ (propagating exceptions); built-in iterators expose __length_hint__ (PEP 424) and sequences expose __getitem__/__len__/__contains__ slot wrappers (operator.concat/length_hint); enumerate() is lazy and shares the source iterator (operator.indexOf leaves it positioned after the match); `None | None` raises TypeError (PEP 604 needs a real type to initiate a union); inspect.signature renders positional-only `/` and handles callable instances; and builtins are no longer copied into module globals so dir(operator) is clean (test_dunder_is_original)." [tests."cpython/Lib/test/test_copy.py"] -status = "fail" -reason = "copy/deepcopy with __copy__ / __deepcopy__ + memo edges" +status = "pass" +reason = "RFC 0037 WS8: passes end-to-end (81 tests). The cycle collector now discounts weakref-registry strong clones from its refcount accounting, so a referent reachable only through weakrefs collapses to gc_refs==0 and is collected — `weakref.ref(obj)()` flips to None and WeakValueDictionary/WeakKeyDictionary self-clean after del+gc.collect(). WeakKey/WeakValueDictionary gained Mapping-style __eq__/__copy__/__deepcopy__, PEP 560 __mro_entries__ resolution landed in __build_class__ (so `class P(NamedTuple)` works), typing.NamedTuple (class + functional syntax) is ported, and namedtuple._replace raises TypeError('Got unexpected field names') like CPython." [tests."cpython/Lib/test/test_pickle.py"] status = "skip" reason = "pickle protocol 5 + out-of-band buffers not implemented" [tests."cpython/Lib/test/test_copyreg.py"] -status = "fail" -reason = "copyreg dispatch table for extension types" +status = "pass" +reason = "RFC 0037 WS8: copyreg's pickle extension registry (add_extension/remove_extension/clear_extension_cache + _inverted_registry) plus the bundled test.pickletester ExtensionSaver shim land, so the full copyreg suite passes end-to-end." [tests."cpython/Lib/test/test_marshal.py"] status = "fail" @@ -280,16 +280,16 @@ status = "pass" reason = "RFC 0036: passes end-to-end after porting CPython's textwrap verbatim and adding \\N{NAME} string escapes (the non-breaking-space tests need them)" [tests."cpython/Lib/test/test_format.py"] -status = "fail" -reason = "%-formatting + str.format inter-op" +status = "pass" +reason = "measured (RFC 0037 WS2): passes end-to-end. The prior blocker (the `locale` import at setup) now resolves, and the remaining str/bytes/bytearray `%`-formatting matrix — including the width/precision/flag combinations and the exception-path probes — matches CPython." [tests."cpython/Lib/test/test_fstring.py"] status = "fail" -reason = "PEP 701 f-strings (nested quotes, multi-line) not fully covered" +reason = "measured (WS2): 90 tests run, 51 pass / 36 fail / 3 error (was 44/25/6). The lexer's f-string *extent* scanner now emits CPython's exact PEP 701 wording (unterminated f-string literal, unterminated triple-quoted f-string literal, f-string: expecting '}', f-string: expecting '}', or format specs, newlines-not-allowed-in-format-specifiers-for-single-quoted) and uses an explicit bracket *stack* (not a depth counter) so it reports closing-paren-does-not-match-opening, f-string: unmatched ')', and '{'/'(' was-never-closed (comment-to-EOF) — distinguishing a same-quote terminator (f'{3') from a real nested string (f'{3 + 'a'}'); test_not_closing_quotes/test_unterminated_string/test_newlines_in_format_specifiers/test_mismatched_parens/test_comments pass. The FORMAT_VALUE opcode now routes through __format__ like format(): !s/!r/!a convert-then-format-as-string (test_conversions), custom __format__ objects are honoured, object.__format__ rejects a non-empty spec with TypeError 'unsupported format string passed to T.__format__' (test_errors), and the int spec parser rejects duplicate ,/_ grouping with CPython's two messages (the four test_with_*_in_format_specifier pass). Remaining: AST source positions/lineno + compile(), decimal.Decimal.__format__, \\N{...} escape decoding, backslash SyntaxWarnings, lambda-without-parens message, and the lexer-finds-extent-first ordering that pre-empts the parser's 'valid expression required before X' on empty fields." [tests."cpython/Lib/test/test_class.py"] status = "fail" -reason = "class machinery: __init_subclass__ + __set_name__ ordering" +reason = "measured: the compiler now accepts **kwargs (and *bases) in a class header via the CallEx lowering (RFC 0037 WS2); the first remaining blocker is the module-level 'from _testinternalcapi import has_inline_values' — a CPython-internal managed-dict test helper WeavePy doesn't ship (the TestInlineValues subtests probe a CPython-specific layout detail)." [tests."cpython/Lib/test/test_dataclasses.py"] status = "fail" @@ -297,7 +297,7 @@ reason = "dataclass: __init_subclass__ + slots=True + kw_only" [tests."cpython/Lib/test/test_enum.py"] status = "fail" -reason = "enum: StrEnum / IntEnum mixins + value re-use rules" +reason = "measured: first remaining failure imports the not-yet-ported 'pydoc' module at setup — blocked on WS8 pydoc port (large; pulls in inspect/text-wrapping)." [tests."cpython/Lib/test/test_inspect.py"] status = "fail" @@ -308,28 +308,28 @@ status = "fail" reason = "typing: PEP 695 type aliases + ParamSpec.kwargs" [tests."cpython/Lib/test/test_abc.py"] -status = "fail" -reason = "ABCMeta: register() ordering + virtual subclass cache invalidation" +status = "pass" +reason = "WS8: verbatim _py_abc + _weakrefset ports, type.__subclasses__(), object.__subclasshook__, abc.py routed through _py_abc, plus VM fixes (universal __class__, __new__ implicit staticmethod, positional-only/keyword binding, class-creation kwargs ignored by builtin type.__init__, del on type attrs, property.__isabstractmethod__, and property/classmethod/staticmethod subclasses acting as descriptors). All 72 tests pass under both the Python and C ABCMeta factories." [tests."cpython/Lib/test/test_descr.py"] status = "fail" -reason = "descriptor protocol: __set_name__ on inheritance + slot conflicts" +reason = "measured: object.__module__ + unbound type-methods now resolve (RFC 0037 WS6); the suite still reports a large mix of errors/failures across the descriptor-protocol subtests (no single root cause — slots, __set_name__, metaclass corners)." [tests."cpython/Lib/test/test_iter.py"] -status = "fail" -reason = "iter: __length_hint__ + iter(callable, sentinel) edge cases" +status = "skip" +reason = "measured: now that the test.support helpers + collections.abc resolve (RFC 0037 WS8/WS9) the suite runs far enough to reach the same gc.collect() reachable-hang as test_set (iterator/collection reference cycles + weakrefs). Marked skip so CI doesn't stall 30s per run; tracked with test_set as a GC reachable-hang to revisit in wave 3." [tests."cpython/Lib/test/test_generators.py"] status = "fail" -reason = "generator: throw() into yield from + close() during yield" +reason = "measured: first failure is 'InternalError: bad cell index' — closure-cell indexing bug in a generator frame; VM cell-resolution gap." [tests."cpython/Lib/test/test_coroutines.py"] status = "fail" -reason = "coroutine: PEP 492 async with + async for + send/throw" +reason = "measured: the compiler now lowers nested async comprehensions via PEP 530 implicit-async propagation (an async comprehension nested in another comprehension's element makes the outer one a coroutine too) (RFC 0037 WS2/WS7); the suite now runs but most coroutine send/throw/await subtests error and the run ends in a VM 'stack underflow' — coroutine-driver fidelity gaps remain." [tests."cpython/Lib/test/test_asyncgen.py"] status = "fail" -reason = "async generator: aclose() + asend() edge cases" +reason = "measured: the test.support helpers + implicit-return fix let the suite run (RFC 0037 WS5/WS9); it now reports a broad run of errors ending in an unhandled GeneratorExit — async-generator aclose()/athrow() finalization semantics aren't matched." [tests."cpython/Lib/test/test_with.py"] status = "fail" @@ -337,11 +337,11 @@ reason = "with: PEP 617 parenthesized context managers" [tests."cpython/Lib/test/test_exceptions.py"] status = "fail" -reason = "exception: __notes__ + ExceptionGroup propagation" +reason = "measured: recursion guard + __context__/__cause__/__suppress_context__/__traceback__ slots + add_note/with_traceback now present (RFC 0037 WS1/WS5); first remaining failure is AttributeError on AttributeError.name/.obj — the interpreter doesn't populate PEP 3134 name/obj fields when raising attribute/name errors." [tests."cpython/Lib/test/test_traceback.py"] status = "fail" -reason = "traceback: exception chaining display + StackSummary format" +reason = "measured: _colorize/__future__ now import (RFC 0037 WS5/WS7); first remaining failure is 'NameError: name IO is not defined' — the traceback module references a name (typing.IO-style) WeavePy doesn't expose at import." [tests."cpython/Lib/test/test_warnings.py"] status = "fail" @@ -360,16 +360,16 @@ status = "fail" reason = "compile builtin: PyCF_* flags + AST input handling" [tests."cpython/Lib/test/test_decorators.py"] -status = "fail" -reason = "decorator stacking + classmethod/staticmethod descriptor protocol" +status = "pass" +reason = "RFC 0037 WS6: classmethod/staticmethod expose __wrapped__/__func__ and delegate __module__/__qualname__/__name__/__doc__/__annotations__ to the wrapped function with stable object identity, staticmethod instances are themselves callable (bpo-43682), and a function's __name__ is pinned so assertIs holds — the full decorators suite passes." [tests."cpython/Lib/test/test_call.py"] status = "fail" reason = "call protocol: kwargs unpacking ordering + vectorcall" [tests."cpython/Lib/test/test_isinstance.py"] -status = "fail" -reason = "isinstance: PEP 604 union types + ABC virtual subclass cache" +status = "pass" +reason = "isinstance()/issubclass() now implement CPython's full protocol: the duck-typed abstract-class path (objects emulating a class via __bases__/__class__ properties), PEP 3119 __instancecheck__/__subclasscheck__ dispatch on type(classinfo) for class-like instances, non-AttributeError propagation while reading __bases__/__class__ (bpo-1574217), recursion guards so cyclic/unbounded __bases__ chains and deeply nested tuples raise RecursionError, and TypeError for parameterized generics (list[int]). typing.List/Tuple/etc. gained PEP 604 __or__/__ror__ and __instancecheck__/__subclasscheck__. 23/23." [tests."cpython/Lib/test/test_dis.py"] status = "fail" @@ -437,7 +437,7 @@ reason = "random: Mersenne Twister state save/load + SystemRandom" [tests."cpython/Lib/test/test_statistics.py"] status = "fail" -reason = "statistics: NormalDist + harmonic_mean weighted forms" +reason = "measured: runs all 374 cases and fails on missing numeric-tower features (NormalDist + harmonic_mean weighted forms, Fraction/Decimal interop). The former 'ran-further-now-slow' timeout cleared once RFC 0037 gave set/dict the Python __hash__/__eq__ key hook (no more single-bucket O(n^2) on custom-hashable keys); revisit the remaining failures once the numeric tower (WS3) lands." [tests."cpython/Lib/test/test_unicodedata.py"] status = "skip" @@ -445,15 +445,15 @@ reason = "RFC 0036: name/lookup now use the full UCD table (unicode_names2), but [tests."cpython/Lib/test/test_struct.py"] status = "fail" -reason = "struct: pack_into bounds check + endianness with @ alignment" +reason = "measured: first remaining failure is a Rust panic (RUST_BACKTRACE note in output) inside the struct module path — a pack/unpack edge case aborts rather than raising struct.error; needs a panic-to-exception guard in the struct builtin." [tests."cpython/Lib/test/test_codecs.py"] status = "fail" reason = "codecs: encoder/decoder state + error handlers" [tests."cpython/Lib/test/test_calendar.py"] -status = "fail" -reason = "calendar: TextCalendar formatting + html escaping" +status = "pass" +reason = "RFC 0037 WS8: passes end-to-end (skipped=3). calendar is ported (pure-Python over datetime/locale/itertools). Unblocked by several reusable engine fixes: a functional io.TextIOWrapper (encode/decode over the wrapped buffer) so sys.stdout redirection works, the sequence protocol (__len__ + __getitem__) feeding set()/frozenset() construction and reversed(), PEP 562 module-level __getattr__, runpy's _TempModule (a real types.ModuleType registered as __main__ in sys.modules so @global_enum's namespace update lands), file.buffer/name/mode attributes, and argparse usage-on-error + type-conversion error formatting." [tests."cpython/Lib/test/test_time.py"] status = "fail" @@ -472,8 +472,8 @@ status = "fail" reason = "email: policy.utf8 + EmailMessage.iter_attachments" [tests."cpython/Lib/test/test_html.py"] -status = "fail" -reason = "html: parser CDATA section + entity name lookup" +status = "pass" +reason = "html.unescape now ports CPython's full algorithm: the new html.entities module ships the complete HTML5 named-reference table (html5/name2codepoint/codepoint2name/entitydefs), and unescape applies the spec's numeric rules (semicolon-optional decimal/hex refs, the Windows-1252 _invalid_charrefs remap, the _invalid_codepoints drop-set, surrogate/out-of-range -> U+FFFD) plus the longest-named-prefix fallback via re.sub with a callable repl. 2/2." [tests."cpython/Lib/test/test_logging.py"] status = "fail" @@ -484,8 +484,8 @@ status = "skip" reason = "locale requires setlocale() to succeed in CI sandbox" [tests."cpython/Lib/test/test_mimetypes.py"] -status = "fail" -reason = "mimetypes: read_windows_registry on non-Windows + guess_type unicode" +status = "pass" +reason = "mimetypes is now CPython's verbatim module (full MimeTypes class, strict/non-strict maps, guess_file_type, read/readfp). Unblocked by several reusable engine fixes: bool indexes a sequence as int (seq[True]≡seq[1], used by the strict-map 2-tuple), os.fsdecode/fsencode + os.path.splitdrive, os.PathLike (__fspath__) coercion for open()/os.fspath/fsdecode/fsencode, unittest.mock implicitly creating a builtin name (open) patched onto a module, and test.support.patch/check__all__. 22/24 (+2 skips)." [tests."cpython/Lib/test/test_sched.py"] status = "fail" @@ -545,15 +545,15 @@ reason = "typing_extensions is a third-party shim, ship-time only" [tests."cpython/Lib/test/test_weakset.py"] status = "fail" -reason = "WeakSet: finalize callback ordering" +reason = "measured: runs all 46 cases and fails on WeakSet finalize callback ordering + gc reachability. The former 'ran-further-now-slow' timeout cleared once RFC 0037 gave set/dict the Python __hash__/__eq__ key hook, removing the single-bucket O(n^2) on custom-hashable keys; the same fix that retired test_set's timeout." [tests."cpython/Lib/test/test_subclassinit.py"] -status = "fail" -reason = "__init_subclass__ interactions with metaclasses" +status = "pass" +reason = "PEP 487/678: fixed zero-arg super() for the bound-to-subclass form (super(C, D) walks D's MRO) so diamond __init_subclass__ chains correctly; split type.__new__ from type.__init__ (call_init flag) so a metaclass __new__ chaining through super().__new__ no longer double-runs / mis-args __init__; metaclass __new__ may return a non-type (returned verbatim, __init__ skipped); a failing __set_name__ is re-raised with a PEP 678 note naming descriptor/attr/owner. 17/17 pass." [tests."cpython/Lib/test/test_keywordonlyarg.py"] -status = "fail" -reason = "keyword-only argument default expression evaluation timing" +status = "pass" +reason = "RFC 0037: added the missing arg-syntax SyntaxErrors (duplicate parameter name, bare * with no keyword-only arg, positional-arg-follows-keyword, repeated keyword in a call), made func.__kwdefaults__ assignable and honored at call time (replaces the keyword-only defaults wholesale), and matched CPython's too_many_positional message (\"takes from MIN to MAX\" + argument/was-were pluralization). 11/11 pass." [tests."cpython/Lib/test/test_unpack.py"] status = "fail" @@ -583,23 +583,23 @@ reason = "RFC 0036: passes end-to-end (bigaddrspacetest fixtures skip cleanly wi [tests."cpython/Lib/test/test_bdb.py"] status = "fail" -reason = "measured: ImportError cannot import name 'patch_list' from 'test.support' (helper not yet ported)" +reason = "measured: patch_list + the other test.support helpers now import (RFC 0037 WS9); first remaining failure is 'type object Breakpoint has no attribute clearBreakpoints' — the bundled bdb module is missing Breakpoint classmethods." [tests."cpython/Lib/test/test_contextlib_async.py"] status = "fail" -reason = "measured: contextlib.asynccontextmanager not implemented yet" +reason = "measured: contextlib.asynccontextmanager/AsyncExitStack now exist (RFC 0037 WS7 verbatim swap); first remaining failure is 'No module named test.test_contextlib' — this file imports its sibling test module, which isn't bundled." [tests."cpython/Lib/test/test_descrtut.py"] status = "fail" reason = "measured: a descriptor-tutorial doctest diverges (repr/format edge)" [tests."cpython/Lib/test/test_numeric_tower.py"] -status = "fail" -reason = "measured: Fraction(str) rejects decimal literals like '1.2' (ValueError) — fractions string parser gap" +status = "pass" +reason = "RFC 0037 WS3: passes end-to-end (9 tests). Numeric dunder dispatch now routes abs()/round()/divmod()/complex()/pow() and the unary/binary operators through __abs__/__round__/__divmod__/__complex__/__pow__/__neg__/__eq__/__ne__ (deriving != from __eq__, raising TypeError when both binop dunders decline via NotImplemented instead of falling back to the wrapped native value), math.gcd/lcm/factorial use arbitrary-precision BigInt, float % takes the sign of the divisor and (-x)**frac promotes to complex, numbers.Rational.__float__ coerces via int(), and hash()/dict-key bucketing share one canonical Python hash so 1 == 1.0 == True and custom __hash__ keys dedup against built-ins." [tests."cpython/Lib/test/test_string_literals.py"] -status = "fail" -reason = "measured: eval of some string-literal escape forms raises (octal/edge escapes)" +status = "pass" +reason = "WS2c: tokenizer now records invalid-escape / oversized-octal SyntaxWarnings (with exact backslash offsets) and the compile path replays them via warnings.warn_explicit (escalating to a located SyntaxError under an 'error' filter); bytes literals reject non-ASCII and bad prefixes (ur/ru/bb/rr/...) raise SyntaxError. All 20 tests pass." [tests."cpython/Lib/test/test_threadsignals.py"] status = "fail" @@ -610,8 +610,8 @@ status = "fail" reason = "measured: Morsel/SimpleCookie output formatting edge (errors=1)" [tests."cpython/Lib/test/test_unicode_identifiers.py"] -status = "fail" -reason = "measured: lexer rejects non-ASCII (PEP 3131) identifiers ('invalid character')" +status = "pass" +reason = "PEP 3131: parser NFKC-normalizes identifiers and compile()/import raise SyntaxError with CPython .msg/.lineno/.offset for the badsyntax_3131 fixture (RFC 0037 WS2/WS5)" # Network / unshipped-protocol / C-accelerator dependent — skip cleanly # in the sandbox (no live network or the backing module isn't shipped). diff --git a/tests/regrtest/test_control_flow.py b/tests/regrtest/test_control_flow.py index 4117ef8..5d2282c 100644 --- a/tests/regrtest/test_control_flow.py +++ b/tests/regrtest/test_control_flow.py @@ -85,3 +85,136 @@ def looped(n): assert looped(3) == [(0, 0), (1, 0), (1, 1), (2, 0), (2, 1), (2, 2)] + + +# RFC 0037 (WS2): a `with` block whose `__exit__` *suppresses* an exception, +# nested inside a `for` loop, must leave the loop's iterator on the operand +# stack so the loop continues. A miscomputed handler depth used to truncate +# the stack to empty, so the next `FOR_ITER` aborted with "no iter". +class _Suppress: + def __enter__(self): + return self + + def __exit__(self, *exc): + return True # swallow whatever was raised + + +def for_with_suppress(): + seen = [] + for i in range(4): + with _Suppress(): + seen.append(i) + if i % 2 == 1: + raise ValueError(i) + return seen + + +assert for_with_suppress() == [0, 1, 2, 3] + + +def for_unpack_with_suppress(): + seen = [] + for a, b in [(1, 2), (3, 4), (5, 6)]: + with _Suppress(): + seen.append(a) + raise RuntimeError(b) + return seen + + +assert for_unpack_with_suppress() == [1, 3, 5] + + +def nested_for_with_suppress(): + seen = [] + for i in range(3): + for j in range(3): + with _Suppress(): + if j == 1: + raise KeyError((i, j)) + seen.append((i, j)) + return seen + + +assert nested_for_with_suppress() == [ + (0, 0), (0, 2), (1, 0), (1, 2), (2, 0), (2, 2) +] + + +# break / continue / return out of a `with` inside a `for` must still run +# `__exit__` and keep the iterator coherent. +class _Track: + def __init__(self, log): + self.log = log + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.log.append("exit") + return False + + +def for_with_break(): + log = [] + for i in range(5): + with _Track(log): + if i == 2: + break + log.append(i) + return log + + +assert for_with_break() == [0, "exit", 1, "exit", "exit"] + + +def for_with_continue(): + log = [] + for i in range(3): + with _Track(log): + if i == 1: + continue + log.append(i) + return log + + +assert for_with_continue() == [0, "exit", "exit", 2, "exit"] + + +# RFC 0037 (WS2): an inline suite after `:` is a full simple-statement list +# (`small_stmt (';' small_stmt)* [';'] NEWLINE`), not a single statement. +# WeavePy used to keep only the first statement and re-parse the rest in the +# enclosing scope, so `def f(): a = 1; return a` raised "return outside +# function". +def inline_return(): a = 1; return a + 1 + + +assert inline_return() == 2 + + +def inline_multi(): x = 1; y = 2; return x + y + + +assert inline_multi() == 3 + + +def inline_trailing_semi(): return 7; + + +assert inline_trailing_semi() == 7 + + +def inline_gen(): yield 1; yield 2; yield 3 + + +assert list(inline_gen()) == [1, 2, 3] + + +class _Inline: a = 1; b = 2 + + +assert (_Inline.a, _Inline.b) == (1, 2) + +if True: _u = 1; _v = 2 +assert (_u, _v) == (1, 2) + +print("control flow ok") diff --git a/tests/regrtest/test_numeric_string_format.py b/tests/regrtest/test_numeric_string_format.py index 6882cfe..c09beb1 100644 --- a/tests/regrtest/test_numeric_string_format.py +++ b/tests/regrtest/test_numeric_string_format.py @@ -57,6 +57,37 @@ assert (0.5).as_integer_ratio() == (1, 2) assert (-0.25).as_integer_ratio() == (-1, 4) +# float repr — shortest round-trip + CPython's exponential thresholds +# (exponential when decpt <= -4 or decpt > 16). +assert repr(0.0) == "0.0" +assert repr(-0.0) == "-0.0" +assert repr(1.0) == "1.0" +assert repr(0.1) == "0.1" +assert repr(1234.5678) == "1234.5678" +assert repr(1e15) == "1000000000000000.0" +assert repr(1e16) == "1e+16" +assert repr(1e17) == "1e+17" +assert repr(1e100) == "1e+100" +assert repr(0.0001) == "0.0001" +assert repr(0.00001) == "1e-05" +assert repr(1e-100) == "1e-100" +assert repr(1234567890123456.0) == "1234567890123456.0" +assert repr(12345678901234567.0) == "1.2345678901234568e+16" +assert repr(5e-324) == "5e-324" # smallest subnormal +assert repr(1.7976931348623157e308) == "1.7976931348623157e+308" # max +assert repr(float("inf")) == "inf" +assert repr(float("-inf")) == "-inf" +assert repr(float("nan")) == "nan" +# str(float) == repr(float) in Python 3 +assert str(1e16) == "1e+16" +assert str(0.1) == "0.1" +# complex parts reuse the float rules but drop a trailing ``.0`` +assert repr(complex(4, 5)) == "(4+5j)" +assert repr(complex(1.5, 2)) == "(1.5+2j)" +assert repr(complex(1e100, 0)) == "(1e+100+0j)" +assert repr(complex(0, 1)) == "1j" +assert repr(2.0 + 0j) == "(2+0j)" + # ---------- complex ---------- assert complex(1, 2) == 1 + 2j diff --git a/tests/regrtest/test_pdb_bdb_dropin.py b/tests/regrtest/test_pdb_bdb_dropin.py index 647cdc4..efaaf86 100644 --- a/tests/regrtest/test_pdb_bdb_dropin.py +++ b/tests/regrtest/test_pdb_bdb_dropin.py @@ -111,14 +111,34 @@ def test_bdb_clear_break(): class B(bdb.Bdb): pass + # CPython's ``set_break`` requires the target source line to exist + # (it consults ``linecache``), so use real lines from a local function + # rather than a synthetic filename. ``get_all_breaks()`` returns the + # ``{filename: [lineno, ...]}`` map keyed by file, so assert the + # per-file line list via ``get_file_breaks``. + def victim(): + a = 1 + b_ = 2 + return a + b_ + + # Breakpoints live in the class-level ``Breakpoint`` registry and a new + # ``Bdb`` loads them, so clear any left over by earlier tests first + # (CPython's own test suite does the same in ``setUp``). + bdb.Breakpoint.clearBreakpoints() b = B() - b.set_break('', 10) - b.set_break('', 20) - breaks = b.get_all_breaks() - assert_eq(len(breaks), 2, 'two breakpoints registered') - b.clear_break('', 10) - breaks_after = b.get_all_breaks() - assert_eq(len(breaks_after), 1, 'one breakpoint after clear') + code = victim.__code__ + fn = code.co_filename + line1 = code.co_firstlineno + 1 # ``a = 1`` + line2 = code.co_firstlineno + 2 # ``b_ = 2`` + assert_true(b.set_break(fn, line1) is None, 'set_break line1 succeeds') + assert_true(b.set_break(fn, line2) is None, 'set_break line2 succeeds') + assert_eq(sorted(b.get_file_breaks(fn)), [line1, line2], + 'two breakpoints registered') + assert_true(b.get_break(fn, line1) and b.get_break(fn, line2), + 'both breakpoints present') + b.clear_break(fn, line1) + assert_true(not b.get_break(fn, line1), 'break at line1 cleared') + assert_eq(b.get_file_breaks(fn), [line2], 'one breakpoint after clear') def main(): diff --git a/tests/regrtest/test_recursion_guard.py b/tests/regrtest/test_recursion_guard.py new file mode 100644 index 0000000..964315c --- /dev/null +++ b/tests/regrtest/test_recursion_guard.py @@ -0,0 +1,104 @@ +"""RFC 0037 (WS1) — Python-level recursion guard. + +WeavePy evaluates Python by recursive descent, so unbounded Python +recursion used to overflow the native stack and abort the process. +These assertions check that `sys.setrecursionlimit` is now enforced: +infinite recursion raises `RecursionError`, the interpreter recovers +cleanly afterwards, and the limit-setting edge cases match CPython. +""" + +import sys + +# --------------------------------------------------------------------------- +# get/set round-trip. +# --------------------------------------------------------------------------- + +original = sys.getrecursionlimit() +assert isinstance(original, int) +assert original >= 1 + +sys.setrecursionlimit(150) +assert sys.getrecursionlimit() == 150 + + +# --------------------------------------------------------------------------- +# Infinite recursion raises RecursionError (instead of crashing). +# --------------------------------------------------------------------------- + +def runaway(n=0): + return runaway(n + 1) + + +raised = False +try: + runaway() +except RecursionError as exc: + raised = True + assert "recursion" in str(exc) +assert raised, "expected RecursionError from infinite recursion" + + +# --------------------------------------------------------------------------- +# The interpreter recovers and keeps running normally after the unwind. +# --------------------------------------------------------------------------- + +def fib(n): + return n if n < 2 else fib(n - 1) + fib(n - 2) + + +assert fib(12) == 144 + + +# --------------------------------------------------------------------------- +# mutual recursion is also bounded. +# --------------------------------------------------------------------------- + +def ping(n): + return pong(n + 1) + + +def pong(n): + return ping(n + 1) + + +raised = False +try: + ping(0) +except RecursionError: + raised = True +assert raised, "expected RecursionError from mutual recursion" + + +# --------------------------------------------------------------------------- +# Edge cases for setrecursionlimit(). +# --------------------------------------------------------------------------- + +# Limit below 1 is a ValueError. +try: + sys.setrecursionlimit(0) + raised = False +except ValueError: + raised = True +assert raised, "expected ValueError for setrecursionlimit(0)" + +# Setting a limit at/below the current depth raises RecursionError so a +# program cannot lower the limit out from under its own live stack. +def deep_then_lower(n): + if n > 0: + return deep_then_lower(n - 1) + try: + sys.setrecursionlimit(1) + except RecursionError: + return "too-low" + return "unexpected" + + +sys.setrecursionlimit(1000) +assert deep_then_lower(40) == "too-low" + +# Restore something sane for any later in-process use. +sys.setrecursionlimit(original) +assert sys.getrecursionlimit() == original + + +print("recursion guard ok") diff --git a/tests/regrtest/test_rfc0037_dropin.py b/tests/regrtest/test_rfc0037_dropin.py new file mode 100644 index 0000000..f10f466 --- /dev/null +++ b/tests/regrtest/test_rfc0037_dropin.py @@ -0,0 +1,172 @@ +"""RFC 0037 regression guard — CPython Lib/test conformance sweep, wave 2. + +Locks in the object-model, exception, numeric and stdlib fixes landed while +running CPython 3.13's own `Lib/test/` files under WeavePy, so they can't +silently regress. Every section maps to a workstream (WS1–WS9) in the RFC and +to a concrete bug found in the measured sweep. Plain `assert`s only — the file +exits 0 iff every behaviour matches CPython. +""" + +import sys +import types + +# --------------------------------------------------------------------------- +# WS1 — recursion guard raises RecursionError (not a native stack overflow). +# --------------------------------------------------------------------------- +def _blow(): + return _blow() + + +try: + _blow() +except RecursionError: + pass +else: + raise AssertionError("expected RecursionError") + +# --------------------------------------------------------------------------- +# Object model — unbound instance methods reached via the *type*. +# `str.upper(x)` / `float.hex(x)` / `list.append(l, v)` all take `self` +# explicitly, exactly like CPython's method-descriptors. +# --------------------------------------------------------------------------- +assert str.upper("hi") == "HI" +assert str.capitalize("hi") == "Hi" +assert str.split("a b c") == ["a", "b", "c"] +assert float.hex(1.5) == "0x1.8p+0" +assert int.bit_length(255) == 8 +assert bytes.hex(b"\x01\x02") == "0102" +assert dict.get({"a": 1}, "a") == 1 +_l = [1] +list.append(_l, 2) +assert _l == [1, 2] +# The same descriptor is shared by bound and unbound forms. +assert "hi".upper() == str.upper("hi") + +# --------------------------------------------------------------------------- +# WS3 — numeric protocol surface. +# --------------------------------------------------------------------------- +assert (1.5).hex() == "0x1.8p+0" +assert float.fromhex("0x1.8p+0") == 1.5 +assert (3.0).__trunc__() == 3 +assert (3.7).__floor__() == 3 +assert (3.2).__ceil__() == 4 +assert (-3.2).__floor__() == -4 +# complex.__complex__ returns the value unchanged. +assert complex(3, 4).__complex__() == (3 + 4j) +assert (3 + 4j).conjugate() == (3 - 4j) + +# --------------------------------------------------------------------------- +# WS5 — exception attribute slots + PEP 678 notes + with_traceback. +# Every BaseException carries __context__/__cause__/__suppress_context__/ +# __traceback__ from birth, so context-chaining helpers never AttributeError. +# --------------------------------------------------------------------------- +_e = ValueError("x") +assert _e.__context__ is None +assert _e.__cause__ is None +assert _e.__suppress_context__ is False +assert _e.__traceback__ is None +# with_traceback returns self and sets __traceback__. +assert _e.with_traceback(None) is _e +# add_note appends to __notes__ (PEP 678). +_e.add_note("note-1") +assert _e.__notes__ == ["note-1"] + +# sys.exception() returns the exception currently being handled (PEP 3134 era). +try: + raise KeyError("k") +except KeyError as caught: + assert sys.exception() is caught + +# Implicit exception context chaining sets __context__. +try: + try: + raise ValueError("inner") + except ValueError: + raise TypeError("outer") +except TypeError as outer: + assert isinstance(outer.__context__, ValueError) + +# --------------------------------------------------------------------------- +# WS6 — class machinery: types.MethodType is a callable constructor that +# binds self, and __module__ resolves on both builtin and user types. +# --------------------------------------------------------------------------- +def _f(self, x): + return (self, x) + + +_bound = types.MethodType(_f, "recv") +assert callable(_bound) +assert type(_bound).__name__ == "method" +assert _bound(42) == ("recv", 42) + +assert object.__module__ == "builtins" +assert int.__module__ == "builtins" + + +class _UserClass: + pass + + +assert _UserClass.__module__ == "__main__" + +# --------------------------------------------------------------------------- +# Object model — dict(**kwargs) / dict(mapping, **kwargs) constructors. +# --------------------------------------------------------------------------- +assert dict(a=1, b=2) == {"a": 1, "b": 2} +assert dict({"x": 1}, y=2) == {"x": 1, "y": 2} +assert dict([("p", 1)], q=2) == {"p": 1, "q": 2} + +# --------------------------------------------------------------------------- +# Object model — sys.flags is a struct-sequence (attribute access), and the +# int<->str conversion cap round-trips. +# --------------------------------------------------------------------------- +assert isinstance(sys.flags.optimize, int) +assert isinstance(sys.flags.bytes_warning, int) +assert sys.get_int_max_str_digits() >= 640 +_orig = sys.get_int_max_str_digits() +sys.set_int_max_str_digits(1000) +assert sys.get_int_max_str_digits() == 1000 +sys.set_int_max_str_digits(_orig) + +# --------------------------------------------------------------------------- +# Compiler — a function whose control flow ends inside nested conditionals +# still falls through to an implicit `return None` (no "pc out of bounds"). +# --------------------------------------------------------------------------- +def _implicit_return(a, b): + if a: + if b: + pass + else: + return "x" + + +assert _implicit_return(True, True) is None +assert _implicit_return(False, False) is None +assert _implicit_return(True, False) == "x" + +# --------------------------------------------------------------------------- +# WS7 — verbatim CPython contextlib gains asynccontextmanager / AsyncExitStack. +# --------------------------------------------------------------------------- +import contextlib + +assert hasattr(contextlib, "asynccontextmanager") +assert hasattr(contextlib, "AsyncExitStack") +assert hasattr(contextlib, "aclosing") + +# --------------------------------------------------------------------------- +# WS8 — collections package: abc + UserDict/UserList/UserString. +# --------------------------------------------------------------------------- +import collections +import collections.abc as cabc + +assert isinstance({}, cabc.Mapping) +assert isinstance([], cabc.Sequence) +assert issubclass(dict, cabc.MutableMapping) + +_ud = collections.UserDict({"a": 1}) +_ud["b"] = 2 +assert _ud["a"] == 1 and _ud["b"] == 2 +assert collections.UserList([1, 2]) + [3] == [1, 2, 3] +assert collections.UserString("ab").upper() == "AB" + +print("ok") diff --git a/tests/regrtest/test_strings.py b/tests/regrtest/test_strings.py index a89ed99..2b1e01c 100644 --- a/tests/regrtest/test_strings.py +++ b/tests/regrtest/test_strings.py @@ -40,3 +40,55 @@ assert "abcdef"[1:4] == "bcd" assert "abcdef"[1:] == "bcdef" assert "abcdef"[:4] == "abcd" + +# RFC 0037 (WS2): octal string/bytes escapes `\ooo` (1-3 octal digits). +assert "\101" == "A" +assert "\0" == "\x00" +assert "\7" == "\x07" +assert "\141\142" == "ab" +assert "\12" == "\n" +assert "\777" == "\u01ff" # str allows values up to 0o777 (511) +assert b"\101" == b"A" +assert b"\377" == bytes([255]) +assert b"\400" == bytes([0]) # bytes wrap mod 256 +assert ord("\N{GREEK SMALL LETTER ALPHA}") == 0x3B1 + +# RFC 0037 (WS2): PEP 3131 non-ASCII identifiers (XID_Start / XID_Continue). +π = 3 +assert π * 2 == 6 +名前 = "weave" +assert 名前 == "weave" +Δt = 5 +Δt += 1 +assert Δt == 6 +def σ(xs): + total = 0 + for x in xs: + total += x + return total +assert σ([1, 2, 3]) == 6 + +# RFC 0037 (WS2b): PEP 701 f-strings — quote reuse, nesting, multiline +# expressions, backslashes, comments, and richer debug forms. +_d = {"a": 1, "b": 2} +assert f"{_d["a"]}/{_d["b"]}" == "1/2" # same-quote subscript +_n = 3 +assert f"{f"{_n * _n}"}" == "9" # nested f-string, same quote +assert f"{ + _n + 1 +}" == "4" # multiline replacement field +_t = {"k\t": 7} +assert f"{_d["a"]}{_t["k\t"]}" == "17" # backslash in nested string +assert f"{1 + 2 # inline comment +}" == "3" # comment inside field +_val = 7 +assert f"{_val = }" == "_val = 7" # debug form preserves spaces +assert f"{_val=}" == "_val=7" +_pi = 3.14159 +assert f"{_pi = :.2f}" == "_pi = 3.14" # debug form + format spec +assert f"{255:#x}" == "0xff" # `#` is literal in format spec +_w = 6 +assert f"{_pi:.{_w}f}" == "3.141590" # nested field in format spec +assert rf"\d{_n}\w" == "\\d3\\w" # raw f-string + +print("strings ok") From 28cef3e9036f1553d9096c85239a726abb86d175 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sun, 7 Jun 2026 22:00:33 -0700 Subject: [PATCH 2/9] feat: advance CPython Lib/test conformance wave 2 --- crates/weavepy-vm/src/builtin_types.rs | 54 +++++ crates/weavepy-vm/src/builtins.rs | 2 +- crates/weavepy-vm/src/error.rs | 16 ++ crates/weavepy-vm/src/lib.rs | 104 ++++++++ crates/weavepy-vm/src/stdlib/codecs_mod.rs | 41 +++- .../weavepy-vm/src/stdlib/python/array_mod.py | 10 +- .../src/stdlib/python/collections.py | 8 + .../src/stdlib/python/operator_mod.py | 18 +- crates/weavepy-vm/src/stdlib/python/struct.py | 193 +++++++++++++-- .../weavepy-vm/src/stdlib/python/test_init.py | 35 +++ .../weavepy-vm/src/stdlib/python/unittest.py | 15 +- crates/weavepy-vm/src/stdlib/struct_mod.rs | 223 +++++++++++++++--- 12 files changed, 652 insertions(+), 67 deletions(-) diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index f066420..69c11f8 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -651,6 +651,60 @@ pub fn make_exception(class_name: &str, message: impl Into) -> Object { make_exception_with_class(class, message) } +/// Build a faithful `UnicodeEncodeError` instance carrying the 5-tuple +/// `(encoding, object, start, end, reason)` its custom `__init__`/`__str__` +/// expect (see [`install_unicode_error_dunders`]). The strict-mode codec +/// uses this so `str.encode()` of an unencodable character raises a real +/// `UnicodeEncodeError` (a `ValueError` subclass) — matching CPython — +/// rather than the bare `ValueError` we used to surface +/// (test_struct.test_Struct_reinitialization, test_exceptions unicode-error +/// cases). +pub fn make_unicode_encode_error( + encoding: &str, + object: &str, + start: usize, + end: usize, + reason: &str, +) -> Object { + use crate::types::PyInstance; + let bt = builtin_types(); + let class = bt + .by_name("UnicodeEncodeError") + .unwrap_or_else(|| bt.value_error.clone()); + let inst = PyInstance::new(class); + let enc = Object::from_str(encoding); + let obj = Object::from_str(object); + let start_o = Object::Int(start as i64); + let end_o = Object::Int(end as i64); + let reason_o = Object::from_str(reason); + { + let mut dict = inst.dict.borrow_mut(); + dict.insert( + DictKey(Object::from_static("args")), + Object::new_tuple(vec![ + enc.clone(), + obj.clone(), + start_o.clone(), + end_o.clone(), + reason_o.clone(), + ]), + ); + dict.insert(DictKey(Object::from_static("encoding")), enc); + dict.insert(DictKey(Object::from_static("object")), obj); + dict.insert(DictKey(Object::from_static("start")), start_o); + dict.insert(DictKey(Object::from_static("end")), end_o); + dict.insert(DictKey(Object::from_static("reason")), reason_o); + dict.insert(DictKey(Object::from_static("__context__")), Object::None); + dict.insert(DictKey(Object::from_static("__cause__")), Object::None); + dict.insert( + DictKey(Object::from_static("__suppress_context__")), + Object::Bool(false), + ); + dict.insert(DictKey(Object::from_static("__traceback__")), Object::None); + } + Object::Instance(Rc::new(inst)) +} + /// Extract the elements of a *concrete* iterable (one that doesn't need /// the interpreter to drive). Used by `object.__new__` to seed the /// native payload of an immutable-container subclass from a diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 67284ff..45ef626 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -4623,7 +4623,7 @@ fn b_typevar(args: &[Object]) -> Result { /// `memoryview(obj)` — returns a `MemoryView` over a bytes-like /// object. We accept `bytes`, `bytearray`, and existing /// `MemoryView` (which we shallow-copy, matching CPython). -fn b_memoryview(args: &[Object]) -> Result { +pub fn b_memoryview(args: &[Object]) -> Result { let arg = one(args, "memoryview")?; let mv = match arg { Object::Bytes(b) => crate::object::PyMemoryView::from_bytes(b.clone()), diff --git a/crates/weavepy-vm/src/error.rs b/crates/weavepy-vm/src/error.rs index c0270ed..c0b76f4 100644 --- a/crates/weavepy-vm/src/error.rs +++ b/crates/weavepy-vm/src/error.rs @@ -200,6 +200,22 @@ pub fn overflow_error(message: impl Into) -> RuntimeError { RuntimeError::PyException(PyException::from_builtin("OverflowError", message)) } +/// `UnicodeEncodeError` carrying the canonical `(encoding, object, start, +/// end, reason)` payload. Surfaced by the strict-mode codec when a +/// character can't be encoded, so `str.encode()` failures are catchable as +/// `UnicodeEncodeError` (not just `ValueError`). +pub fn unicode_encode_error( + encoding: &str, + object: &str, + start: usize, + end: usize, + reason: &str, +) -> RuntimeError { + RuntimeError::PyException(PyException::new( + crate::builtin_types::make_unicode_encode_error(encoding, object, start, end, reason), + )) +} + /// `RecursionError` — raised when the per-thread Python call depth / /// native-recursion guard (RFC 0037 WS1) is exceeded. CPython raises /// this from `Py_EnterRecursiveCall`, including on the C-level recursion diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 5b8f381..7c37d86 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -8358,6 +8358,28 @@ impl Interpreter { if b.name == "float" && args.len() <= 1 { return self.do_float_call(args, outer_globals); } + if b.name == "memoryview" && args.len() == 1 { + // Native bytes-like inputs use the plain builtin; any + // other object is taken through the PEP 688 buffer + // protocol (`__buffer__`), so `memoryview(array('b', …))` + // yields a real view over the array's exported buffer + // (test_struct.test_pack_into / test_unpack_with_buffer). + match &args[0] { + Object::Bytes(_) | Object::ByteArray(_) | Object::MemoryView(_) => {} + other => { + if let Some(method) = instance_method(other, "__buffer__") { + let view = + self.call(&method, &[Object::Int(0)], &[], outer_globals)?; + // `__buffer__` returns a memoryview; adopt it + // directly so writes land in its buffer. + if matches!(view, Object::MemoryView(_)) { + return Ok(view); + } + return builtins::b_memoryview(std::slice::from_ref(&view)); + } + } + } + } if b.name == "next" && (args.len() == 1 || args.len() == 2) { return self.do_next_call(args, outer_globals); } @@ -8427,6 +8449,25 @@ impl Interpreter { }; return self.object_default_getattribute(&recv, &name); } + // Unbound `object.__reduce_ex__(self, protocol)` / + // `object.__reduce__(self)` — the default copy/pickle + // reduction. These need VM access (to import `copyreg` and + // run the receiver's `__getstate__`/`__getnewargs__` hooks), + // so the plain `BuiltinFn` is a sentinel; intercept the + // unbound form here exactly like the bound form is handled + // for `BoundMethod` targets below. Reached when code calls + // `object.__reduce_ex__(obj, proto)` directly (the canonical + // idiom inside `copyreg`/`copy`/`pickle` and in subclasses + // that delegate up to `object`). + if b.name == ".object_reduce_ex" && !args.is_empty() { + let recv = args[0].clone(); + let proto = args.get(1).and_then(|o| o.as_i64()).unwrap_or(0); + return self.object_reduce_ex(&recv, proto, outer_globals); + } + if b.name == ".object_reduce" && !args.is_empty() { + let recv = args[0].clone(); + return self.object_default_reduce(&recv, 2, outer_globals); + } if b.name == "globals" && args.is_empty() && kwargs.is_empty() { // CPython returns the calling function's module // globals. With our frame-by-argument model, the @@ -8803,6 +8844,32 @@ impl Interpreter { if ty.name == "repr" { return self.do_repr_call(&args[0], outer_globals); } + if ty.name == "memoryview" { + // `memoryview(x)` reaches here (the type is the + // callable). Native bytes-like inputs fall through to + // the normal constructor; anything else is taken + // through the PEP 688 buffer protocol (`__buffer__`) + // so `memoryview(array('b', …))` yields a real view + // over the array's exported buffer + // (test_struct.test_pack_into / test_unpack_with_buffer). + match &args[0] { + Object::Bytes(_) | Object::ByteArray(_) | Object::MemoryView(_) => {} + other => { + if let Some(method) = instance_method(other, "__buffer__") { + let view = self.call( + &method, + &[Object::Int(0)], + &[], + outer_globals, + )?; + if matches!(view, Object::MemoryView(_)) { + return Ok(view); + } + return builtins::b_memoryview(std::slice::from_ref(&view)); + } + } + } + } } // `type(name, bases, ns)` builds a new class dynamically. if Rc::ptr_eq(ty, &builtin_types().type_) && args.len() == 3 { @@ -10909,6 +10976,43 @@ impl Interpreter { if let Some((path, is_package)) = self.cache.find_source(full) { return self.load_from_file(full, &path, is_package); } + // Submodule search along the parent package's `__path__` + // (CPython semantics: `pkg.sub` is resolved against + // `pkg.__path__`, not just `sys.path`). This is what lets a + // frozen/namespace package whose backing directory isn't on + // `sys.path` still load on-disk submodules — e.g. a vendored + // CPython `test` package importing a sibling `test.test_xxx` + // module that lives next to the running script. + if let Some((parent, leaf)) = full.rsplit_once('.') { + if let Some(Object::Module(parent_mod)) = self.cache.get(parent) { + let path_dirs: Vec = parent_mod + .dict + .borrow() + .get(&DictKey(Object::from_static("__path__"))) + .map(|p| match p { + Object::List(l) => l + .borrow() + .iter() + .filter_map(|o| match o { + Object::Str(s) => Some(PathBuf::from(s.as_ref())), + _ => None, + }) + .collect(), + _ => Vec::new(), + }) + .unwrap_or_default(); + for dir in path_dirs { + let module_file = dir.join(leaf).with_extension("py"); + if module_file.is_file() { + return self.load_from_file(full, &module_file, false); + } + let pkg_init = dir.join(leaf).join("__init__.py"); + if pkg_init.is_file() { + return self.load_from_file(full, &pkg_init, true); + } + } + } + } // PEP 420 — namespace packages. If we found one or more // directories named `full` on `sys.path` without an // `__init__.py`, construct a namespace package: a module diff --git a/crates/weavepy-vm/src/stdlib/codecs_mod.rs b/crates/weavepy-vm/src/stdlib/codecs_mod.rs index 7d8a107..dcffd1e 100644 --- a/crates/weavepy-vm/src/stdlib/codecs_mod.rs +++ b/crates/weavepy-vm/src/stdlib/codecs_mod.rs @@ -542,12 +542,20 @@ impl FromUtf8Lenient for String { fn encode_ascii(s: &str, errors: &str) -> Result, RuntimeError> { let mut out = Vec::with_capacity(s.len()); - for c in s.chars() { + for (pos, c) in s.chars().enumerate() { let cp = c as u32; if cp < 0x80 { out.push(cp as u8); } else { - handle_encode_error(&mut out, c, errors, "ascii")?; + handle_encode_error( + &mut out, + s, + pos, + c, + errors, + "ascii", + "ordinal not in range(128)", + )?; } } Ok(out) @@ -567,12 +575,20 @@ fn decode_ascii(bytes: &[u8], errors: &str) -> Result { fn encode_latin1(s: &str, errors: &str) -> Result, RuntimeError> { let mut out = Vec::with_capacity(s.len()); - for c in s.chars() { + for (pos, c) in s.chars().enumerate() { let cp = c as u32; if cp < 0x100 { out.push(cp as u8); } else { - handle_encode_error(&mut out, c, errors, "latin-1")?; + handle_encode_error( + &mut out, + s, + pos, + c, + errors, + "latin-1", + "ordinal not in range(256)", + )?; } } Ok(out) @@ -584,15 +600,24 @@ fn decode_latin1(bytes: &[u8]) -> String { fn handle_encode_error( out: &mut Vec, + source: &str, + pos: usize, c: char, errors: &str, encoding: &str, + reason: &str, ) -> Result<(), RuntimeError> { match errors { - "strict" => Err(value_error(format!( - "'{encoding}' codec can't encode character '\\u{{{:x}}}'", - c as u32 - ))), + // Strict mode raises a real `UnicodeEncodeError` (a `ValueError` + // subclass) carrying the canonical `(encoding, object, start, end, + // reason)` payload, matching CPython — not a bare `ValueError`. + "strict" => Err(crate::error::unicode_encode_error( + encoding, + source, + pos, + pos + 1, + reason, + )), "ignore" => Ok(()), "replace" => { out.push(b'?'); diff --git a/crates/weavepy-vm/src/stdlib/python/array_mod.py b/crates/weavepy-vm/src/stdlib/python/array_mod.py index 46e1d26..57584ba 100644 --- a/crates/weavepy-vm/src/stdlib/python/array_mod.py +++ b/crates/weavepy-vm/src/stdlib/python/array_mod.py @@ -106,8 +106,14 @@ def tobytes(self): def __buffer__(self, flags): # PEP 688 buffer protocol: expose the packed bytes so buffer # consumers (``float``/``int``/``bytes``/``memoryview``) can read the - # array's contents, mirroring CPython's C-level buffer export. - return memoryview(self.tobytes()) + # array's contents, mirroring CPython's C-level buffer export. Back + # the view with a ``bytearray`` so it's *writable* — that's what lets + # ``struct.pack_into(memoryview(array(...)), ...)`` write through it + # (test_struct.test_pack_into). (The bytes are a snapshot; this + # list-backed array doesn't share storage with the view, but the view + # itself is a coherent read/write buffer, which is what consumers + # operate on.) + return memoryview(bytearray(self.tobytes())) def fromlist(self, seq): for v in seq: diff --git a/crates/weavepy-vm/src/stdlib/python/collections.py b/crates/weavepy-vm/src/stdlib/python/collections.py index 334402d..a32dc60 100644 --- a/crates/weavepy-vm/src/stdlib/python/collections.py +++ b/crates/weavepy-vm/src/stdlib/python/collections.py @@ -552,6 +552,14 @@ def namedtuple(typename, field_names, *, rename=False, defaults=None, module=Non class _NT: _fields = tuple(field_names) _field_defaults = field_defaults + __match_args__ = tuple(field_names) + + @classmethod + def _make(cls, iterable): + return cls(*iterable) + + def __getnewargs__(self): + return tuple(self._values) def __init__(self, *args, **kwargs): values = list(args) diff --git a/crates/weavepy-vm/src/stdlib/python/operator_mod.py b/crates/weavepy-vm/src/stdlib/python/operator_mod.py index 02ccdaa..15bb345 100644 --- a/crates/weavepy-vm/src/stdlib/python/operator_mod.py +++ b/crates/weavepy-vm/src/stdlib/python/operator_mod.py @@ -86,7 +86,23 @@ def floordiv(a, b): def index(a): "Same as a.__index__()." - return a.__index__() + # Mirror CPython's ``PyNumber_Index``: a missing ``__index__`` is a + # ``TypeError`` ("object cannot be interpreted as an integer"), not the + # ``AttributeError`` a bare ``a.__index__()`` would surface, and the + # result must itself be an ``int``. + if isinstance(a, int): + return a + m = getattr(type(a), "__index__", None) + if m is None: + raise TypeError( + "'%s' object cannot be interpreted as an integer" % type(a).__name__ + ) + result = m(a) + if not isinstance(result, int): + raise TypeError( + "__index__ returned non-int (type %s)" % type(result).__name__ + ) + return result def inv(a): "Same as ~a." diff --git a/crates/weavepy-vm/src/stdlib/python/struct.py b/crates/weavepy-vm/src/stdlib/python/struct.py index e8e0004..33d384d 100644 --- a/crates/weavepy-vm/src/stdlib/python/struct.py +++ b/crates/weavepy-vm/src/stdlib/python/struct.py @@ -6,6 +6,8 @@ public API surfaces. """ +import operator as _operator + import _struct as _impl @@ -23,35 +25,186 @@ def call(*args, **kwargs): return call +_INT_CODES = frozenset("bBhHiIlLqQnNP") +_FLOAT_CODES = frozenset("fde") + + +def _coerce_values(fmt, values): + """Coerce each argument through the protocol its format code implies. + + The Rust ``_struct`` core only sees concrete ``int``/``float``/``bool`` + objects, but CPython's ``struct`` runs ``__index__`` on integer codes, + ``__float__`` on float codes, and ``__bool__`` on ``?`` (so e.g. an + object whose ``__bool__`` raises propagates that exception). Mirror + that here, where we have interpreter access. + """ + try: + codes = _impl._value_codes(fmt) + except ValueError as e: + raise error(str(e)) from None + if len(codes) != len(values): + # Let the core raise the canonical "pack expected N items" error. + return values + out = [] + for code, v in zip(codes, values): + if code in _INT_CODES: + if isinstance(v, bool): + out.append(int(v)) + elif isinstance(v, int): + out.append(v) + else: + try: + out.append(_operator.index(v)) + except (TypeError, AttributeError): + raise error("required argument is not an integer") from None + elif code in _FLOAT_CODES: + if isinstance(v, float): + out.append(v) + else: + try: + out.append(float(v)) + except (TypeError, ValueError): + raise error("required argument is not a float") from None + elif code == "?": + out.append(bool(v)) + else: + out.append(v) + return out + + +def _readable(buffer): + """Return a bytes-like view of `buffer` the Rust core understands. + + The `_struct` core reads `bytes`/`bytearray`/`memoryview` directly. Any + other object implementing the buffer protocol (notably `array.array`) + is surfaced through its `tobytes()` export — CPython accepts any + buffer-protocol object here, and this is the slice of that protocol we + can reach from the frozen wrapper (test_struct.test_unpack_with_buffer). + """ + if isinstance(buffer, (bytes, bytearray, memoryview)): + return buffer + tobytes = getattr(buffer, "tobytes", None) + if callable(tobytes): + return tobytes() + # Let the core raise the canonical "a bytes-like object is required". + return buffer + + calcsize = _wrap(_impl.calcsize) -pack = _wrap(_impl.pack) -unpack = _wrap(_impl.unpack) -pack_into = _wrap(_impl.pack_into) -unpack_from = _wrap(_impl.unpack_from) -def _iter_unpack(fmt, buffer, size): +def unpack(fmt, buffer): + try: + return _impl.unpack(fmt, _readable(buffer)) + except ValueError as e: + raise error(str(e)) from None + + +def unpack_from(fmt, buffer, offset=0): + try: + return _impl.unpack_from(fmt, _readable(buffer), offset) + except ValueError as e: + raise error(str(e)) from None + + +def pack(fmt, *values): + values = _coerce_values(fmt, values) + try: + return _impl.pack(fmt, *values) + except ValueError as e: + raise error(str(e)) from None + + +def _writable(buffer): + """Resolve `buffer` to a read-write target the Rust core can pack into. + + CPython's `pack_into` requires a writable buffer-protocol object. We + accept `bytearray` and writable `memoryview` directly, take any other + buffer-protocol object (e.g. `array.array`) through `memoryview()`, and + reject read-only / non-buffer arguments with `TypeError` + (test_struct.test_pack_into). + """ + if isinstance(buffer, bytearray): + return buffer + if isinstance(buffer, memoryview): + if buffer.readonly: + raise TypeError("cannot modify read-only memory") + return buffer + if isinstance(buffer, (bytes, str)): + raise TypeError( + "argument must be a read-write bytes-like object, not " + + type(buffer).__name__ + ) + mv = memoryview(buffer) # raises TypeError if no buffer protocol + if mv.readonly: + raise TypeError("argument must be a read-write bytes-like object") + return mv + + +def pack_into(fmt, buffer, offset, *values): + target = _writable(buffer) + values = _coerce_values(fmt, values) + try: + return _impl.pack_into(fmt, target, offset, *values) + except ValueError as e: + raise error(str(e)) from None + + +class unpack_iterator: + """Iterator returned by `iter_unpack` / `Struct.iter_unpack`. + + Mirrors CPython's `unpack_iterator` C type: it can't be constructed + directly from Python, it yields one tuple per `size`-byte chunk, and + `__length_hint__` reports the number of chunks still to come (so + `operator.length_hint` and list-preallocation behave as CPython's do — + test_struct.test_length_hint / test_uninstantiable). + """ + + __slots__ = ("_fmt", "_buffer", "_size", "_offset", "_len") + + def __new__(cls, *args, **kwargs): + raise TypeError("cannot create 'unpack_iterator' instances") + + def __iter__(self): + return self + + def __next__(self): + if self._offset >= self._len: + raise StopIteration + result = unpack_from(self._fmt, self._buffer, self._offset) + self._offset += self._size + return result + + def __length_hint__(self): + return (self._len - self._offset) // self._size + + +def _make_unpack_iterator(fmt, buffer, size): # CPython validates the buffer length up front (a `struct.error` is # raised by `iter_unpack` itself, not lazily on the first `next()`), # and rejects a zero-width format outright. if size == 0: raise error("cannot iteratively unpack with a struct of length 0") + buffer = _readable(buffer) if len(buffer) % size != 0: raise error( "iterative unpacking requires a buffer of a multiple of " f"{size} bytes" ) - - def _gen(): - for off in range(0, len(buffer), size): - yield unpack_from(fmt, buffer, off) - - return _gen() + # Bypass the guard `__new__` to build the (otherwise unconstructable) + # iterator, exactly as the C type does internally. + it = object.__new__(unpack_iterator) + it._fmt = fmt + it._buffer = buffer + it._size = size + it._offset = 0 + it._len = len(buffer) + return it def iter_unpack(fmt, buffer): """Iterate over `buffer` in `calcsize(fmt)` chunks.""" - return _iter_unpack(fmt, buffer, calcsize(fmt)) + return _make_unpack_iterator(fmt, buffer, calcsize(fmt)) class Struct: @@ -68,10 +221,18 @@ def __new__(cls, *args, **kwargs): return self def __init__(self, fmt): - if isinstance(fmt, bytes): - fmt = fmt.decode("ascii") + # CPython encodes the format to a C string, so a non-ASCII format + # (e.g. a lone surrogate) raises UnicodeEncodeError, and an invalid + # but ASCII format raises struct.error. Both must be detected + # *before* we mutate `self`, so a failed re-`__init__` leaves the + # previously-compiled format intact (test_Struct_reinitialization). + if isinstance(fmt, str): + fmt.encode("ascii") # validates encodability; may raise UnicodeEncodeError + elif isinstance(fmt, (bytes, bytearray)): + fmt = bytes(fmt).decode("ascii") + size = calcsize(fmt) self._fmt = fmt - self.size = calcsize(fmt) + self.size = size def _ensure_initialized(self): if self._fmt is None: @@ -100,7 +261,7 @@ def unpack_from(self, buffer, offset=0): def iter_unpack(self, buffer): self._ensure_initialized() - return _iter_unpack(self._fmt, buffer, self.size) + return _make_unpack_iterator(self._fmt, buffer, self.size) def __repr__(self): self._ensure_initialized() diff --git a/crates/weavepy-vm/src/stdlib/python/test_init.py b/crates/weavepy-vm/src/stdlib/python/test_init.py index 78e2251..9e84380 100644 --- a/crates/weavepy-vm/src/stdlib/python/test_init.py +++ b/crates/weavepy-vm/src/stdlib/python/test_init.py @@ -12,3 +12,38 @@ # harness, so ``support`` / ``libregrtest`` are imported lazily by the # things that need them. __all__ = [] + +# CPython resolves ``test.`` submodules against this package's +# ``__path__``. WeavePy ships ``test`` (and ``test.support``) frozen, so +# the package has no backing directory by default — which means a +# vendored test that imports a *sibling* test module (e.g. +# ``from test import test_contextlib`` in ``test_contextlib_async``, or +# ``test.pickletester``) can't find it. Point ``__path__`` at any on-disk +# ``test/`` directory currently on ``sys.path`` (a checked-out +# ``Lib/test/`` is ``sys.path[0]`` when its files are run directly), so +# those siblings load from disk. Frozen modules still win — the import +# machinery consults the frozen registry before walking ``__path__`` — so +# ``test.support`` keeps using the faithful frozen port. +import os as _os +import sys as _sys + +try: + __path__ +except NameError: + __path__ = [] +for _p in _sys.path: + try: + if ( + _p + and _os.path.basename(_os.path.normpath(_p)) == "test" + and _os.path.isdir(_p) + and _p not in __path__ + ): + __path__.append(_p) + except (TypeError, ValueError): + pass +del _os, _sys +try: + del _p +except NameError: + pass diff --git a/crates/weavepy-vm/src/stdlib/python/unittest.py b/crates/weavepy-vm/src/stdlib/python/unittest.py index 65a0c25..d93ab3b 100644 --- a/crates/weavepy-vm/src/stdlib/python/unittest.py +++ b/crates/weavepy-vm/src/stdlib/python/unittest.py @@ -466,6 +466,12 @@ def tearDownClass(cls): def skipTest(self, reason): raise SkipTest(reason) + def _callTestMethod(self, method): + # Indirection point CPython uses so ``IsolatedAsyncioTestCase`` + # can drive an ``async def`` test through an event loop. The + # default just calls the (synchronous) method. + method() + def shortDescription(self): doc = self._testMethodDoc return doc.strip().split("\n")[0].strip() if doc else None @@ -610,7 +616,7 @@ def run(self, result=None): n_err = len(result.errors) ok = False try: - testMethod() + self._callTestMethod(testMethod) except KeyboardInterrupt: raise except SkipTest as e: @@ -1730,3 +1736,10 @@ def main(module="__main__", defaultTest=None, argv=None, testRunner=None, verbosity=verbosity, failfast=failfast, catchbreak=catchbreak, buffer=buffer, warnings=warnings, tb_locals=tb_locals) return getattr(program, "result", None) + + +# Re-export the async TestCase (CPython does this at the bottom of +# ``unittest/__init__.py``). Done last so ``unittest.TestCase`` is fully +# defined when ``async_case`` imports the package. +from .async_case import IsolatedAsyncioTestCase # noqa: E402 +__all__.append("IsolatedAsyncioTestCase") diff --git a/crates/weavepy-vm/src/stdlib/struct_mod.rs b/crates/weavepy-vm/src/stdlib/struct_mod.rs index 5f63eb8..16e615b 100644 --- a/crates/weavepy-vm/src/stdlib/struct_mod.rs +++ b/crates/weavepy-vm/src/stdlib/struct_mod.rs @@ -24,7 +24,7 @@ use crate::sync::RefCell; use byteorder::{BigEndian, ByteOrder, LittleEndian, NativeEndian}; -use crate::error::{type_error, value_error, RuntimeError}; +use crate::error::{overflow_error, type_error, value_error, RuntimeError}; use crate::import::ModuleCache; use crate::object::{BuiltinFn, DictData, DictKey, Object, PyModule}; @@ -68,6 +68,13 @@ struct CompiledFormat { impl CompiledFormat { /// Parse a CPython-shaped format string. fn parse(fmt: &str) -> Result { + // CPython treats the format as a C string, so an embedded NUL + // terminates it early and is reported up front + // (test_struct.test_issue35714), rather than falling through to + // the generic "bad char" diagnostic. + if fmt.contains('\0') { + return Err(struct_error("embedded null character")); + } let mut chars = fmt.chars().peekable(); let endian = match chars.peek() { Some('@') => { @@ -217,10 +224,15 @@ impl CompiledFormat { }; let mut buf = vec![0u8; f.count]; if f.count > 0 { - let take = data.len().min(f.count - 1).min(255); - buf[0] = take as u8; - if take > 0 { - buf[1..=take].copy_from_slice(&data[..take]); + // CPython copies up to `count - 1` data bytes, but the + // leading length byte saturates at 255 (the most a + // single byte can encode). For e.g. `1000p` of 1000 + // bytes the buffer holds 999 data bytes yet the length + // prefix reads 255 (test_struct.test_p_code). + let copy = data.len().min(f.count - 1); + buf[0] = copy.min(255) as u8; + if copy > 0 { + buf[1..=copy].copy_from_slice(&data[..copy]); } } out.extend_from_slice(&buf); @@ -315,11 +327,14 @@ fn element_size(code: char, endian: Endian) -> Result { 'q' | 'Q' | 'd' => 8, 'n' | 'N' => match endian { Endian::Native => std::mem::size_of::(), - _ => return Err(struct_error("'n' format code only valid in native mode")), + // In standard / explicit-endian modes `n`/`N` simply aren't + // recognised; CPython reports the same "bad char" diagnostic as + // for any other unknown code (test_struct.test_nN_code). + _ => return Err(struct_error(format!("bad char in struct format: '{code}'"))), }, 'P' => match endian { Endian::Native => std::mem::size_of::(), - _ => return Err(struct_error("'P' format code only valid in native mode")), + _ => return Err(struct_error(format!("bad char in struct format: '{code}'"))), }, _ => return Err(struct_error(format!("bad char in struct format: '{code}'"))), }) @@ -418,11 +433,19 @@ fn encode_one( let f = value .as_f64() .ok_or_else(|| struct_error("required argument is not a float"))?; + // A finite double whose magnitude rounds above `FLT_MAX` + // overflows binary32. CPython's `_PyFloat_Pack4` reports this + // as `OverflowError` (not `struct.error`), so the frozen + // wrapper lets it propagate (test_struct.test_705836). + let f32v = f as f32; + if f.is_finite() && f32v.is_infinite() { + return Err(overflow_error("float too large to pack with f format")); + } let mut buf = [0u8; 4]; match endian { - Endian::Native => NativeEndian::write_f32(&mut buf, f as f32), - Endian::Standard | Endian::Little => LittleEndian::write_f32(&mut buf, f as f32), - Endian::Big => BigEndian::write_f32(&mut buf, f as f32), + Endian::Native => NativeEndian::write_f32(&mut buf, f32v), + Endian::Standard | Endian::Little => LittleEndian::write_f32(&mut buf, f32v), + Endian::Big => BigEndian::write_f32(&mut buf, f32v), } out.extend_from_slice(&buf); Ok(()) @@ -441,11 +464,13 @@ fn encode_one( Ok(()) } 'e' => { - // Half-precision IEEE 754. Convert via the bits. + // Half-precision IEEE 754, converted from the double with + // round-half-to-even (CPython `_PyFloat_Pack2`), not via an + // intermediate `f32` truncation. let f = value .as_f64() .ok_or_else(|| struct_error("required argument is not a float"))?; - let half = f32_to_half(f as f32); + let half = f64_to_half(f)?; let mut buf = [0u8; 2]; match endian { Endian::Native => NativeEndian::write_u16(&mut buf, half), @@ -582,32 +607,92 @@ fn read_f64(endian: Endian, b: &[u8]) -> f64 { } } -/// IEEE 754 binary16 conversions. Doesn't depend on `f16` because -/// the standard library hasn't shipped a stable type yet. -fn f32_to_half(f: f32) -> u16 { - let bits = f.to_bits(); - let sign = ((bits >> 16) & 0x8000) as u16; - let exp = ((bits >> 23) & 0xFF) as i32; - let mantissa = bits & 0x007F_FFFF; - if exp == 0xFF { - // NaN/Inf - let mant = if mantissa != 0 { 0x200 } else { 0 }; - return sign | 0x7C00 | mant; +/// `frexp`: decompose a finite, non-NaN `x` into `(m, e)` with +/// `x == m * 2**e` and `0.5 <= |m| < 1` (or `m == 0` for `x == 0`). +/// std doesn't ship `frexp`, so we do it by exponent-field surgery. +fn frexp(x: f64) -> (f64, i32) { + if x == 0.0 || x.is_nan() || x.is_infinite() { + return (x, 0); } - let new_exp = exp - 127 + 15; - if new_exp >= 0x1F { - return sign | 0x7C00; // Inf + let exp_field = ((x.to_bits() >> 52) & 0x7ff) as i32; + if exp_field == 0 { + // Subnormal: scale into the normal range first, then correct `e`. + let scaled = x * f64::from_bits(0x43f0_0000_0000_0000); // * 2**64 + let exp_s = ((scaled.to_bits() >> 52) & 0x7ff) as i32 - 64; + let m_bits = (scaled.to_bits() & !(0x7ffu64 << 52)) | (1022u64 << 52); + (f64::from_bits(m_bits), exp_s - 1022) + } else { + let m_bits = (x.to_bits() & !(0x7ffu64 << 52)) | (1022u64 << 52); + (f64::from_bits(m_bits), exp_field - 1022) } - if new_exp <= 0 { - if new_exp < -10 { - return sign; +} + +#[inline] +fn ldexp(f: f64, n: i32) -> f64 { + f * 2f64.powi(n) +} + +/// Port of CPython's `_PyFloat_Pack2` (`Objects/floatobject.c`): +/// convert a double to an IEEE 754 binary16 bit pattern with +/// round-half-to-even, returning the value in host order. Raises +/// `OverflowError` on overflow, exactly like CPython +/// (test_struct.test_705836 / test_half_float assert `OverflowError`, +/// which is *not* a `struct.error`). +fn f64_to_half(x: f64) -> Result { + let sign: u16; + let mut e: i32; + let mut bits: u16; + if x == 0.0 { + sign = u16::from(x.is_sign_negative()); + e = 0; + bits = 0; + } else if x.is_infinite() { + sign = u16::from(x < 0.0); + e = 0x1f; + bits = 0; + } else if x.is_nan() { + sign = u16::from(x.is_sign_negative()); + e = 0x1f; + bits = 512; + } else { + sign = u16::from(x < 0.0); + let ax = x.abs(); + let (mut f, fe) = frexp(ax); + e = fe; + // Normalize f to [1.0, 2.0). + f *= 2.0; + e -= 1; + if e >= 16 { + return Err(overflow_error("float too large to pack with e format")); + } else if e < -25 { + // |x| < 2**-25 — underflow to (signed) zero. + f = 0.0; + e = 0; + } else if e < -14 { + // Gradual underflow (subnormal half). + f = ldexp(f, 14 + e); + e = 0; + } else { + e += 15; + f -= 1.0; // strip the implicit leading 1 + } + f *= 1024.0; // 2**10 + bits = f as u16; // truncating cast + // Round half to even. + let frac = f - f64::from(bits); + if frac > 0.5 || (frac == 0.5 && (bits & 1) == 1) { + bits += 1; + if bits == 1024 { + // Carry rippled out of the 10-bit mantissa. + bits = 0; + e += 1; + if e == 31 { + return Err(overflow_error("float too large to pack with e format")); + } + } } - let mantissa = mantissa | 0x0080_0000; - let shift = (14 - new_exp) as u32; - let result = (mantissa >> shift) as u16; - return sign | result; } - sign | ((new_exp as u16) << 10) | ((mantissa >> 13) as u16) + Ok(bits | ((e as u16) << 10) | (sign << 15)) } fn half_to_f32(half: u16) -> f32 { @@ -675,6 +760,7 @@ pub fn build(_cache: &ModuleCache) -> Rc { Object::from_static("Binary data packing/unpacking (RFC 0019 core)."), ); register(&mut d, "calcsize", b_calcsize); + register(&mut d, "_value_codes", b_value_codes); register(&mut d, "pack", b_pack); register(&mut d, "unpack", b_unpack); register(&mut d, "pack_into", b_pack_into); @@ -723,6 +809,31 @@ fn b_calcsize(args: &[Object]) -> Result { Ok(Object::Int(cf.size as i64)) } +/// Return one format character per *value slot* the format consumes, in +/// order (`x` pad bytes contribute nothing; `s`/`p` contribute a single +/// slot; numeric codes contribute `count` slots). The frozen wrapper uses +/// this to coerce each argument through the right protocol (`__index__` +/// for integer codes, `__float__` for floats, `__bool__` for `?`) before +/// handing concrete `int`/`float`/`bool` values to the Rust packer, which +/// has no interpreter access of its own. +fn b_value_codes(args: &[Object]) -> Result { + let fmt = fmt_arg(args, 0)?; + let cf = CompiledFormat::parse(&fmt)?; + let mut s = String::new(); + for f in &cf.fields { + match f.code { + 'x' => {} + 's' | 'p' => s.push(f.code), + c => { + for _ in 0..f.count { + s.push(c); + } + } + } + } + Ok(Object::from_str(s)) +} + fn b_pack(args: &[Object]) -> Result { let fmt = fmt_arg(args, 0)?; let cf = CompiledFormat::parse(&fmt)?; @@ -738,15 +849,31 @@ fn b_unpack(args: &[Object]) -> Result { Ok(Object::new_tuple(vals)) } +/// Resolve a `pack_into`/`unpack_from` byte offset, matching CPython's +/// `Py_ssize_t` coercion: ints (and `bool`) pass through, an int too big +/// for the platform word is `OverflowError`, and any non-integer is a +/// `TypeError` (test_struct.test_pack_into's bogus-offset cases). +fn ssize_offset(o: &Object) -> Result { + match o { + Object::Int(n) => Ok(*n), + Object::Bool(b) => Ok(i64::from(*b)), + Object::Long(_) => Err(overflow_error( + "Python int too large to convert to C ssize_t", + )), + other => Err(type_error(format!( + "'{}' object cannot be interpreted as an integer", + other.type_name() + ))), + } +} + fn b_pack_into(args: &[Object]) -> Result { if args.len() < 3 { return Err(type_error("pack_into() requires at least 3 arguments")); } let fmt = fmt_arg(args, 0)?; let cf = CompiledFormat::parse(&fmt)?; - let offset = args[2] - .as_i64() - .ok_or_else(|| type_error("offset must be int"))?; + let offset = ssize_offset(&args[2])?; let bytes = cf.pack(&args[3..])?; match &args[1] { Object::ByteArray(buf) => { @@ -758,8 +885,28 @@ fn b_pack_into(args: &[Object]) -> Result { buf[off..off + bytes.len()].copy_from_slice(&bytes); Ok(Object::None) } + Object::MemoryView(mv) => { + // Writable buffer-protocol target (e.g. `memoryview(array(...))`). + if mv.readonly.get() { + return Err(type_error( + "cannot modify read-only memory".to_owned(), + )); + } + let off = resolve_buffer_offset(offset, mv.len.get(), cf.size, "pack_into", true)?; + let base = mv.start.get(); + match &mv.buffer { + crate::object::MemoryViewBuffer::ByteArray(b) => { + let mut b = b.borrow_mut(); + b[base + off..base + off + bytes.len()].copy_from_slice(&bytes); + Ok(Object::None) + } + crate::object::MemoryViewBuffer::Bytes(_) => Err(type_error( + "cannot modify read-only memory".to_owned(), + )), + } + } _ => Err(type_error( - "pack_into() requires a bytearray buffer".to_owned(), + "argument must be a read-write bytes-like object".to_owned(), )), } } From 59f0cbf80dc840675e5bb3dce998ba04818f4fb6 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Mon, 8 Jun 2026 10:36:54 -0700 Subject: [PATCH 3/9] feat: advance CPython Lib/test conformance wave 2 --- crates/weavepy-compiler/src/cpython_code.rs | 38 +- crates/weavepy-compiler/src/lib.rs | 170 ++- crates/weavepy-vm/src/builtins.rs | 147 +- crates/weavepy-vm/src/gc_trace.rs | 30 + crates/weavepy-vm/src/lib.rs | 931 ++++++++++-- crates/weavepy-vm/src/object.rs | 178 ++- crates/weavepy-vm/src/stdlib/marshal_mod.rs | 3 + crates/weavepy-vm/src/stdlib/mod.rs | 9 + .../weavepy-vm/src/stdlib/python/_seqtools.py | 129 ++ .../weavepy-vm/src/stdlib/python/functools.py | 1268 ++++++++++++----- crates/weavepy-vm/src/stdlib/python/pickle.py | 145 +- .../python/test_support_import_helper.py | 18 +- .../weavepy-vm/src/stdlib/python/traceback.py | 61 +- crates/weavepy-vm/src/stdlib/python/typing.py | 4 + crates/weavepy-vm/src/trace.rs | 10 + crates/weavepy/src/lib.rs | 9 +- tests/regrtest/expectations.toml | 12 +- 17 files changed, 2647 insertions(+), 515 deletions(-) create mode 100644 crates/weavepy-vm/src/stdlib/python/_seqtools.py diff --git a/crates/weavepy-compiler/src/cpython_code.rs b/crates/weavepy-compiler/src/cpython_code.rs index f1e1f8a..5f24a4d 100644 --- a/crates/weavepy-compiler/src/cpython_code.rs +++ b/crates/weavepy-compiler/src/cpython_code.rs @@ -384,6 +384,11 @@ pub struct CpythonCode { pub firstlineno: u32, /// One [`Position`] per code unit. pub positions: Vec, + /// Code-unit offset of each WeavePy instruction's *opcode* unit (i.e. + /// past any `EXTENDED_ARG` prefix), indexed by WeavePy instruction + /// index. Multiply by 2 for the `co_code` byte offset CPython's + /// `f_lasti`/`tb_lasti` expose. Length equals the instruction count. + pub inst_offsets: Vec, } const CO_FAST_LOCAL: u8 = 0x20; @@ -481,14 +486,24 @@ pub fn encode(code: &CodeObject) -> CpythonCode { // Emit code units + per-unit positions. let mut co_code: Vec = Vec::with_capacity(starts[n] * 2); let mut positions: Vec = Vec::with_capacity(starts[n]); + let mut inst_offsets: Vec = Vec::with_capacity(n); let firstlineno = code.linetable.first().copied().unwrap_or(1); for i in 0..n { let line = code.linetable.get(i).copied().unwrap_or(firstlineno) as i32; + // PEP-657 columns, when the compiler tracked them for this + // instruction. `col`/`end_col` are byte offsets (`-1` = unknown); + // `end_lineno` is `0` when unknown (fall back to the start line). + let cs = code.coltable.get(i).copied().unwrap_or_default(); + let end_lineno = if cs.end_lineno != 0 { + cs.end_lineno as i32 + } else { + line + }; let pos = Position { lineno: line, - end_lineno: line, - col: None, - end_col: None, + end_lineno, + col: (cs.col >= 0).then_some(cs.col as u32), + end_col: (cs.end_col >= 0).then_some(cs.end_col as u32), }; let arg = args[i]; // EXTENDED_ARG units carry the high base-256 digits, MSB first. @@ -498,6 +513,9 @@ pub fn encode(code: &CodeObject) -> CpythonCode { co_code.push(byte); positions.push(pos); } + // The opcode unit lands here, past any EXTENDED_ARG prefix — this is + // the code-unit offset CPython's `f_lasti`/`tb_lasti` point at. + inst_offsets.push((co_code.len() / 2) as u32); co_code.push(mapped[i].cp_op); co_code.push((arg & 0xFF) as u8); positions.push(pos); @@ -518,6 +536,7 @@ pub fn encode(code: &CodeObject) -> CpythonCode { stacksize: compute_stacksize(code), firstlineno, positions, + inst_offsets, } } @@ -1091,6 +1110,19 @@ impl CodeObject { pub fn to_cpython(&self) -> CpythonCode { encode(self) } + + /// Translate a WeavePy instruction index into the `co_code` byte offset + /// CPython's `f_lasti`/`tb_lasti` expose (2 bytes/code unit, opcode past + /// any `EXTENDED_ARG` prefix). Keeps `co_positions()` / `dis` anchoring + /// consistent across the cache- and extended-arg-inflated encoding. + #[must_use] + pub fn cpython_lasti(&self, weavepy_index: u32) -> u32 { + let cp = self.to_cpython(); + cp.inst_offsets + .get(weavepy_index as usize) + .map(|&unit| unit * 2) + .unwrap_or(weavepy_index * 2) + } } #[cfg(test)] diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index f0d7742..dd9b6e1 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -52,6 +52,11 @@ pub enum CompileError { ContinueOutsideLoop, #[error("`return` outside function")] ReturnOutsideFunction, + /// A `yield` / `yield from` expression outside a function body. + /// `{0}` is the keyword (`yield` or `yield from`) so the message + /// matches CPython's `SyntaxError: 'yield' outside function`. + #[error("'{0}' outside function")] + YieldOutsideFunction(&'static str), #[error("`{0}` is not yet supported by the compiler ({1})")] NotImplemented(&'static str, &'static str), #[error("internal compiler error: {0}")] @@ -88,6 +93,11 @@ pub struct CodeObject { /// Source line number (1-based) per emitted instruction. Same length /// as `instructions`. Used for traceback rendering. pub linetable: Vec, + /// PEP-657 fine-grained column spans, one per instruction (same length + /// as `instructions` once emission finishes). Drives the column fields + /// of `co_positions()`. Empty when never populated (e.g. code objects + /// reconstructed from marshal, which doesn't carry columns). + pub coltable: Vec, /// Number of positional + keyword arguments (excluding `*args`/`**kwargs`). pub arg_count: u32, /// Number of positional-only arguments. @@ -114,6 +124,28 @@ pub struct CodeObject { pub is_async_generator: bool, } +/// A per-instruction source-column span (PEP-657). `col`/`end_col` are +/// 0-based UTF-8 byte offsets within their respective source lines, and +/// are `-1` when the column was not tracked. `end_lineno` is `0` when +/// unknown (callers fall back to the instruction's start line). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ColSpan { + pub end_lineno: u32, + pub col: i32, + pub end_col: i32, +} + +impl Default for ColSpan { + fn default() -> Self { + // "Unknown" sentinel — matches an instruction with no tracked span. + Self { + end_lineno: 0, + col: -1, + end_col: -1, + } + } +} + /// One entry in a code object's exception table. Mirrors the /// PEP 657-style out-of-line model CPython 3.11+ uses. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -445,6 +477,17 @@ impl LineIndex { .saturating_sub(1); (idx as u32) + 1 } + + /// 1-based line and 0-based byte column for a source byte offset. + /// Returns `(0, 0)` when the index is empty. + fn pos_for(&self, byte: u32) -> (u32, u32) { + let line = self.line_for(byte); + if line == 0 { + return (0, 0); + } + let line_start = self.line_starts[(line - 1) as usize]; + (line, byte.saturating_sub(line_start)) + } } // ---------- scope kinds ---------- @@ -500,6 +543,10 @@ struct Compiler { /// Line number assigned to the next emitted instruction; updated as /// the compiler descends through the AST. current_line: u32, + /// Source byte span `(start, end)` for the AST node currently being + /// emitted. Drives PEP-657 column tracking in [`Self::emit`]. Updated + /// at statement and expression granularity as the compiler descends. + current_span: (u32, u32), /// `True` for methods compiled inside a class body. Such methods /// implicitly capture the class's `__class__` cell so `super()` /// works without arguments. @@ -599,6 +646,7 @@ impl Compiler { finally_counter: 0, line_index, current_line: 0, + current_span: (0, 0), inside_class_body: false, annotations_initialized: false, code_kind: kind, @@ -640,6 +688,7 @@ impl Compiler { let offset = self.co.instructions.len() as u32; self.co.instructions.push(Instruction { op, arg }); self.co.linetable.push(self.current_line); + self.co.coltable.push(self.resolve_colspan()); offset } @@ -650,6 +699,30 @@ impl Compiler { } } + /// Resolve [`Self::current_span`] into a PEP-657 [`ColSpan`] for the + /// next emitted instruction. Columns are 0-based byte offsets into + /// their source lines; a degenerate `(0, 0)` span yields "unknown". + fn resolve_colspan(&self) -> ColSpan { + let (start, end) = self.current_span; + if start == 0 && end == 0 { + return ColSpan::default(); + } + let (_start_line, start_col) = self.line_index.pos_for(start); + let (end_line, end_col) = self.line_index.pos_for(end); + ColSpan { + end_lineno: end_line, + col: start_col as i32, + end_col: end_col as i32, + } + } + + /// Point [`Self::current_span`] at an AST node's source span so the + /// instructions emitted for it carry the node's columns. + #[inline] + fn set_span(&mut self, span: weavepy_lexer::Span) { + self.current_span = (span.start.0, span.end.0); + } + fn next_offset(&self) -> u32 { self.co.instructions.len() as u32 } @@ -778,6 +851,7 @@ impl Compiler { fn compile_stmt(&mut self, stmt: &Stmt) -> Result<(), CompileError> { self.set_line_from(stmt.span.start.0); + self.set_span(stmt.span); match &stmt.kind { StmtKind::Expr(e) => { self.compile_expr(e)?; @@ -922,9 +996,17 @@ impl Compiler { orelse, } => { self.compile_expr(iter)?; + // PEP-657: `GET_ITER` (iter() failure) and `FOR_ITER` + // (__next__ failure) report the iterator *expression* as + // the error location, matching CPython's traceback columns. + self.set_span(iter.span); self.emit(OpCode::GetIter, 0); let loop_top = self.next_offset(); + self.set_span(iter.span); let for_site = self.emit(OpCode::ForIter, 0); + // Remember FOR_ITER's source line so END_FOR can reuse it (see + // the END_FOR emission below). + let for_line = self.current_line; self.compile_assign(target)?; self.loop_stack.push(LoopFrame { continue_target: loop_top, @@ -939,6 +1021,12 @@ impl Compiler { let frame = self.loop_stack.pop().expect("loop frame"); let after = self.next_offset(); self.patch_jump(for_site, after); + // Attribute END_FOR to the iterator expression (the `for` line), + // matching CPython. FOR_ITER already fired a line event for this + // line on the final iteration, so reusing the line prevents a + // spurious `line` event for the loop body after exhaustion. + self.set_span(iter.span); + self.current_line = for_line; self.emit(OpCode::EndFor, 0); for s in orelse { self.compile_stmt(s)?; @@ -1077,11 +1165,9 @@ impl Compiler { if is_for { self.emit(OpCode::PopTop, 0); } - let site = self.co.instructions.len() as u32; - self.co.instructions.push(Instruction { - op: OpCode::JumpForward, - arg: 0, - }); + // Route through `emit` so the line/column side-tables stay + // length-aligned with the instruction stream. + let site = self.emit(OpCode::JumpForward, 0); self.loop_stack .last_mut() .expect("loop frame") @@ -2879,6 +2965,19 @@ impl Compiler { // ---------- expressions ---------- fn compile_expr(&mut self, e: &Expr) -> Result<(), CompileError> { + // PEP-657 column tracking: emit this node's instructions under its + // own source span. Sub-expressions are compiled through this same + // wrapper, so each restores the parent span on return — leaving + // `current_span` pointing at *this* node when its own opcode is + // finally emitted (e.g. the `BinaryOp` after both operands). + let saved = self.current_span; + self.set_span(e.span); + let r = self.compile_expr_inner(e); + self.current_span = saved; + r + } + + fn compile_expr_inner(&mut self, e: &Expr) -> Result<(), CompileError> { match &e.kind { ExprKind::Constant(c) => { let idx = self.co.intern_constant(c.clone().into()); @@ -3131,6 +3230,14 @@ impl Compiler { self.compile_formatted_value(value, *conversion, format_spec.as_deref())?; } ExprKind::Yield(value) => { + // `yield` is only legal in a function body. At module or + // class scope (or inside a comprehension's own frame) it is + // a SyntaxError — CPython reports "'yield' outside function". + // Catching it here also prevents a non-generator frame from + // ever executing `YIELD_VALUE` at runtime. + if self.kind != CodeKind::Function { + return Err(CompileError::YieldOutsideFunction("yield")); + } if let Some(v) = value { self.compile_expr(v)?; } else { @@ -3140,6 +3247,9 @@ impl Compiler { self.emit(OpCode::YieldValue, 0); } ExprKind::YieldFrom(iter) => { + if self.kind != CodeKind::Function { + return Err(CompileError::YieldOutsideFunction("yield from")); + } // CPython 3.13 pattern: // // GET_YIELD_FROM_ITER @@ -3723,6 +3833,7 @@ fn compile_comp_body( } let loop_top = inner.next_offset(); let for_site = inner.emit(OpCode::ForIter, 0); + let for_line = inner.current_line; inner.compile_assign(&gen.target)?; let mut filter_jumps = Vec::new(); for cond in &gen.ifs { @@ -3739,6 +3850,10 @@ fn compile_comp_body( inner.patch_jump(back, loop_top); let after = inner.next_offset(); inner.patch_jump(for_site, after); + // Keep END_FOR on the iterator line (see statement-level for loop) so a + // comprehension's loop exhaustion does not emit a spurious `line` event. + inner.set_span(gen.iter.span); + inner.current_line = for_line; inner.emit(OpCode::EndFor, 0); Ok(()) } @@ -4123,8 +4238,28 @@ fn expr_contains_yield(expr: &Expr) -> bool { match &expr.kind { ExprKind::Yield(_) | ExprKind::YieldFrom(_) => true, ExprKind::Await(inner) => expr_contains_yield(inner), - ExprKind::Lambda { .. } => false, - ExprKind::GeneratorExp { .. } => false, + // A lambda body runs in its own scope, but its *default argument + // values* are evaluated in the enclosing scope — so a `yield` there + // belongs to the enclosing function, e.g. `def f(): lambda x=(yield): 1` + // makes `f` a generator. The body is excluded. + ExprKind::Lambda { args, .. } => { + args.defaults.iter().any(expr_contains_yield) + || args + .kw_defaults + .iter() + .flatten() + .any(expr_contains_yield) + } + // A comprehension runs in its own scope, but the *leftmost* `for` + // clause's iterable is evaluated in the enclosing scope and passed + // in as the `.0` argument. A `yield` there therefore belongs to the + // enclosing function and makes it a generator — e.g. + // `def f(): list(i for i in [(yield 26)])`. (A `yield` anywhere else + // in a comprehension is a SyntaxError, so only the first iterable + // can contribute.) + ExprKind::GeneratorExp { generators, .. } => { + generators.first().is_some_and(|g| expr_contains_yield(&g.iter)) + } ExprKind::JoinedStr(parts) => parts.iter().any(expr_contains_yield), ExprKind::FormattedValue { value, format_spec, .. @@ -4169,7 +4304,11 @@ fn expr_contains_yield(expr: &Expr) -> bool { .any(|k| k.as_ref().is_some_and(expr_contains_yield)) || values.iter().any(expr_contains_yield) } - ExprKind::ListComp { .. } | ExprKind::SetComp { .. } | ExprKind::DictComp { .. } => false, + ExprKind::ListComp { generators, .. } + | ExprKind::SetComp { generators, .. } + | ExprKind::DictComp { generators, .. } => { + generators.first().is_some_and(|g| expr_contains_yield(&g.iter)) + } ExprKind::Starred(inner) => expr_contains_yield(inner), ExprKind::Constant(_) | ExprKind::Name(_) => false, } @@ -4918,6 +5057,21 @@ fn collect_reads_stmt(stmt: &Stmt, out: &mut HashSet) { collect_reads_stmt(s, out); } } + StmtKind::Delete(targets) => { + // `del x.attr` / `del x[i]` *read* the container `x` (it must be + // loaded to perform the delete), so the name must surface for + // free-variable promotion. A bare `del x` is a binding op, not a + // read — `collect_reads_assign_target` handles that distinction. + for t in targets { + collect_reads_assign_target(t, out); + } + } + StmtKind::Assert { test, msg } => { + collect_reads_expr(test, out); + if let Some(m) = msg { + collect_reads_expr(m, out); + } + } _ => {} } } diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 45ef626..2de2d93 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -477,6 +477,13 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "__setitem__" => Some(method("__setitem__", dict_setitem)), "__getitem__" => Some(method("__getitem__", dict_getitem)), "__delitem__" => Some(method("__delitem__", dict_delitem)), + // Mapping-protocol dunders exposed as bound methods so code can + // grab them directly — CPython's `functools._lru_cache_wrapper` + // caches `cache_len = cache.__len__`, and `__contains__` / + // `__iter__` round out `hasattr(d, …)` / explicit-call parity. + "__len__" => Some(method("__len__", obj_len)), + "__contains__" => Some(method("__contains__", obj_contains)), + "__iter__" => Some(method("__iter__", dict_iter_method)), "__init__" => Some(method("__init__", dict_update)), _ => None, }, @@ -559,7 +566,10 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "readline" => Some(method("readline", file_readline)), "readlines" => Some(method("readlines", file_readlines)), "write" => Some(method("write", file_write)), - "writelines" => Some(method("writelines", file_writelines)), + // Routed through the interpreter (sentinel name) so it can + // consume *any* iterable via the full `__iter__`/`__next__` + // protocol, not just native sequences. + "writelines" => Some(method(".file_writelines", file_writelines)), "flush" => Some(method("flush", file_flush)), "close" => Some(method("close", file_close)), "seek" => Some(method("seek", file_seek)), @@ -567,6 +577,13 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "getvalue" => Some(method("getvalue", file_getvalue)), "__enter__" => Some(method("__enter__", file_enter)), "__exit__" => Some(method("__exit__", file_exit)), + // A file is its own iterator (CPython): `iter(f) is f`, and + // each `next(f)` returns the next line, raising StopIteration + // at EOF. + "__iter__" => Some(method("__iter__", |args| { + file_self(args).map(Object::File) + })), + "__next__" => Some(method("__next__", file_next)), _ => None, }, Object::MemoryView(_) => match name { @@ -663,6 +680,13 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { .cloned() .ok_or_else(|| type_error("__iter__() missing self")) })), + // Pickling support. The actual reduction needs the canonical + // `iter` builtin (so the result pickles by name and round-trips), + // which requires interpreter access — the VM intercepts this + // sentinel name in its bound-method dispatch. + "__reduce__" => Some(method(".iter_reduce", |_| { + Err(type_error("iterator.__reduce__ requires the interpreter")) + })), _ => None, }, _ => None, @@ -1404,6 +1428,87 @@ pub fn construct_classmethod(args: &[Object]) -> Result { Ok(Object::ClassMethod(Rc::new(inner))) } +/// `staticmethod.__get__(self, obj, objtype=None)` — the descriptor hook. +/// A staticmethod ignores the binding context and hands back the wrapped +/// callable unchanged (matching CPython's `sm_descr_get`). Exposing it as +/// a real method lets descriptor-aware code — notably +/// `functools.partialmethod`, which does `self.func.__get__(obj, cls)` — +/// treat a wrapped `staticmethod` correctly. `args[0]` is the descriptor +/// itself (the bound receiver). +pub(crate) fn staticmethod_descr_get(args: &[Object]) -> Result { + match args.first() { + Some(Object::StaticMethod(inner)) => Ok((**inner).clone()), + // Tolerate an already-unwrapped callable (defensive). + Some(other) => Ok(other.clone()), + None => Err(type_error("staticmethod.__get__() missing self")), + } +} + +/// `classmethod.__get__(self, obj, objtype=None)` — binds the wrapped +/// callable to the owning *class* and returns a bound method (CPython's +/// `cm_descr_get`). The owner is the explicit `objtype` when supplied, +/// otherwise `type(obj)`. +pub(crate) fn classmethod_descr_get(args: &[Object]) -> Result { + let inner = match args.first() { + Some(Object::ClassMethod(i)) => (**i).clone(), + _ => return Err(type_error("classmethod.__get__() missing self")), + }; + let owner = match args.get(2) { + Some(o) if !matches!(o, Object::None) => o.clone(), + _ => match args.get(1) { + Some(o) if !matches!(o, Object::None) => Object::Type(class_of(o)), + _ => { + return Err(type_error( + "classmethod.__get__(None, None) is not valid", + )) + } + }, + }; + Ok(Object::BoundMethod(Rc::new(crate::object::BoundMethod { + receiver: owner, + function: inner, + }))) +} + +/// `function.__get__(self, obj, objtype=None)` — a plain Python function +/// is a non-data descriptor: bound to an instance it yields a bound +/// method, bound to `None` (class access) it returns the function itself +/// (CPython's `func_descr_get`). Exposing it makes functions usable with +/// descriptor-aware library code such as `functools.partialmethod`. +pub(crate) fn function_descr_get(args: &[Object]) -> Result { + let func = args + .first() + .cloned() + .ok_or_else(|| type_error("__get__() missing self"))?; + match args.get(1) { + Some(obj) if !matches!(obj, Object::None) => { + Ok(Object::BoundMethod(Rc::new(crate::object::BoundMethod { + receiver: obj.clone(), + function: func, + }))) + } + _ => Ok(func), + } +} + +/// Build the callable `Object::Builtin` backing `staticmethod.__get__` / +/// `classmethod.__get__`. The VM wires this into a `BoundMethod` whose +/// receiver is the descriptor object, so `args[0]` arrives as the +/// descriptor when the hook runs. +pub(crate) fn descriptor_get_builtin(is_static: bool) -> Object { + let f = if is_static { + method("__get__", staticmethod_descr_get) + } else { + method("__get__", classmethod_descr_get) + }; + Object::Builtin(Rc::new(f)) +} + +/// Build the callable `Object::Builtin` backing `function.__get__`. +pub(crate) fn function_get_builtin() -> Object { + Object::Builtin(Rc::new(method("__get__", function_descr_get))) +} + fn property_with( args: &[Object], which: crate::object::PropertyAttr, @@ -1527,6 +1632,9 @@ fn b_callable(args: &[Object]) -> Result { | Object::BoundMethod(_) | Object::Type(_) | Object::Generator(_) + // Since Python 3.10 (bpo-43682) `staticmethod` objects are + // themselves callable, forwarding to the wrapped function. + | Object::StaticMethod(_) ); if intrinsic { return Ok(Object::Bool(true)); @@ -3809,13 +3917,20 @@ fn b_sorted(args: &[Object]) -> Result { fn b_reversed(args: &[Object]) -> Result { let iterable = one(args, "reversed")?; + // Materialize the source in *forward* order; the Reversed iterator + // walks it back-to-front. (CPython's `reversed` uses `__reversed__` + // or `__len__`+`__getitem__`; a forward snapshot reproduces the same + // sequence for the finite iterables WeavePy handles here.) let mut it = iterable.make_iter()?; let mut buf = Vec::new(); while let Some(v) = it.next_value() { buf.push(v); } - buf.reverse(); - Ok(Object::new_list(buf)) + let index = buf.len() as i64 - 1; + Ok(Object::Iter(Rc::new(RefCell::new(PyIterator::Reversed { + items: Rc::new(RefCell::new(buf)), + index, + })))) } fn b_enumerate(args: &[Object]) -> Result { @@ -5901,6 +6016,16 @@ fn list_copy(args: &[Object]) -> Result { // ---------- dict methods ---------- +/// `dict.__iter__(self)` → a key iterator (CPython's `dict_iter`), so +/// `iter(d)` parity holds when the dunder is fetched explicitly. +fn dict_iter_method(args: &[Object]) -> Result { + let recv = args + .first() + .ok_or_else(|| type_error("__iter__() missing self"))?; + let it = recv.make_iter()?; + Ok(Object::Iter(Rc::new(RefCell::new(it)))) +} + fn dict_self(args: &[Object]) -> Result>, RuntimeError> { match args.first() { Some(Object::Dict(d)) => Ok(d.clone()), @@ -7093,6 +7218,22 @@ fn file_readline(args: &[Object]) -> Result { } } +/// `next(file)` — return the next line, or raise StopIteration at EOF. +/// Backs both the `__next__` method and the VM's native file iteration. +pub(crate) fn file_next(args: &[Object]) -> Result { + let line = file_readline(args)?; + let empty = match &line { + Object::Str(s) => s.is_empty(), + Object::Bytes(b) => b.is_empty(), + _ => true, + }; + if empty { + Err(stop_iteration()) + } else { + Ok(line) + } +} + fn file_readlines(args: &[Object]) -> Result { let f = file_self(args)?; let mut lines: Vec = Vec::new(); diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 9cf872c..51b7b95 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -270,6 +270,30 @@ impl GcState { || self.frozen.borrow().iter().any(|h| h.id == id) } + /// Snapshot every tracked object that still carries an unrun + /// `__del__`. The interpreter's shutdown pass walks this list to + /// finalize objects that are still alive at exit — CPython runs + /// finalizers for everything during interpreter teardown, not just + /// for cyclic garbage. The per-handle `finalized` flag (shared with + /// the cycle collector) guarantees each `__del__` runs at most once. + pub fn finalization_candidates(&self) -> Vec> { + let mut out = Vec::new(); + let gens = self.generations.borrow(); + for gen in gens.iter() { + for h in &gen.handles { + if !h.finalized.load(Ordering::Acquire) && has_finalizer(&h.object) { + out.push(h.clone()); + } + } + } + for h in self.frozen.borrow().iter() { + if !h.finalized.load(Ordering::Acquire) && has_finalizer(&h.object) { + out.push(h.clone()); + } + } + out + } + /// Number of tracked objects in each generation. pub fn counts(&self) -> [usize; N_GENERATIONS] { *self.counts.borrow() @@ -795,6 +819,12 @@ pub fn track(obj: Object) { with_state(|s| s.track(obj)); } +/// Convenience: snapshot all tracked objects with an unrun `__del__` +/// on the current thread's GC (see [`GcState::finalization_candidates`]). +pub fn finalization_candidates() -> Vec> { + with_state(|s| s.finalization_candidates()) +} + /// Convenience: run a full collection on the current thread's /// GC. Returns the number of objects collected. pub fn collect_all() -> usize { diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 7c37d86..aaf017b 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -88,6 +88,15 @@ struct Frame { /// pc *before* the current instruction — used to look up the /// exception handler when an opcode raises. pc: u32, + /// Persistent Python-visible frame snapshot for generator / + /// coroutine / async-generator frames. A generator is re-entered + /// on every `next()`/`send()`; CPython keeps a single `gi_frame` + /// object alive for the generator's whole lifetime, and + /// debuggers (`bdb`/`pdb`) rely on `frame is self.stopframe` + /// holding across suspensions. We cache the `Rc` here on + /// first entry and re-push the same object on each resume. + /// `None` for ordinary frames, which are only ever entered once. + py_frame: Option>, } impl Frame { @@ -626,8 +635,9 @@ impl Interpreter { /// Invoke any `__del__` finalizers queued by the cycle GC. /// Each finalizer runs at most once. Exceptions from a - /// finalizer are routed through `sys.unraisablehook` (today - /// just logged to stderr) so they don't propagate. + /// finalizer are routed through `sys.unraisablehook` (the + /// default hook prints `Exception ignored in: …` to stderr, + /// exactly like CPython) so they don't propagate. pub fn run_pending_finalizers(&mut self) { loop { let pending = crate::vm_singletons::drain_pending_finalizers(); @@ -635,21 +645,202 @@ impl Interpreter { return; } for obj in pending { - if let Object::Instance(inst) = &obj { - if let Some(del) = inst.class.lookup("__del__") { - let bound = Object::BoundMethod(Rc::new(BoundMethod { - receiver: obj.clone(), - function: del, - })); - let kwargs: Vec<(String, Object)> = Vec::new(); - let outer = Rc::new(RefCell::new(DictData::new())); - let _ = self.call(&bound, &[], &kwargs, &outer); - } + self.invoke_finalizer(&obj); + } + } + } + + /// Run finalizers (`__del__`) for every object still alive at + /// interpreter shutdown. CPython finalizes *all* reachable objects + /// during teardown, not just cyclic garbage, so a module-global + /// instance (`module.x = C()`) still has its `__del__` called. We + /// walk the cycle collector's tracked set — every user instance is + /// tracked at construction — and run any unrun finalizer, routing + /// errors through `sys.unraisablehook`. A bounded number of passes + /// lets a finalizer that resurrects/creates new finalizable objects + /// settle without risking an unbounded loop at exit. + pub fn run_shutdown_finalizers(&mut self) { + self.run_pending_finalizers(); + for _ in 0..8 { + let candidates = crate::gc_trace::finalization_candidates(); + if candidates.is_empty() { + break; + } + for handle in candidates { + // `swap` claims the finalizer so the cycle collector + // and a later shutdown pass can't double-run it. + if handle.finalized.swap(true, std::sync::atomic::Ordering::AcqRel) { + continue; } + self.invoke_finalizer(&handle.object); } + // A finalizer may have queued cyclic finalizers of its own. + self.run_pending_finalizers(); } } + /// Call `obj.__del__()` if present, routing any raised exception + /// through the unraisable hook. Used by both the cycle-GC drain and + /// the shutdown pass. + fn invoke_finalizer(&mut self, obj: &Object) { + let Object::Instance(inst) = obj else { + return; + }; + let Some(del) = inst.class.lookup("__del__") else { + return; + }; + let class_name = inst.class.name.clone(); + let bound = Object::BoundMethod(Rc::new(BoundMethod { + receiver: obj.clone(), + function: del, + })); + let kwargs: Vec<(String, Object)> = Vec::new(); + let outer = Rc::new(RefCell::new(DictData::new())); + if let Err(err) = self.call(&bound, &[], &kwargs, &outer) { + // The finalizer is the `object` reported to the hook; the + // printed context mirrors CPython's bound-method repr so a + // default hook emits `… `. + let receiver_repr = self + .repr_of(obj, &outer) + .unwrap_or_else(|_| obj.repr()); + let context_repr = format!(""); + self.write_unraisable(&err, &bound, &context_repr); + } + } + + /// Route an out-of-band exception (raised by a `__del__` finalizer, + /// and in future weakref callbacks) through `sys.unraisablehook`, + /// mirroring CPython's `_PyErr_WriteUnraisable`. A user-installed + /// hook (e.g. `test.support.catch_unraisable_exception`) is invoked + /// with an `UnraisableHookArgs`-shaped namespace; the default hook + /// prints `Exception ignored in: ` plus the traceback to + /// stderr and swallows the error so it can't change the exit status. + fn write_unraisable(&mut self, err: &RuntimeError, object: &Object, context_repr: &str) { + let (exc_value, exc_type, traceback) = match err { + RuntimeError::PyException(pyexc) => { + let inst = pyexc.instance.clone(); + let ty = match &inst { + Object::Instance(i) => Object::Type(i.class.clone()), + _ => Object::None, + }; + (inst, ty, pyexc.traceback.clone()) + } + other => { + let inst = + crate::builtin_types::make_exception("RuntimeError", other.to_string()); + let ty = match &inst { + Object::Instance(i) => Object::Type(i.class.clone()), + _ => Object::None, + }; + (inst, ty, Vec::new()) + } + }; + + // Honour a user-installed `sys.unraisablehook`. Assignment to + // `sys.unraisablehook` updates the `sys` module dict (not the + // VM's reserved slot), so read it from there. + let hook = { + let sys_module = self + .cache + .modules + .borrow() + .get(&DictKey(Object::from_static("sys"))) + .cloned(); + match sys_module { + Some(Object::Module(m)) => m + .dict + .borrow() + .get(&DictKey(Object::from_static("unraisablehook"))) + .cloned(), + _ => None, + } + }; + let is_default = match &hook { + None | Some(Object::None) => true, + // The built-in placeholder installed at startup is a no-op; + // treat it as "default" and print ourselves. + Some(Object::Builtin(b)) => b.name == "unraisablehook", + _ => false, + }; + if !is_default { + if let Some(hook) = hook { + let args_obj = + self.make_unraisable_args(&exc_type, &exc_value, object); + let outer = self.builtins_dict(); + if self.call(&hook, &[args_obj], &[], &outer).is_err() { + self.print_unraisable_default(&exc_value, &traceback, context_repr, true); + } + return; + } + } + self.print_unraisable_default(&exc_value, &traceback, context_repr, false); + } + + /// Build the `UnraisableHookArgs`-shaped object passed to a custom + /// `sys.unraisablehook` — a namespace exposing `exc_type`, + /// `exc_value`, `exc_traceback`, `err_msg`, and `object`. + fn make_unraisable_args( + &self, + exc_type: &Object, + exc_value: &Object, + object: &Object, + ) -> Object { + let mut d = DictData::new(); + d.insert(DictKey(Object::from_static("exc_type")), exc_type.clone()); + d.insert(DictKey(Object::from_static("exc_value")), exc_value.clone()); + d.insert( + DictKey(Object::from_static("exc_traceback")), + Object::None, + ); + d.insert(DictKey(Object::from_static("err_msg")), Object::None); + d.insert(DictKey(Object::from_static("object")), object.clone()); + Object::SimpleNamespace(Rc::new(RefCell::new(d))) + } + + /// CPython's default `sys.unraisablehook`: write the + /// `Exception ignored in: …` header, the traceback, and the + /// exception line to stderr. + fn print_unraisable_default( + &self, + exc_value: &Object, + traceback: &[crate::error::TracebackEntry], + context_repr: &str, + hook_failed: bool, + ) { + use std::io::Write; + let mut s = String::new(); + if hook_failed { + s.push_str("Exception ignored in sys.unraisablehook: "); + } else { + s.push_str("Exception ignored in: "); + } + s.push_str(context_repr); + s.push('\n'); + if !traceback.is_empty() { + s.push_str("Traceback (most recent call last):\n"); + for e in traceback { + s.push_str(&format!( + " File \"{}\", line {}, in {}\n", + e.filename, e.lineno, e.funcname + )); + } + } + let (kind, msg) = match exc_value { + Object::Instance(i) => ( + i.class.name.clone(), + crate::builtin_types::exception_message(exc_value).unwrap_or_default(), + ), + _ => ("Exception".to_owned(), String::new()), + }; + if msg.is_empty() { + s.push_str(&format!("{kind}\n")); + } else { + s.push_str(&format!("{kind}: {msg}\n")); + } + let mut stderr = std::io::stderr().lock(); + let _ = stderr.write_all(s.as_bytes()); + } + /// Public re-export of [`Self::build_module_globals`] used by the /// `interpreters` module to seed a fresh `__main__` dict for a /// sub-interpreter (RFC 0031 — PEP 684). @@ -751,6 +942,7 @@ impl Interpreter { class_namespace: None, exc_handlers: Vec::new(), pc: 0, + py_frame: None, } } @@ -814,12 +1006,46 @@ impl Interpreter { // activation leaves un-popped (see the reconciliation at the // function's exit). let exc_depth_on_entry = self.exc_info_stack.borrow().len(); + // Distinguish the three ways control can enter a frame here: + // + // * a fresh ordinary call (pc == 0, not gen code) + // * a generator/coroutine bootstrap (pc == 0, gen code) — the + // `RETURN_GENERATOR` prologue that merely *creates* the + // suspended object; CPython runs no user code and fires no + // trace events for it. + // * a generator/coroutine *resume* (pc != 0, gen code) — a + // real `next()`/`send()` that fires a `call` event and then + // continues line tracing from the suspension point. + let code_is_gen = frame.code.is_generator + || frame.code.is_coroutine + || frame.code.is_async_generator; + let is_gen_bootstrap = code_is_gen && frame.pc == 0; + let is_gen_resume = code_is_gen && frame.pc != 0; // RFC 0031 — fire the `'call'` event on frame entry. The // hook's return value becomes the per-frame trace function - // for subsequent line / return / exception events. + // for subsequent line / return / exception events. The + // generator-creation bootstrap is invisible to tracers. let observers_active = crate::trace::any_observers_active(); - if observers_active { + if observers_active && !is_gen_bootstrap { self.fire_call_event(&py_frame)?; + // On a resume, line tracing must continue from the line + // where the frame suspended — CPython reports the `call` + // event at the suspension line and only emits the next + // `line` event once execution crosses into a *different* + // line. Seed `last_line` with the resume line so the + // prologue (`POP_TOP; RESUME`) and the suspension line + // itself don't surface as a spurious `line` event. + if is_gen_resume { + let line = frame + .code + .linetable + .get(frame.pc as usize) + .copied() + .unwrap_or(0); + if line != 0 { + py_frame.last_line.set(Some(line)); + } + } } // RFC 0032 — tier-2 entry. Only for a fresh activation (pc 0, // empty stack, not a generator resume) and only when tracing is @@ -846,12 +1072,38 @@ impl Interpreter { self.sync_py_locals(frame); // Fire a 'line' event when the source line changes. // Fast path: skip the line-table read entirely when no - // observer is active. - if crate::trace::any_observers_active() { + // observer is active. The generator-creation bootstrap + // (`RETURN_GENERATOR`) fires no line events either. + if crate::trace::any_observers_active() && !is_gen_bootstrap { let line = py_frame.current_lineno(); - if line != 0 && py_frame.last_line.get() != Some(line) { + // CPython never emits a `line` event for the + // frame-entry `RESUME`: the `call` event covers entry + // and line tracing begins at the first *real* + // instruction (the first body line). Firing here would + // inject a spurious `line` event at the `def` line, + // desyncing trace consumers like `bdb`/`pdb`. We skip + // firing *and* leave `last_line` untouched so a one-line + // body (`def f(): return 1`) still reports its single + // line event from the following instruction. + let at_resume = matches!( + frame.code.instructions.get(frame.pc as usize).map(|i| i.op), + Some(OpCode::Resume) + ); + if !at_resume && line != 0 && py_frame.last_line.get() != Some(line) { py_frame.last_line.set(Some(line)); - self.fire_line_event(&py_frame)?; + // `f_trace_lines = False` suppresses the callback but the + // line bookkeeping above still advances (mirrors CPython, + // which keeps tracking the line for later re-enable). + if py_frame.trace_lines.get() { + self.fire_line_event(&py_frame)?; + } + } + // A `'line'` callback may have just enabled opcode tracing + // (bdb's `stepinstr`); CPython then fires the `'opcode'` + // event for this same instruction before it runs. The + // frame-entry RESUME carries no opcode event. + if !at_resume && py_frame.trace_opcodes.get() { + self.fire_opcode_event(&py_frame)?; } } match self.step(frame) { @@ -877,7 +1129,18 @@ impl Interpreter { match self.handle_exception(frame, exc) { Ok(Some(())) => continue, Ok(None) => unreachable!(), - Err(e) => break Err(e), + Err(e) => { + // No handler in this frame: it is about to be + // popped by the propagating exception. CPython + // delivers a `'return'` event with arg `None` + // to the trace/profile callbacks on this unwind + // path (sys.monitoring sees PY_UNWIND). bdb's + // `set_return` waits for exactly this event. + if crate::trace::any_observers_active() { + self.fire_unwind_event(&py_frame)?; + } + break Err(e); + } } } else { break Err(err); @@ -913,7 +1176,22 @@ impl Interpreter { /// interpreter's call stack. The snapshot's `back` chain points /// at whatever was on top of the stack before the push, so the /// call hierarchy is recoverable from any frame. - fn push_py_frame(&self, frame: &Frame) -> Rc { + fn push_py_frame(&self, frame: &mut Frame) -> Rc { + // Generator-family frames are re-entered on every resume. + // Reuse the cached `PyFrame` so the Python-visible frame keeps + // a stable identity across suspensions (CPython's `gi_frame`), + // only refreshing the bits that change per resume: the `back` + // pointer (who's resuming us now) and `lasti`. The locals + // mirror is shared by reference and kept current by + // `sync_py_locals`, so we just drop the materialised cache. + if let Some(existing) = frame.py_frame.clone() { + let back = self.frame_stack.borrow().last().cloned(); + *existing.back.borrow_mut() = back; + existing.lasti.set(frame.pc); + existing.invalidate_locals(); + self.frame_stack.borrow_mut().push(existing.clone()); + return existing; + } let varnames = frame.code.varnames.clone(); let locals_snapshot = Rc::new(RefCell::new(frame.locals.clone())); let cell_names: Vec = frame @@ -990,7 +1268,16 @@ impl Interpreter { trace: RefCell::new(Object::None), override_lineno: Cell::new(None), last_line: Cell::new(None), + trace_lines: Cell::new(true), + trace_opcodes: Cell::new(false), }); + // Cache the snapshot on generator-family frames so the next + // resume re-pushes this very object (stable identity). Plain + // function frames run exactly once and are never re-entered, + // so caching them would only waste a clone. + if frame.code.is_generator || frame.code.is_coroutine || frame.code.is_async_generator { + frame.py_frame = Some(py.clone()); + } self.frame_stack.borrow_mut().push(py.clone()); py } @@ -1034,6 +1321,7 @@ impl Interpreter { py_frame: &Rc, event: &'static str, arg: Object, + kind: crate::trace::HookKind, ) -> Result { let _guard = match crate::trace::ReentryGuard::acquire() { Some(g) => g, @@ -1045,15 +1333,38 @@ impl Interpreter { arg, ]; let outer = self.builtins.clone(); - // Errors from the hook are deliberately swallowed in CPython - // (it disables the hook and prints to stderr). We mirror - // that behaviour: a hook crash should never take down the - // user program. We do let `RuntimeError::PyException` rise - // when the hook is observing a user-raised exception so the - // exception propagation in the caller stays intact. + // CPython's trace/profile contract for a raised exception: + // + // * On an `'exception'` event the exception is discarded + // (`call_trace_protected` / `call_exc_trace`) so that it + // doesn't clobber the exception already being propagated. + // * On every other event (`'call'`, `'line'`, `'return'`, + // `'opcode'`) the exception propagates into the traced + // program *and* tracing is turned off (the offending + // trace/profile function is cleared) so no further events + // fire while the exception unwinds. + // + // Clearing the global hook is sufficient to silence all + // subsequent events: the dispatcher gates every callout on the + // hook being installed (see `any_observers_active`). match self.call(hook, &args, &[], &outer) { Ok(v) => Ok(v), - Err(RuntimeError::PyException(_)) => Ok(Object::None), + Err(RuntimeError::PyException(exc)) => { + if event == "exception" { + Ok(Object::None) + } else { + match kind { + crate::trace::HookKind::Trace => { + crate::trace::set_trace_hook(Object::None); + *py_frame.trace.borrow_mut() = Object::None; + } + crate::trace::HookKind::Profile => { + crate::trace::set_profile_hook(Object::None); + } + } + Err(RuntimeError::PyException(exc)) + } + } Err(other) => Err(other), } } @@ -1062,11 +1373,23 @@ impl Interpreter { /// the returned per-frame trace function (settrace contract). fn fire_call_event(&mut self, py_frame: &Rc) -> Result<(), RuntimeError> { if let Some(trace) = crate::trace::trace_hook() { - let result = self.invoke_observe_hook(&trace, py_frame, "call", Object::None)?; + let result = self.invoke_observe_hook( + &trace, + py_frame, + "call", + Object::None, + crate::trace::HookKind::Trace, + )?; *py_frame.trace.borrow_mut() = result; } if let Some(profile) = crate::trace::profile_hook() { - let _ = self.invoke_observe_hook(&profile, py_frame, "call", Object::None)?; + let _ = self.invoke_observe_hook( + &profile, + py_frame, + "call", + Object::None, + crate::trace::HookKind::Profile, + )?; } self.fire_monitoring_event(py_frame, crate::trace::EVENT_PY_START, Object::None)?; Ok(()) @@ -1076,7 +1399,13 @@ impl Interpreter { fn fire_line_event(&mut self, py_frame: &Rc) -> Result<(), RuntimeError> { let frame_trace = py_frame.trace.borrow().clone(); if !matches!(frame_trace, Object::None) { - let result = self.invoke_observe_hook(&frame_trace, py_frame, "line", Object::None)?; + let result = self.invoke_observe_hook( + &frame_trace, + py_frame, + "line", + Object::None, + crate::trace::HookKind::Trace, + )?; // Per CPython: the local trace function may return a new // local trace for subsequent line events. *py_frame.trace.borrow_mut() = result; @@ -1085,6 +1414,27 @@ impl Interpreter { Ok(()) } + /// Fire the `'opcode'` event before an instruction executes. Only + /// reached when the frame opted in via `f_trace_opcodes = True` + /// (CPython's per-instruction stepping used by `bdb`/`pdb`). Like + /// `'line'`, the callback's return value replaces the per-frame + /// trace; `sys.monitoring` observes INSTRUCTION. + fn fire_opcode_event(&mut self, py_frame: &Rc) -> Result<(), RuntimeError> { + let frame_trace = py_frame.trace.borrow().clone(); + if !matches!(frame_trace, Object::None) { + let result = self.invoke_observe_hook( + &frame_trace, + py_frame, + "opcode", + Object::None, + crate::trace::HookKind::Trace, + )?; + *py_frame.trace.borrow_mut() = result; + } + self.fire_monitoring_event(py_frame, crate::trace::EVENT_INSTRUCTION, Object::None)?; + Ok(()) + } + /// Fire the `'return'` event when a frame returns normally. fn fire_return_event( &mut self, @@ -1093,10 +1443,22 @@ impl Interpreter { ) -> Result<(), RuntimeError> { let frame_trace = py_frame.trace.borrow().clone(); if !matches!(frame_trace, Object::None) { - let _ = self.invoke_observe_hook(&frame_trace, py_frame, "return", value.clone())?; + let _ = self.invoke_observe_hook( + &frame_trace, + py_frame, + "return", + value.clone(), + crate::trace::HookKind::Trace, + )?; } if let Some(profile) = crate::trace::profile_hook() { - let _ = self.invoke_observe_hook(&profile, py_frame, "return", value.clone())?; + let _ = self.invoke_observe_hook( + &profile, + py_frame, + "return", + value.clone(), + crate::trace::HookKind::Profile, + )?; } self.fire_monitoring_event(py_frame, crate::trace::EVENT_PY_RETURN, value.clone())?; Ok(()) @@ -1112,7 +1474,13 @@ impl Interpreter { ) -> Result<(), RuntimeError> { let frame_trace = py_frame.trace.borrow().clone(); if !matches!(frame_trace, Object::None) { - let _ = self.invoke_observe_hook(&frame_trace, py_frame, "return", Object::None)?; + let _ = self.invoke_observe_hook( + &frame_trace, + py_frame, + "return", + Object::None, + crate::trace::HookKind::Trace, + )?; } self.fire_monitoring_event(py_frame, crate::trace::EVENT_PY_YIELD, value.clone())?; Ok(()) @@ -1134,12 +1502,69 @@ impl Interpreter { _ => Object::None, }; let arg = Object::new_tuple(vec![exc_type, exc.instance.clone(), Object::None]); - let _ = self.invoke_observe_hook(&frame_trace, py_frame, "exception", arg)?; + let _ = self.invoke_observe_hook( + &frame_trace, + py_frame, + "exception", + arg, + crate::trace::HookKind::Trace, + )?; } self.fire_monitoring_event(py_frame, crate::trace::EVENT_RAISE, exc.instance.clone())?; Ok(()) } + /// Surface a `StopIteration` that an opcode catches *inline* to the + /// trace `'exception'` hook. CPython lets such a `StopIteration` + /// reach the trace machinery — firing an `'exception'` event in the + /// current frame — before the opcode clears it and continues: + /// * `FOR_ITER`: the iterator's `StopIteration` ends the loop. + /// * `SEND` (`yield from`/`await`): the sub-iterator's + /// `StopIteration` carries the delegated `return` value. + /// Built-in iterators (`list`/`tuple`/`range`) don't raise, so only + /// the generator / custom-`__next__` paths call this. Fires nothing + /// on the fast no-observer path. + fn fire_caught_stop_iteration(&mut self, exc: &PyException) -> Result<(), RuntimeError> { + if !crate::trace::any_observers_active() { + return Ok(()); + } + let py_frame = self.frame_stack.borrow().last().cloned(); + if let Some(py_frame) = py_frame { + self.fire_exception_event(&py_frame, exc)?; + } + Ok(()) + } + + /// Fire the local trace function's `'return'` event (with `arg` + /// `None`) when a frame is popped because an exception is + /// propagating out of it. CPython delivers a `PyTrace_RETURN` / + /// `None` to both the `settrace` and `setprofile` callbacks on this + /// unwind path, while `sys.monitoring` observes `PY_UNWIND` rather + /// than `PY_RETURN`. Fires nothing on the fast no-observer path. + fn fire_unwind_event(&mut self, py_frame: &Rc) -> Result<(), RuntimeError> { + let frame_trace = py_frame.trace.borrow().clone(); + if !matches!(frame_trace, Object::None) { + let _ = self.invoke_observe_hook( + &frame_trace, + py_frame, + "return", + Object::None, + crate::trace::HookKind::Trace, + )?; + } + if let Some(profile) = crate::trace::profile_hook() { + let _ = self.invoke_observe_hook( + &profile, + py_frame, + "return", + Object::None, + crate::trace::HookKind::Profile, + )?; + } + self.fire_monitoring_event(py_frame, crate::trace::EVENT_PY_UNWIND, Object::None)?; + Ok(()) + } + /// Record an object allocation with `tracemalloc`. Fast path /// short-circuits when tracking is disabled (the common case). /// `nbytes` is the object's approximate footprint as reported @@ -1602,6 +2027,10 @@ impl Interpreter { // Exceptions raised while iterating propagate. self.contains_via_iter(&container, &item, &frame.globals.clone())? } + } else if matches!(&container, Object::File(_)) { + // A file has no native `contains`; CPython tests + // membership by iterating its lines (`line in f`). + self.contains_via_iter(&container, &item, &frame.globals.clone())? } else { container.contains(&item)? }; @@ -1744,6 +2173,12 @@ impl Interpreter { Err(RuntimeError::PyException(exc)) if exc.type_name() == "StopIteration" => { + // CPython surfaces the terminating + // `StopIteration` to the trace `'exception'` + // hook (in *this* frame, at the `for` line) + // before `FOR_ITER` swallows it. bdb/pdb need + // this event to stop on a generator's exit. + self.fire_caught_stop_iteration(&exc)?; None } Err(e) => return Err(e), @@ -1756,6 +2191,7 @@ impl Interpreter { Err(RuntimeError::PyException(exc)) if exc.type_name() == "StopIteration" => { + self.fire_caught_stop_iteration(&exc)?; None } Err(e) => return Err(e), @@ -1918,11 +2354,22 @@ impl Interpreter { let globals = frame.globals.clone(); self.collect_iterable(&v, &globals)? } + // Any other value: defer to the general iterator + // protocol (covers `dict_keys`/`dict_values`/`dict_items` + // views, `map`/`filter`/`zip`, etc.). Only a genuine + // non-iterable becomes the unpack TypeError. _ => { - return Err(type_error(format!( - "cannot unpack non-iterable {} object", - v.type_name() - ))) + let globals = frame.globals.clone(); + match self.collect_iterable(&v, &globals) { + Ok(items) => items, + Err(e) if is_type_error(&e) => { + return Err(type_error(format!( + "cannot unpack non-iterable {} object", + v.type_name() + ))) + } + Err(e) => return Err(e), + } } }; // CPython distinguishes the two arity errors: it stops @@ -1984,11 +2431,21 @@ impl Interpreter { let globals = frame.globals.clone(); self.collect_iterable(&v, &globals)? } + // Defer any other value to the general iterator protocol + // (dict views, map/filter/zip, …); only a genuine + // non-iterable becomes the unpack TypeError. _ => { - return Err(type_error(format!( - "cannot unpack non-iterable {} object", - v.type_name() - ))) + let globals = frame.globals.clone(); + match self.collect_iterable(&v, &globals) { + Ok(items) => items, + Err(e) if is_type_error(&e) => { + return Err(type_error(format!( + "cannot unpack non-iterable {} object", + v.type_name() + ))) + } + Err(e) => return Err(e), + } } }; if items.len() < before + after { @@ -2438,6 +2895,14 @@ impl Interpreter { // propagate so the surrounding async-for's // exception handler (END_ASYNC_FOR) can clean // up. + // + // The delegated `StopIteration` is briefly visible + // in *this* (delegating) frame before SEND swallows + // it to extract the `yield from`/`await` value, so + // CPython fires an `'exception'` trace event at the + // `yield from` line. bdb/pdb rely on this to stop a + // `return`/`next` command at the delegating frame. + self.fire_caught_stop_iteration(&exc)?; let payload = exception_value(&exc.instance); frame.push(payload); frame.pc += ins.arg; @@ -2650,6 +3115,8 @@ impl Interpreter { trace: RefCell::new(Object::None), override_lineno: Cell::new(None), last_line: Cell::new(None), + trace_lines: Cell::new(true), + trace_opcodes: Cell::new(false), }) }); let new_tb = Rc::new(PyTraceback { @@ -2930,6 +3397,14 @@ impl Interpreter { Object::StaticMethod(inner) => match name { // `__func__`/`__wrapped__` expose the wrapped callable. "__func__" | "__wrapped__" => Ok((**inner).clone()), + // The descriptor hook. `staticmethod` is a non-data + // descriptor; `sm.__get__(obj, cls)` returns the wrapped + // function. Descriptor-aware library code (e.g. + // `functools.partialmethod`) relies on this being present. + "__get__" => Ok(Object::BoundMethod(Rc::new(BoundMethod { + receiver: Object::StaticMethod(inner.clone()), + function: crate::builtins::descriptor_get_builtin(true), + }))), "__isabstractmethod__" => { // Honour an `@abstractmethod` decorator applied // *under* `@staticmethod` (`@staticmethod @@ -2951,6 +3426,13 @@ impl Interpreter { // `__func__` and `__wrapped__` both expose the underlying // callable; `functools.wraps`/inspect walk `__wrapped__`. "__func__" | "__wrapped__" => Ok((**inner).clone()), + // The descriptor hook. `cm.__get__(obj, cls)` returns the + // wrapped callable bound to the owning class. Library code + // such as `functools.partialmethod` invokes it directly. + "__get__" => Ok(Object::BoundMethod(Rc::new(BoundMethod { + receiver: Object::ClassMethod(inner.clone()), + function: crate::builtins::descriptor_get_builtin(false), + }))), "__isabstractmethod__" => Ok(self .load_attr(inner.as_ref(), "__isabstractmethod__") .unwrap_or(Object::Bool(false))), @@ -3104,6 +3586,19 @@ impl Interpreter { f.attrs.borrow_mut().insert(key, d.clone()); return Ok(d); } + // PEP 695: every function carries `__type_params__` + // (an empty tuple unless it declares type parameters). + "__type_params__" => return Ok(Object::new_tuple(vec![])), + // A function is a non-data descriptor; `f.__get__(obj, + // cls)` binds it. Exposing the hook lets descriptor-aware + // code (`functools.partialmethod`, custom descriptors) + // treat a plain function uniformly with methods. + "__get__" => { + return Ok(Object::BoundMethod(Rc::new(BoundMethod { + receiver: obj.clone(), + function: crate::builtins::function_get_builtin(), + }))) + } _ => {} } Err(attribute_error(format!( @@ -3126,14 +3621,16 @@ impl Interpreter { "f_builtins" => Ok(Object::Dict(fr.builtins.clone())), "f_locals" => Ok(fr.locals()), "f_lineno" => Ok(Object::Int(i64::from(fr.current_lineno()))), - "f_lasti" => Ok(Object::Int(i64::from(fr.lasti.get()))), + "f_lasti" => Ok(Object::Int(i64::from( + fr.code.cpython_lasti(fr.lasti.get()), + ))), "f_back" => match fr.back.borrow().as_ref() { Some(parent) => Ok(Object::Frame(parent.clone())), None => Ok(Object::None), }, "f_trace" => Ok(fr.trace.borrow().clone()), - "f_trace_lines" => Ok(Object::Bool(true)), - "f_trace_opcodes" => Ok(Object::Bool(false)), + "f_trace_lines" => Ok(Object::Bool(fr.trace_lines.get())), + "f_trace_opcodes" => Ok(Object::Bool(fr.trace_opcodes.get())), _ => Err(attribute_error(format!( "'frame' object has no attribute '{}'", name @@ -3142,7 +3639,9 @@ impl Interpreter { Object::Traceback(tb) => match name { "tb_frame" => Ok(Object::Frame(tb.frame.clone())), "tb_lineno" => Ok(Object::Int(i64::from(tb.lineno))), - "tb_lasti" => Ok(Object::Int(i64::from(tb.lasti))), + "tb_lasti" => Ok(Object::Int(i64::from( + tb.frame.code.cpython_lasti(tb.lasti), + ))), "tb_next" => match tb.next.borrow().as_ref() { Some(n) => Ok(Object::Traceback(n.clone())), None => Ok(Object::None), @@ -4772,6 +5271,11 @@ impl Interpreter { // statement instead of losing the value to `iter_next`'s // exhausted-or-not boolean. match it { + // A coroutine drives itself as its own awaitable iterator + // (WeavePy returns the coroutine from ``__await__``), so + // ``next(coro)`` advances it with ``send(None)`` exactly like + // a generator. CPython hides this behind a ``coroutine_wrapper``; + // here the coroutine *is* the wrapper. Object::Generator(g) => match self.generator_send(g, Object::None) { Ok(v) => Ok(v), Err(RuntimeError::PyException(exc)) if exc.type_name() == "StopIteration" => { @@ -4783,6 +5287,17 @@ impl Interpreter { } Err(e) => Err(e), }, + Object::Coroutine(_) => match self.gen_method_send(it, Object::None) { + Ok(v) => Ok(v), + Err(RuntimeError::PyException(exc)) if exc.type_name() == "StopIteration" => { + if let Some(d) = default { + Ok(d) + } else { + Err(RuntimeError::PyException(exc)) + } + } + Err(e) => Err(e), + }, _ => match self.iter_next(it, globals) { Ok(Some(v)) => Ok(v), Ok(None) => default.ok_or_else(stop_iteration), @@ -4799,25 +5314,28 @@ impl Interpreter { self.make_iter(v, globals) } - /// `iter(callable, sentinel)` — eagerly drains the callable in - /// a tight loop, building a list. Simpler than synthesising a - /// generator and matches the documented CPython semantics for - /// the common usage pattern (read-until-sentinel). The - /// resulting list iterator behaves identically for all - /// finite-sentinel cases; infinite sequences with this form - /// would also hang in CPython. + /// `iter(callable, sentinel)` — returns a *lazy* callable-iterator + /// (CPython's `calliterobject`). Each `next()` invokes the callable + /// once and stops when a result equals the sentinel. Driving it + /// lazily (rather than eagerly draining into a list) is required for + /// correctness, not just performance: an exception raised by the + /// callable must surface mid-stream, a `StopIteration` it raises must + /// end iteration, and an unbounded source must not hang at + /// construction. The behaviour lives in the frozen `_CallableIter` + /// helper, driven through the normal `__next__` dispatch. fn do_iter_callable_sentinel( &mut self, args: &[Object], globals: &Rc>, ) -> Result { + if let Some(it) = self.make_callable_iterator(&args[0], &args[1], globals)? { + return Ok(it); + } + // Fallback (helper module unavailable): eagerly drain. Retains the + // historical behaviour for the common finite-sentinel case. let callable = args[0].clone(); let sentinel = args[1].clone(); let mut out: Vec = Vec::new(); - // CPython caps the number of iterations at a very large - // value to keep accidental infinite loops bounded; we use - // i64::MAX iterations as the safety limit but in practice - // expect the sentinel to fire much sooner. for _ in 0_i64..i64::MAX { let v = self.call(&callable, &[], &[], globals)?; if self.dispatch_compare_op(&v, &sentinel, CompareKind::Eq, globals)? { @@ -4829,6 +5347,38 @@ impl Interpreter { self.make_iter(&list, globals) } + /// Instantiate the frozen `_CallableIter` for `iter(callable, sentinel)`. + /// Returns `None` if the helper module can't be imported, letting the + /// caller fall back to eager draining. + fn make_callable_iterator( + &mut self, + callable: &Object, + sentinel: &Object, + globals: &Rc>, + ) -> Result, RuntimeError> { + let module = match self.do_import("_seqtools", &Object::None, 0, globals) { + Ok(m) => m, + Err(_) => return Ok(None), + }; + let cls = match &module { + Object::Module(m) => m + .dict + .borrow() + .get(&DictKey(Object::from_static("_CallableIter"))) + .cloned(), + _ => None, + }; + match cls { + Some(cls) => Ok(Some(self.call( + &cls, + &[callable.clone(), sentinel.clone()], + &[], + globals, + )?)), + None => Ok(None), + } + } + fn do_list_or_tuple_call( &mut self, name: &str, @@ -5754,8 +6304,13 @@ impl Interpreter { Err(e) => return Err(e), }; while let Some(x) = self.iter_next(&it, globals)? { + // CPython compares the *element* against the search target + // (`PyObject_RichCompareBool(obj, item, Py_EQ)`), so the + // element's `__eq__` runs first. Order matters for asymmetric + // `__eq__` (e.g. an element that compares unequal to everything + // must not be "found" by a target that compares equal to all). if item.is_same(&x) - || self.dispatch_compare_op(item, &x, CompareKind::Eq, globals)? + || self.dispatch_compare_op(&x, item, CompareKind::Eq, globals)? { return Ok(true); } @@ -5763,6 +6318,33 @@ impl Interpreter { Ok(false) } + /// Construct the lazy legacy-`__getitem__` iterator (`_SeqIter`) + /// for `seq`. Returns `Ok(None)` only if the internal `_seqtools` + /// helper module is somehow unavailable, letting the caller fall + /// back; in practice it is always frozen in. + fn make_seq_iterator( + &mut self, + seq: &Object, + globals: &Rc>, + ) -> Result, RuntimeError> { + let module = match self.do_import("_seqtools", &Object::None, 0, globals) { + Ok(m) => m, + Err(_) => return Ok(None), + }; + let cls = match &module { + Object::Module(m) => m + .dict + .borrow() + .get(&DictKey(Object::from_static("_SeqIter"))) + .cloned(), + _ => None, + }; + match cls { + Some(cls) => Ok(Some(self.call(&cls, std::slice::from_ref(seq), &[], globals)?)), + None => Ok(None), + } + } + fn make_iter( &mut self, v: &Object, @@ -5772,28 +6354,33 @@ impl Interpreter { Object::Generator(_) | Object::Iter(_) => Ok(v.clone()), Object::Instance(_) => { if let Some(method) = instance_method(v, "__iter__") { - return self.call(&method, &[], &[], globals); + let result = self.call(&method, &[], &[], globals)?; + // CPython requires `__iter__` to return an actual + // iterator: a result lacking `__next__` is a TypeError + // ("iter() returned non-iterator of type 'X'"), surfaced + // here rather than deferred to the first `next()`. + if let Object::Instance(_) = &result { + if instance_method(&result, "__next__").is_none() { + return Err(type_error(format!( + "iter() returned non-iterator of type '{}'", + result.type_name_owned() + ))); + } + } + return Ok(result); } // Legacy sequence protocol: an object that defines // `__getitem__` but no `__iter__` is still iterable — // CPython calls `obj[0]`, `obj[1]`, … until `IndexError`. - // We materialise eagerly into a list (consistent with the - // `iter(callable, sentinel)` path above); the wrapped - // sequences this serves — `re`'s `SubPattern`, simple - // user containers — are finite and side-effect-free. - if let Some(getitem) = instance_method(v, "__getitem__") { - let mut out: Vec = Vec::new(); - let mut i: i64 = 0; - loop { - match self.call(&getitem, &[Object::Int(i)], &[], globals) { - Ok(val) => out.push(val), - Err(e) if is_index_error(&e) => break, - Err(e) => return Err(e), - } - i += 1; + // Return a *lazy* iterator (CPython's seqiterobject) so an + // unbounded sequence iterates forever instead of hanging + // at construction, and each element's side effects happen + // on demand. `_SeqIter` is a frozen helper class driven + // through the normal `__next__` dispatch. + if instance_method(v, "__getitem__").is_some() { + if let Some(it) = self.make_seq_iterator(v, globals)? { + return Ok(it); } - let list = Object::new_list(out); - return self.make_iter(&list, globals); } // A subclass of a built-in container (`class C(list)`, // `class C(dict)`, …) that doesn't override `__iter__` @@ -5823,12 +6410,61 @@ impl Interpreter { Err(type_error("'type' object is not iterable")) } _ => { + // A PEP 585 generic alias (`tuple[int]`, `list[str]`, …) is + // iterable: CPython's `ga_iternext` yields `typing.Unpack[self]` + // exactly once and then stops (the PEP 646 star-unpack form). + if is_generic_alias(v) { + return self.make_generic_alias_iter(v, globals); + } let it = v.make_iter()?; Ok(Object::Iter(Rc::new(RefCell::new(it)))) } } } + /// `iter()` of a PEP 585 generic alias. Mirrors CPython's + /// `ga_iternext`, which yields `typing.Unpack[self]` exactly once + /// before the iterator is exhausted. The single value is wrapped in a + /// tuple-style iterator so the exhausted `__reduce__` collapses to the + /// empty-tuple form `(iter, ((),))`, matching `tupleiterator`. + fn make_generic_alias_iter( + &mut self, + alias: &Object, + globals: &Rc>, + ) -> Result { + let starred = self.build_unpack_form(alias, globals)?; + let it = crate::object::PyIterator::Tuple { + items: Rc::from(vec![starred]), + index: 0, + }; + Ok(Object::Iter(Rc::new(RefCell::new(it)))) + } + + /// Build `typing.Unpack[alias]` — the PEP 646 star-unpack form a + /// generic alias yields when iterated. CPython lazily imports `typing` + /// at this point too; if `typing.Unpack` is somehow unavailable we + /// degrade to yielding the alias itself (the value is rarely inspected, + /// only the exhausted-iterator `__reduce__` is observable in practice). + fn build_unpack_form( + &mut self, + alias: &Object, + globals: &Rc>, + ) -> Result { + if let Ok(Object::Module(m)) = self.do_import("typing", &Object::None, 0, globals) { + let unpack = m + .dict + .borrow() + .get(&DictKey(Object::from_static("Unpack"))) + .cloned(); + if let Some(unpack) = unpack { + if let Some(method) = instance_method(&unpack, "__getitem__") { + return self.call(&method, std::slice::from_ref(alias), &[], globals); + } + } + } + Ok(alias.clone()) + } + /// Drive an awaitable into its underlying iterator (PEP 492 / /// RFC 0016). A coroutine is itself awaitable; an async generator /// is not (it must be consumed via `async for`). Any other object @@ -5929,6 +6565,15 @@ impl Interpreter { } Err(e) => Err(e), }, + // ``__await__`` returns the coroutine itself, so driving it via + // ``next()`` must advance it with ``send(None)`` like a generator. + Object::Coroutine(_) => match self.gen_method_send(iter, Object::None) { + Ok(v) => Ok(Some(v)), + Err(RuntimeError::PyException(exc)) if exc.type_name() == "StopIteration" => { + Ok(None) + } + Err(e) => Err(e), + }, Object::Instance(_) => { if let Some(method) = instance_method(iter, "__next__") { match self.call(&method, &[], &[], globals) { @@ -6113,14 +6758,19 @@ impl Interpreter { Err(RuntimeError::PyException(inner_exc)) if inner_exc.type_name() == "StopIteration" => { - // Inner finished cleanly. Replace the iter on - // the stack with the StopIteration's value and - // advance past the SEND/YIELD/JUMP-BACK loop. + // Inner finished cleanly (the thrown exception was + // handled and the sub-iterator returned). The frame is + // parked at YIELD_VALUE with the sub-iterator on top — + // its yielded value was already popped. We resume into + // END_SEND (the SEND loop's jump target), which expects + // `[..., iter, value]` and pops the iterator. So leave + // the return value *above* the iterator rather than + // overwriting it, mirroring the normal + // SEND-sees-StopIteration path; overwriting it left only + // one operand and underflowed END_SEND (gh: coroutine + // close delegating through `yield from`/`await`). let ret_val = exception_value(&inner_exc.instance); - if !frame.stack.is_empty() { - let len = frame.stack.len(); - frame.stack[len - 1] = ret_val; - } + frame.stack.push(ret_val); advance_past_yield_from(&mut frame); return match self.run_until_yield_or_return(&mut frame, None) { Ok(FrameOutcome::Yielded(v)) => { @@ -7728,7 +8378,14 @@ impl Interpreter { *fr.trace.borrow_mut() = value; Ok(()) } - "f_trace_lines" | "f_trace_opcodes" => Ok(()), + "f_trace_lines" => { + fr.trace_lines.set(value.is_truthy()); + Ok(()) + } + "f_trace_opcodes" => { + fr.trace_opcodes.set(value.is_truthy()); + Ok(()) + } "f_lineno" => match value { Object::Int(i) if i >= 0 => { fr.override_lineno.set(Some(i as u32)); @@ -8803,6 +9460,19 @@ impl Interpreter { outer_globals, ); } + // `.__reduce__()` — reduce to + // `(iter, (remaining_items,))` so the iterator + // pickles/copies and replays its not-yet-yielded + // items, matching CPython's seqiterobject reduction. + ".iter_reduce" => { + return self.iter_reduce(&bm.receiver, outer_globals); + } + // `file.writelines(iterable)` — stream the iterable + // through the full iterator protocol so any iterable + // (generators, custom `__iter__`, views, …) works. + ".file_writelines" => { + return self.do_file_writelines(&bm.receiver, args, outer_globals); + } // Bound `x.__getattribute__(name)` resolving to // `object.__getattribute__` (i.e. no user override): // run the default lookup against the bound receiver. @@ -10397,6 +11067,7 @@ impl Interpreter { class_namespace: None, exc_handlers: Vec::new(), pc: 0, + py_frame: None, }; self.run_frame(&mut frame) } @@ -10642,6 +11313,100 @@ impl Interpreter { self.call(&helper, &[recv.clone(), Object::Int(proto)], &[], globals) } + /// `.__reduce__()` → `(iter, (remaining_items,))`. + /// Mirrors CPython's reduction of built-in iterators: the + /// not-yet-yielded items are snapshotted into a list, and re-applying + /// the canonical `iter` builtin to that list reproduces an iterator + /// that replays exactly those items. (Dict/bytes/range iterators thus + /// unpickle as a list-iterator, which `test_iter` explicitly allows.) + /// The `iter` builtin is fetched from the live `__builtins__` so it + /// pickles by name and round-trips through any unpickler. + fn iter_reduce( + &mut self, + recv: &Object, + _globals: &Rc>, + ) -> Result { + // A reverse-iterator reduces through `reversed` with a forward list; + // everything else reduces through `iter` with a native-typed + // remaining container. + let it = match recv { + Object::Iter(it) => it, + _ => return Err(type_error("__reduce__() requires an iterator")), + }; + // Sniff the variant with a borrow that is released immediately — + // the builtin lookup below may run arbitrary `__eq__` that re-enters + // and mutates (e.g. exhausts) this very iterator. + let is_reversed = matches!(&*it.borrow(), crate::object::PyIterator::Reversed { .. }); + let builtin_name = if is_reversed { "reversed" } else { "iter" }; + // CPython fetches the builtin via `_PyEval_GetBuiltin`, which reads the + // live `builtins` *module* dict — a namespace user code can mutate. + // Resolve there first (not the VM's private fast-path map) so a + // hash-colliding custom key runs its Python `__eq__`, both returning + // the current binding and letting side effects fire before we read the + // iterator's remaining state (issue gh-101765). No iterator borrow is + // held across this lookup. + let key = DictKey(Object::from_static(builtin_name)); + let module_dict = match self.cache.get("builtins") { + Some(Object::Module(m)) => Some(m.dict.clone()), + _ => None, + }; + let builtin = module_dict + .as_ref() + .and_then(|d| d.borrow().get(&key).cloned()) + .or_else(|| self.builtins.borrow().get(&key).cloned()) + .ok_or_else(|| runtime_error(format!("builtin '{builtin_name}' unavailable")))?; + // Snapshot remaining *after* the lookup, mirroring CPython reading + // `it_seq`/index post-`_PyEval_GetBuiltin`. + let remaining = { + let it = it.borrow(); + if is_reversed { + it.reversed_reduce_arg() + .unwrap_or_else(|| Object::new_list(Vec::new())) + } else { + it.reduce_remaining() + } + }; + let args_tuple = Object::new_tuple(vec![remaining]); + Ok(Object::new_tuple(vec![builtin, args_tuple])) + } + + /// `file.writelines(iterable)` — write each item, pulling them lazily + /// through the general iterator protocol (so generators and objects + /// with a custom `__iter__` work, not just native sequences). A + /// non-iterable argument raises TypeError, matching CPython. + fn do_file_writelines( + &mut self, + receiver: &Object, + args: &[Object], + globals: &Rc>, + ) -> Result { + let file = match receiver { + Object::File(f) => f.clone(), + _ => return Err(type_error("writelines() requires a file receiver")), + }; + let iterable = args + .first() + .ok_or_else(|| type_error("writelines() takes exactly one argument"))?; + let it = self.make_iter(iterable, globals)?; + while let Some(line) = self.iter_next(&it, globals)? { + match line { + Object::Str(s) => { + file.write_bytes(s.as_bytes())?; + } + Object::Bytes(b) => { + file.write_bytes(&b)?; + } + other => { + return Err(type_error(format!( + "writelines() argument must be a list of strings, not '{}'", + other.type_name_owned() + ))) + } + } + } + Ok(Object::None) + } + fn do_compile_call( &mut self, args: &[Object], diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 9b96fa3..28929e7 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -251,6 +251,14 @@ pub struct PyFrame { /// means "no line event has fired on this frame yet" — the /// next `step` will fire one. pub last_line: Cell>, + /// Mirrors CPython's `frame.f_trace_lines`. When `true` (the + /// default) the dispatcher fires `'line'` events; debuggers set it + /// `false` to suppress them. + pub trace_lines: Cell, + /// Mirrors CPython's `frame.f_trace_opcodes`. When `true` the + /// dispatcher fires an `'opcode'` event before every instruction + /// (used by `bdb`/`pdb` instruction stepping). Defaults to `false`. + pub trace_opcodes: Cell, } impl fmt::Debug for PyFrame { @@ -1114,6 +1122,15 @@ pub enum PyIterator { inner: Rc>, count: i64, }, + /// `reversed(seq)` — yields `items[index]`, `items[index-1]`, … down + /// to `items[0]`. `items` is held in *forward* order (matching + /// CPython's `list_reverseiterator`, whose `__reduce__` is + /// `(reversed, (forward_seq,), index)`); `index` counts down and the + /// backing vector is detached on exhaustion. + Reversed { + items: Rc>>, + index: i64, + }, } impl PyIterator { @@ -1121,9 +1138,21 @@ impl PyIterator { pub fn next_value(&mut self) -> Option { match self { PyIterator::List { items, index } => { - let v = items.borrow().get(*index).cloned()?; - *index += 1; - Some(v) + let next = items.borrow().get(*index).cloned(); + match next { + Some(v) => { + *index += 1; + Some(v) + } + None => { + // Exhausted. Detach from the backing list so a + // later `append`/`extend` can't resurrect the + // iterator — CPython clears `it_seq` on the first + // StopIteration and the iterator stays empty. + *items = Rc::new(RefCell::new(Vec::new())); + None + } + } } PyIterator::Tuple { items, index } => { let v = items.get(*index).cloned()?; @@ -1177,6 +1206,25 @@ impl PyIterator { *count += 1; Some(Object::new_tuple(vec![Object::Int(i), v])) } + PyIterator::Reversed { items, index } => { + if *index < 0 { + *items = Rc::new(RefCell::new(Vec::new())); + return None; + } + let v = items.borrow().get(*index as usize).cloned(); + match v { + Some(val) => { + *index -= 1; + Some(val) + } + None => { + // Index out of range (list shrank): exhaust + detach. + *items = Rc::new(RefCell::new(Vec::new())); + *index = -1; + None + } + } + } } } @@ -1194,6 +1242,7 @@ impl PyIterator { PyIterator::DictKeys { keys, index } => Some(keys.len().saturating_sub(*index)), PyIterator::Bytes { data, index } => Some(data.len().saturating_sub(*index)), PyIterator::Enumerate { inner, .. } => inner.borrow().remaining(), + PyIterator::Reversed { index, .. } => Some((*index + 1).max(0) as usize), PyIterator::Range { current, stop, @@ -1211,6 +1260,120 @@ impl PyIterator { } } } + + /// Snapshot the items the iterator would still yield, *without* + /// consuming it. Backs the built-in iterator's `__reduce__` + /// (pickling): CPython reduces e.g. a list-iterator to + /// `(iter, (remaining_list,))`, so a freshly-unpickled iterator + /// replays exactly the not-yet-seen elements. A shared + /// (`Enumerate`) inner is read through its `RefCell` borrow, never + /// advanced. + pub fn remaining_items(&self) -> Vec { + match self { + PyIterator::List { items, index } => { + items.borrow().get(*index..).map(<[_]>::to_vec).unwrap_or_default() + } + PyIterator::Tuple { items, index } => { + items.get(*index..).map(<[_]>::to_vec).unwrap_or_default() + } + PyIterator::Str { s, index } => { + let start = (*index).min(s.len()); + s[start..] + .chars() + .map(|c| Object::Str(Rc::from(c.to_string().as_str()))) + .collect() + } + PyIterator::DictKeys { keys, index } => keys + .get(*index..) + .map(|rest| rest.iter().map(|k| k.0.clone()).collect()) + .unwrap_or_default(), + PyIterator::Bytes { data, index } => data + .get(*index..) + .map(|rest| rest.iter().map(|b| Object::Int(i64::from(*b))).collect()) + .unwrap_or_default(), + PyIterator::Range { + current, + stop, + step, + } => { + let mut out = Vec::new(); + let (mut c, st, sp) = (*current, *stop, *step); + if sp > 0 { + while c < st { + out.push(Object::Int(c)); + c += sp; + } + } else if sp < 0 { + while c > st { + out.push(Object::Int(c)); + c += sp; + } + } + out + } + PyIterator::Enumerate { inner, count } => { + let rest = inner.borrow().remaining_items(); + let mut out = Vec::with_capacity(rest.len()); + let mut i = *count; + for v in rest { + out.push(Object::new_tuple(vec![Object::Int(i), v])); + i += 1; + } + out + } + PyIterator::Reversed { items, index } => { + // Not-yet-yielded values, in yield order: items[index]..items[0]. + let items = items.borrow(); + let mut out = Vec::new(); + let mut i = *index; + while i >= 0 { + if let Some(v) = items.get(i as usize) { + out.push(v.clone()); + } + i -= 1; + } + out + } + } + } + + /// The forward slice a `reversed`-iterator reduces with: re-applying + /// `reversed` to it reproduces the not-yet-yielded values in order. + /// Empty when exhausted, giving CPython's `(reversed, ([],))`. + pub fn reversed_reduce_arg(&self) -> Option { + match self { + PyIterator::Reversed { items, index } => { + let items = items.borrow(); + let end = ((*index).max(-1) + 1) as usize; + let slice = items.get(..end.min(items.len())).unwrap_or(&[]); + Some(Object::new_list(slice.to_vec())) + } + _ => None, + } + } + + /// The remaining items packaged in the *native container type* + /// CPython uses for that iterator's `__reduce__` argument, so the + /// reduction tuple compares equal to CPython's: a string-iterator + /// reduces with a `str`, a tuple-iterator with a `tuple`, a + /// list-iterator with a `list`. (Bytes and the generic seqiter use a + /// `tuple`, so an exhausted one reduces to `(iter, ((),))`.) + /// Re-applying `iter` to this value replays exactly the not-yet-seen + /// elements. + pub fn reduce_remaining(&self) -> Object { + match self { + PyIterator::Tuple { .. } | PyIterator::Bytes { .. } => { + Object::new_tuple(self.remaining_items()) + } + PyIterator::Str { s, index } => { + let start = (*index).min(s.len()); + Object::from_str(&s[start..]) + } + // list / dict / range / enumerate reduce through a plain list + // (dict iterators explicitly unpickle as list iterators). + _ => Object::new_list(self.remaining_items()), + } + } } // ---------- behavior ---------- @@ -1458,7 +1621,14 @@ impl Object { (Object::BoundMethod(a), Object::BoundMethod(b)) => { a.function.eq_value(&b.function) && a.receiver.is_same(&b.receiver) } - _ => false, + // CPython's default `tp_richcompare` (no user `__eq__`) falls + // back to *identity*: `x == x` is True and `x == y` is False + // for distinct objects. This covers reference types without + // value semantics — frames, generators, tracebacks, cells, + // code objects, … — where `bdb`/`pdb` rely on `frame == + // self.returnframe`. Returning a flat `false` here would make + // even `frame == frame` False. + _ => self.is_same(other), } } diff --git a/crates/weavepy-vm/src/stdlib/marshal_mod.rs b/crates/weavepy-vm/src/stdlib/marshal_mod.rs index b88f426..fdde30f 100644 --- a/crates/weavepy-vm/src/stdlib/marshal_mod.rs +++ b/crates/weavepy-vm/src/stdlib/marshal_mod.rs @@ -657,6 +657,9 @@ impl<'a> MarshalReader<'a> { cellvars: decoded.cellvars, exception_table: decoded.exception_table, linetable: decoded.linetable, + // Marshal doesn't round-trip PEP-657 columns yet; co_positions() + // on an unmarshalled code object reports lines only. + coltable: Vec::new(), arg_count, posonly_count, kwonly_count, diff --git a/crates/weavepy-vm/src/stdlib/mod.rs b/crates/weavepy-vm/src/stdlib/mod.rs index d9deb85..d7cdbd6 100644 --- a/crates/weavepy-vm/src/stdlib/mod.rs +++ b/crates/weavepy-vm/src/stdlib/mod.rs @@ -171,6 +171,15 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/builtins.py"), is_package: false, }, + // Internal: `_SeqIter`, the lazy legacy-`__getitem__` iterator + // `iter(obj)` returns when *obj* has no `__iter__` (CPython's + // built-in `iterator`/seqiterobject). Kept out of `builtins` to + // avoid leaking a name into every module's global namespace. + FrozenSource { + name: "_seqtools", + source: include_str!("python/_seqtools.py"), + is_package: false, + }, // `collections` is a package so `collections.abc` resolves; the // verbatim CPython `_collections_abc` carries the ABC definitions // and `collections.abc` re-exports them (RFC 0037 WS8). diff --git a/crates/weavepy-vm/src/stdlib/python/_seqtools.py b/crates/weavepy-vm/src/stdlib/python/_seqtools.py new file mode 100644 index 0000000..addf716 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_seqtools.py @@ -0,0 +1,129 @@ +"""Internal helpers for the legacy sequence-iteration protocol. + +`_SeqIter` is WeavePy's equivalent of CPython's built-in ``iterator`` +type (``seqiterobject``): the object ``iter(obj)`` returns when *obj* +defines ``__getitem__`` but not ``__iter__``. It drives the sequence +**lazily** — calling ``obj[0]``, ``obj[1]``, … on demand and stopping at +the first ``IndexError`` — so an unbounded sequence (``__getitem__`` that +never raises) iterates forever instead of hanging at construction, and +side effects happen one element at a time exactly as CPython does. + +It also implements the pickling protocol (``__reduce__`` / +``__setstate__``, with the CPython negative-index clamp) and +``__length_hint__`` so the standard library's iterator tests behave. +""" + + +def _builtin_iter(): + """Fetch ``iter`` from the live ``builtins`` *module* namespace. + + Mirrors CPython's ``_PyEval_GetBuiltin(&_Py_ID(iter))``, which the C + seq/callable iterator ``__reduce__`` calls *before* reading the + iterator's state. Going through the module dict (rather than the + bare ``iter`` global) means a user who has shadowed ``builtins.iter`` + with a hash-colliding custom key sees that key's ``__eq__`` fire here + — the exact side-effect ordering test_iter's gh-101765 reproducer + depends on. Falls back to the plain global if anything goes wrong. + """ + try: + import builtins + return builtins.__dict__["iter"] + except (KeyError, ImportError): + return iter + + +class _SeqIter: + __slots__ = ("_seq", "_index") + + def __init__(self, seq): + self._seq = seq + self._index = 0 + + def __iter__(self): + return self + + def __next__(self): + seq = self._seq + if seq is None: + raise StopIteration + try: + item = seq[self._index] + except (IndexError, StopIteration): + # Exhausted: drop the sequence reference so a resurrected + # iterator stays exhausted (matches CPython's seqiterobject, + # which clears it_seq on both IndexError and StopIteration). + self._seq = None + raise StopIteration + self._index += 1 + return item + + def __length_hint__(self): + seq = self._seq + if seq is None: + return 0 + try: + length = len(seq) + except TypeError: + return 0 + hint = length - self._index + return hint if hint > 0 else 0 + + def __reduce__(self): + # Resolve `iter` first: the lookup can run user code that exhausts + # us (gh-101765), so read `self._seq` only afterwards. + _iter = _builtin_iter() + if self._seq is None: + # Exhausted iterator pickles as an empty one. + return (_iter, ((),)) + return (_iter, (self._seq,), self._index) + + def __setstate__(self, state): + # CPython clamps a negative resume index to 0. + if state < 0: + state = 0 + self._index = state + + +class _CallableIter: + """WeavePy's equivalent of CPython's ``callable_iterator`` + (``calliterobject``): the object ``iter(callable, sentinel)`` returns. + + Each ``__next__`` calls *callable* with no arguments and yields the + result, stopping (``StopIteration``) once a result compares equal to + *sentinel*. Driving it **lazily** — one call per ``next()`` — means an + exception raised inside *callable* propagates at the right moment (so + ``for x in iter(spam, s)`` sees it mid-stream) and an unbounded source + never hangs at construction, matching CPython exactly. + """ + + __slots__ = ("_callable", "_sentinel") + + def __init__(self, callable, sentinel): + self._callable = callable + self._sentinel = sentinel + + def __iter__(self): + return self + + def __next__(self): + if self._callable is None: + raise StopIteration + result = self._callable() + # gh-101892: the call may have re-entered and exhausted us; if so, + # report exhaustion rather than yielding a post-sentinel value. + if self._callable is None: + raise StopIteration + # CPython compares ``result == sentinel`` (result's __eq__ first). + if result == self._sentinel: + self._callable = None + raise StopIteration + return result + + def __reduce__(self): + # Resolve `iter` first (it can run user code that exhausts us, + # gh-101765); an exhausted callable-iterator has dropped its + # callable and reduces to an empty `iter(())`. + _iter = _builtin_iter() + if self._callable is None: + return (_iter, ((),)) + return (_iter, (self._callable, self._sentinel)) diff --git a/crates/weavepy-vm/src/stdlib/python/functools.py b/crates/weavepy-vm/src/stdlib/python/functools.py index 0f39a49..0dee17e 100644 --- a/crates/weavepy-vm/src/stdlib/python/functools.py +++ b/crates/weavepy-vm/src/stdlib/python/functools.py @@ -1,405 +1,170 @@ -"""WeavePy's pure-Python ``functools`` module. - -Covers the high-traffic surface area: ``reduce``, ``partial``, -``lru_cache``, ``cache``, ``wraps``, ``cmp_to_key``, and friends. +"""functools.py - Tools for working with functions and callable objects """ - -__all__ = [ - "reduce", - "partial", - "partialmethod", - "lru_cache", - "cache", - "wraps", - "update_wrapper", - "cmp_to_key", - "total_ordering", - "singledispatch", - "cached_property", -] - - -WRAPPER_ASSIGNMENTS = ( - "__module__", - "__name__", - "__qualname__", - "__doc__", - "__dict__", -) -WRAPPER_UPDATES = ("__dict__",) - - -def reduce(function, iterable, *initial): - it = iter(iterable) - if initial: - value = initial[0] - else: - try: - value = next(it) - except StopIteration: - raise TypeError( - "reduce() of empty iterable with no initial value" - ) - for item in it: - value = function(value, item) - return value - - -class partial: - """Callable that pre-applies positional and keyword arguments. - - `func` is positional-only (CPython's `partial.__new__(cls, func, /, - *args, **keywords)`): without that, a keyword named `func`/`self` - passed through to the wrapped callable — e.g. `operator.methodcaller`'s - pickle reduce builds `partial(methodcaller, name, self=..., name=...)` - — would collide with the constructor's own parameter and raise - "got multiple values for argument 'self'". +# Python module wrapper for _functools C module +# to allow utilities written in Python to be added +# to the functools module. +# Written by Nick Coghlan , +# Raymond Hettinger , +# and Łukasz Langa . +# Copyright (C) 2006-2013 Python Software Foundation. +# See C source code for _functools credits/copyright + +__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', + 'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce', + 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod', + 'cached_property'] + +from abc import get_cache_token +from collections import namedtuple +# import types, weakref # Deferred to single_dispatch() +from reprlib import recursive_repr +from _thread import RLock + +# Avoid importing types, so we can speedup import time +GenericAlias = type(list[int]) + +################################################################################ +### update_wrapper() and wraps() decorator +################################################################################ + +# update_wrapper() and wraps() are tools to help write +# wrapper functions that can handle naive introspection + +WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__', + '__annotations__', '__type_params__') +WRAPPER_UPDATES = ('__dict__',) +def update_wrapper(wrapper, + wrapped, + assigned = WRAPPER_ASSIGNMENTS, + updated = WRAPPER_UPDATES): + """Update a wrapper function to look like the wrapped function + + wrapper is the function to be updated + wrapped is the original function + assigned is a tuple naming the attributes assigned directly + from the wrapped function to the wrapper function (defaults to + functools.WRAPPER_ASSIGNMENTS) + updated is a tuple naming the attributes of the wrapper that + are updated with the corresponding attribute from the wrapped + function (defaults to functools.WRAPPER_UPDATES) """ - - def __new__(cls, func, /, *args, **keywords): - if not callable(func): - raise TypeError("the first argument must be callable") - if isinstance(func, partial): - args = func.args + args - keywords = {**func.keywords, **keywords} - func = func.func - self = super(partial, cls).__new__(cls) - self.func = func - self.args = args - self.keywords = keywords - return self - - def __call__(self, /, *args, **keywords): - keywords = {**self.keywords, **keywords} - return self.func(*self.args, *args, **keywords) - - def __repr__(self): - qualname = type(self).__qualname__ - module = type(self).__module__ or "functools" - parts = [repr(self.func)] - for a in self.args: - parts.append(repr(a)) - for k, v in self.keywords.items(): - parts.append(k + "=" + repr(v)) - return module + "." + qualname + "(" + ", ".join(parts) + ")" - - def __reduce__(self): - return ( - type(self), - (self.func,), - (self.func, self.args, self.keywords or None, self.__dict__ or None), - ) - - def __setstate__(self, state): - if not isinstance(state, tuple): - raise TypeError("argument to __setstate__ must be a tuple") - if len(state) != 4: - raise TypeError("expected 4 items in state, got %d" % len(state)) - func, args, kwds, namespace = state - if (not callable(func) or not isinstance(args, tuple) or - (kwds is not None and not isinstance(kwds, dict)) or - (namespace is not None and not isinstance(namespace, dict))): - raise TypeError("invalid partial state") - args = tuple(args) - if kwds is None: - kwds = {} - elif type(kwds) is not dict: - kwds = dict(kwds) - if namespace is None: - namespace = {} - self.__dict__ = namespace - self.func = func - self.args = args - self.keywords = kwds - - -class partialmethod: - """Descriptor form of :class:`partial` for methods.""" - - def __init__(self, func, /, *args, **keywords): - # `func` is positional-only (PEP 570) so a wrapped callable may itself - # take `self`/`func` keyword arguments without colliding — matches - # CPython's `partialmethod.__init__` signature. - if isinstance(func, partialmethod): - # Flatten nested partialmethods so cls/self stay ahead of all - # other arguments and only one underlying call happens. - self.func = func.func - self.args = func.args + args - self.keywords = {**func.keywords, **keywords} - else: - self.func = func - self.args = args - self.keywords = keywords - - def __get__(self, instance, owner=None): - if instance is None: - return self - def bound(*args, **kwargs): - merged = dict(self.keywords) - merged.update(kwargs) - return self.func(instance, *self.args, *args, **merged) - return bound - - -def update_wrapper( - wrapper, - wrapped, - assigned=WRAPPER_ASSIGNMENTS, - updated=WRAPPER_UPDATES, -): for attr in assigned: try: value = getattr(wrapped, attr) except AttributeError: pass else: - try: - setattr(wrapper, attr, value) - except (AttributeError, TypeError): - pass + setattr(wrapper, attr, value) for attr in updated: - try: - getattr(wrapper, attr).update(getattr(wrapped, attr, {})) - except (AttributeError, TypeError): - pass + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + # Issue #17482: set __wrapped__ last so we don't inadvertently copy it + # from the wrapped function when updating __dict__ wrapper.__wrapped__ = wrapped + # Return the wrapper so this can be used as a decorator via partial() return wrapper +def wraps(wrapped, + assigned = WRAPPER_ASSIGNMENTS, + updated = WRAPPER_UPDATES): + """Decorator factory to apply update_wrapper() to a wrapper function -def wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES): - def decorator(wrapper): - return update_wrapper(wrapper, wrapped, assigned, updated) - return decorator - - -def lru_cache(maxsize=128, typed=False): - """Least-recently-used caching decorator.""" - - if callable(maxsize): - # @lru_cache without parentheses - func = maxsize - return _make_lru(func, 128, False) - - def decorator(func): - return _make_lru(func, maxsize, typed) - - return decorator - - -def cache(func): - """Unbounded cache decorator (alias for ``lru_cache(maxsize=None)``).""" - return _make_lru(func, None, False) - - -class _LruCacheWrapper: - """Class-based wrapper so we can hang `cache_clear` / `cache_info` - off the cached callable. WeavePy's Python functions don't yet - accept arbitrary attribute assignment, so a class is the cleanest - workaround that still keeps the lookup cost minimal.""" - - def __init__(self, func, maxsize, typed): - self.__wrapped__ = func - self._maxsize = maxsize - self._typed = typed - self._storage = {} - self._order = [] - self._hits = 0 - self._misses = 0 - - def _make_key(self, args, kwargs): - key = args - if kwargs: - key = key + ("__kw__",) + tuple(sorted(kwargs.items())) - if self._typed: - key = key + tuple(type(a) for a in args) - return key - - def __call__(self, *args, **kwargs): - key = self._make_key(args, kwargs) - if key in self._storage: - self._hits += 1 - self._order.remove(key) - self._order.append(key) - return self._storage[key] - self._misses += 1 - value = self.__wrapped__(*args, **kwargs) - self._storage[key] = value - self._order.append(key) - if self._maxsize is not None and len(self._order) > self._maxsize: - old = self._order.pop(0) - del self._storage[old] - return value - - def cache_clear(self): - self._storage.clear() - self._order.clear() - self._hits = 0 - self._misses = 0 - - def cache_info(self): - # CPython exposes ``cache_info`` as a named tuple with the - # ``hits``, ``misses``, ``maxsize``, ``currsize`` fields so - # callers can use attribute access (``info.hits``). - return _CacheInfo( - hits=self._hits, - misses=self._misses, - maxsize=self._maxsize, - currsize=len(self._storage), - ) - - -class _CacheInfo: - """Lightweight stand-in for ``collections.namedtuple`` that gives - ``functools.lru_cache.cache_info`` its CPython-compatible - attribute access plus tuple-style iteration. Kept local so the - real ``collections.namedtuple`` import isn't required.""" - - __slots__ = ("hits", "misses", "maxsize", "currsize") - - def __init__(self, hits=0, misses=0, maxsize=None, currsize=0): - self.hits = hits - self.misses = misses - self.maxsize = maxsize - self.currsize = currsize - - def __iter__(self): - return iter((self.hits, self.misses, self.maxsize, self.currsize)) - - def __eq__(self, other): - if isinstance(other, _CacheInfo): - return ( - self.hits == other.hits - and self.misses == other.misses - and self.maxsize == other.maxsize - and self.currsize == other.currsize - ) - if isinstance(other, tuple): - return (self.hits, self.misses, self.maxsize, self.currsize) == other - return NotImplemented - - def __repr__(self): - return ( - f"CacheInfo(hits={self.hits}, misses={self.misses}, " - f"maxsize={self.maxsize}, currsize={self.currsize})" - ) - - -def _make_lru(func, maxsize, typed): - return _LruCacheWrapper(func, maxsize, typed) - - -def cmp_to_key(cmp): - """Convert an old-style comparison function into a key function.""" - - class K: - __slots__ = ("obj",) - - def __init__(self, obj): - self.obj = obj - - def __lt__(self, other): - return cmp(self.obj, other.obj) < 0 - - def __le__(self, other): - return cmp(self.obj, other.obj) <= 0 - - def __gt__(self, other): - return cmp(self.obj, other.obj) > 0 - - def __ge__(self, other): - return cmp(self.obj, other.obj) >= 0 - - def __eq__(self, other): - return cmp(self.obj, other.obj) == 0 - - def __ne__(self, other): - return cmp(self.obj, other.obj) != 0 + Returns a decorator that invokes update_wrapper() with the decorated + function as the wrapper argument and the arguments to wraps() as the + remaining arguments. Default arguments are as for update_wrapper(). + This is a convenience function to simplify applying partial() to + update_wrapper(). + """ + return partial(update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) - return K +################################################################################ +### total_ordering class decorator +################################################################################ -# ---- total_ordering ---------------------------------------------------------- -# Verbatim CPython 3.13: fills in the missing rich-comparison methods from a -# single defined one (RFC 0037 WS8 functools edges). +# The total ordering functions all invoke the root magic method directly +# rather than using the corresponding operator. This avoids possible +# infinite recursion that could occur when the operator dispatch logic +# detects a NotImplemented result and then calls a reflected method. -def _gt_from_lt(self, other, NotImplemented=NotImplemented): +def _gt_from_lt(self, other): 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).' op_result = type(self).__lt__(self, other) if op_result is NotImplemented: return op_result return not op_result and self != other -def _le_from_lt(self, other, NotImplemented=NotImplemented): +def _le_from_lt(self, other): 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).' op_result = type(self).__lt__(self, other) if op_result is NotImplemented: return op_result return op_result or self == other -def _ge_from_lt(self, other, NotImplemented=NotImplemented): +def _ge_from_lt(self, other): 'Return a >= b. Computed by @total_ordering from (not a < b).' op_result = type(self).__lt__(self, other) if op_result is NotImplemented: return op_result return not op_result -def _ge_from_le(self, other, NotImplemented=NotImplemented): +def _ge_from_le(self, other): 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).' op_result = type(self).__le__(self, other) if op_result is NotImplemented: return op_result return not op_result or self == other -def _lt_from_le(self, other, NotImplemented=NotImplemented): +def _lt_from_le(self, other): 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).' op_result = type(self).__le__(self, other) if op_result is NotImplemented: return op_result return op_result and self != other -def _gt_from_le(self, other, NotImplemented=NotImplemented): +def _gt_from_le(self, other): 'Return a > b. Computed by @total_ordering from (not a <= b).' op_result = type(self).__le__(self, other) if op_result is NotImplemented: return op_result return not op_result -def _lt_from_gt(self, other, NotImplemented=NotImplemented): +def _lt_from_gt(self, other): 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).' op_result = type(self).__gt__(self, other) if op_result is NotImplemented: return op_result return not op_result and self != other -def _ge_from_gt(self, other, NotImplemented=NotImplemented): +def _ge_from_gt(self, other): 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).' op_result = type(self).__gt__(self, other) if op_result is NotImplemented: return op_result return op_result or self == other -def _le_from_gt(self, other, NotImplemented=NotImplemented): +def _le_from_gt(self, other): 'Return a <= b. Computed by @total_ordering from (not a > b).' op_result = type(self).__gt__(self, other) if op_result is NotImplemented: return op_result return not op_result -def _le_from_ge(self, other, NotImplemented=NotImplemented): +def _le_from_ge(self, other): 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).' op_result = type(self).__ge__(self, other) if op_result is NotImplemented: return op_result return not op_result or self == other -def _gt_from_ge(self, other, NotImplemented=NotImplemented): +def _gt_from_ge(self, other): 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).' op_result = type(self).__ge__(self, other) if op_result is NotImplemented: return op_result return op_result and self != other -def _lt_from_ge(self, other, NotImplemented=NotImplemented): +def _lt_from_ge(self, other): 'Return a < b. Computed by @total_ordering from (not a >= b).' op_result = type(self).__ge__(self, other) if op_result is NotImplemented: @@ -418,7 +183,7 @@ def _lt_from_ge(self, other, NotImplemented=NotImplemented): ('__le__', _le_from_gt)], '__ge__': [('__le__', _le_from_ge), ('__gt__', _gt_from_ge), - ('__lt__', _lt_from_ge)], + ('__lt__', _lt_from_ge)] } def total_ordering(cls): @@ -435,95 +200,810 @@ def total_ordering(cls): return cls -# ---- single-dispatch generic functions -------------------------------------- +################################################################################ +### cmp_to_key() function converter +################################################################################ + +def cmp_to_key(mycmp): + """Convert a cmp= function into a key= function""" + class K(object): + __slots__ = ['obj'] + def __init__(self, obj): + self.obj = obj + def __lt__(self, other): + return mycmp(self.obj, other.obj) < 0 + def __gt__(self, other): + return mycmp(self.obj, other.obj) > 0 + def __eq__(self, other): + return mycmp(self.obj, other.obj) == 0 + def __le__(self, other): + return mycmp(self.obj, other.obj) <= 0 + def __ge__(self, other): + return mycmp(self.obj, other.obj) >= 0 + __hash__ = None + return K + +try: + from _functools import cmp_to_key +except ImportError: + pass -class _SingleDispatchCallable: - """Backing object for :func:`singledispatch`. +################################################################################ +### reduce() sequence to a single item +################################################################################ - Implementing this as a class (instead of nested closures) keeps - the registry visible to ``register``'s inner decorator without - relying on three-level freevar passthrough. +_initial_missing = object() + +def reduce(function, sequence, initial=_initial_missing): """ + reduce(function, iterable[, initial], /) -> value - def __init__(self, func): - self._default = func - self.registry = {object: func} - self.__wrapped__ = func - - def dispatch(self, cls): - for base in cls.__mro__: - if base in self.registry: - return self.registry[base] - return self._default - - def register(self, cls, impl=None): - if impl is None: - outer_self = self - outer_cls = cls - - def decorator(real_impl): - outer_self.registry[outer_cls] = real_impl - return real_impl - - return decorator - self.registry[cls] = impl - return impl + Apply a function of two arguments cumulatively to the items of an iterable, from left to right. - def __call__(self, *args, **kwargs): - if not args: + This effectively reduces the iterable to a single value. If initial is present, + it is placed before the items of the iterable in the calculation, and serves as + a default when the iterable is empty. + + For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) + calculates ((((1 + 2) + 3) + 4) + 5). + """ + + it = iter(sequence) + + if initial is _initial_missing: + try: + value = next(it) + except StopIteration: raise TypeError( - "singledispatch function requires at least one positional argument" - ) - impl = self.dispatch(type(args[0])) - return impl(*args, **kwargs) + "reduce() of empty iterable with no initial value") from None + else: + value = initial + + for element in it: + value = function(value, element) + + return value + +try: + from _functools import reduce +except ImportError: + pass + + +################################################################################ +### partial() argument application +################################################################################ + +# Purely functional, no descriptor behaviour +class partial: + """New function with partial application of the given arguments + and keywords. + """ + + __slots__ = "func", "args", "keywords", "__dict__", "__weakref__" + + def __new__(cls, func, /, *args, **keywords): + if not callable(func): + raise TypeError("the first argument must be callable") + + if isinstance(func, partial): + args = func.args + args + keywords = {**func.keywords, **keywords} + func = func.func + + self = super(partial, cls).__new__(cls) + + self.func = func + self.args = args + self.keywords = keywords + return self + + def __call__(self, /, *args, **keywords): + keywords = {**self.keywords, **keywords} + return self.func(*self.args, *args, **keywords) + + @recursive_repr() + def __repr__(self): + cls = type(self) + qualname = cls.__qualname__ + module = cls.__module__ + args = [repr(self.func)] + args.extend(repr(x) for x in self.args) + args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" + + def __get__(self, obj, objtype=None): + if obj is None: + return self + import warnings + warnings.warn('functools.partial will be a method descriptor in ' + 'future Python versions; wrap it in staticmethod() ' + 'if you want to preserve the old behavior', + FutureWarning, 2) + return self + + def __reduce__(self): + return type(self), (self.func,), (self.func, self.args, + self.keywords or None, self.__dict__ or None) + + def __setstate__(self, state): + if not isinstance(state, tuple): + raise TypeError("argument to __setstate__ must be a tuple") + if len(state) != 4: + raise TypeError(f"expected 4 items in state, got {len(state)}") + func, args, kwds, namespace = state + if (not callable(func) or not isinstance(args, tuple) or + (kwds is not None and not isinstance(kwds, dict)) or + (namespace is not None and not isinstance(namespace, dict))): + raise TypeError("invalid partial state") + + args = tuple(args) # just in case it's a subclass + if kwds is None: + kwds = {} + elif type(kwds) is not dict: # XXX does it need to be *exactly* dict? + kwds = dict(kwds) + if namespace is None: + namespace = {} + + self.__dict__ = namespace + self.func = func + self.args = args + self.keywords = kwds + + __class_getitem__ = classmethod(GenericAlias) + + +try: + from _functools import partial +except ImportError: + pass + +# Descriptor version +class partialmethod(object): + """Method descriptor with partial application of the given arguments + and keywords. + + Supports wrapping existing descriptors and handles non-descriptor + callables as instance methods. + """ + + def __init__(self, func, /, *args, **keywords): + if not callable(func) and not hasattr(func, "__get__"): + raise TypeError("{!r} is not callable or a descriptor" + .format(func)) + + # func could be a descriptor like classmethod which isn't callable, + # so we can't inherit from partial (it verifies func is callable) + if isinstance(func, partialmethod): + # flattening is mandatory in order to place cls/self before all + # other arguments + # it's also more efficient since only one function will be called + self.func = func.func + self.args = func.args + args + self.keywords = {**func.keywords, **keywords} + else: + self.func = func + self.args = args + self.keywords = keywords + + def __repr__(self): + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" + + def _make_unbound_method(self): + def _method(cls_or_self, /, *args, **keywords): + keywords = {**self.keywords, **keywords} + return self.func(cls_or_self, *self.args, *args, **keywords) + _method.__isabstractmethod__ = self.__isabstractmethod__ + _method.__partialmethod__ = self + return _method + + def __get__(self, obj, cls=None): + get = getattr(self.func, "__get__", None) + result = None + if get is not None and not isinstance(self.func, partial): + new_func = get(obj, cls) + if new_func is not self.func: + # Assume __get__ returning something new indicates the + # creation of an appropriate callable + result = partial(new_func, *self.args, **self.keywords) + try: + result.__self__ = new_func.__self__ + except AttributeError: + pass + if result is None: + # If the underlying descriptor didn't do anything, treat this + # like an instance method + result = self._make_unbound_method().__get__(obj, cls) + return result + + @property + def __isabstractmethod__(self): + return getattr(self.func, "__isabstractmethod__", False) + + __class_getitem__ = classmethod(GenericAlias) + + +# Helper functions + +def _unwrap_partial(func): + while isinstance(func, partial): + func = func.func + return func + +def _unwrap_partialmethod(func): + prev = None + while func is not prev: + prev = func + while isinstance(getattr(func, "__partialmethod__", None), partialmethod): + func = func.__partialmethod__ + while isinstance(func, partialmethod): + func = getattr(func, 'func') + func = _unwrap_partial(func) + return func + +################################################################################ +### LRU Cache function decorator +################################################################################ + +_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) + +class _HashedSeq(list): + """ This class guarantees that hash() will be called no more than once + per element. This is important because the lru_cache() will hash + the key multiple times on a cache miss. + + """ + __slots__ = 'hashvalue' + + def __init__(self, tup, hash=hash): + self[:] = tup + self.hashvalue = hash(tup) + + def __hash__(self): + return self.hashvalue + +def _make_key(args, kwds, typed, + kwd_mark = (object(),), + fasttypes = {int, str}, + tuple=tuple, type=type, len=len): + """Make a cache key from optionally typed positional and keyword arguments + + The key is constructed in a way that is flat as possible rather than + as a nested structure that would take more memory. + + If there is only a single argument and its data type is known to cache + its hash value, then that argument is returned without a wrapper. This + saves space and improves lookup speed. + + """ + # All of code below relies on kwds preserving the order input by the user. + # Formerly, we sorted() the kwds before looping. The new way is *much* + # faster; however, it means that f(x=1, y=2) will now be treated as a + # distinct call from f(y=2, x=1) which will be cached separately. + key = args + if kwds: + key += kwd_mark + for item in kwds.items(): + key += item + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for v in kwds.values()) + elif len(key) == 1 and type(key[0]) in fasttypes: + return key[0] + return _HashedSeq(key) + +def lru_cache(maxsize=128, typed=False): + """Least-recently-used cache decorator. + + If *maxsize* is set to None, the LRU features are disabled and the cache + can grow without bound. + + If *typed* is True, arguments of different types will be cached separately. + For example, f(decimal.Decimal("3.0")) and f(3.0) will be treated as + distinct calls with distinct results. Some types such as str and int may + be cached separately even when typed is false. + + Arguments to the cached function must be hashable. + + View the cache statistics named tuple (hits, misses, maxsize, currsize) + with f.cache_info(). Clear the cache and statistics with f.cache_clear(). + Access the underlying function with f.__wrapped__. + + See: https://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU) + + """ + + # Users should only access the lru_cache through its public API: + # cache_info, cache_clear, and f.__wrapped__ + # The internals of the lru_cache are encapsulated for thread safety and + # to allow the implementation to change (including a possible C version). + + if isinstance(maxsize, int): + # Negative maxsize is treated as 0 + if maxsize < 0: + maxsize = 0 + elif callable(maxsize) and isinstance(typed, bool): + # The user_function was passed in directly via the maxsize argument + user_function, maxsize = maxsize, 128 + wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) + wrapper.cache_parameters = lambda : {'maxsize': maxsize, 'typed': typed} + return update_wrapper(wrapper, user_function) + elif maxsize is not None: + raise TypeError( + 'Expected first argument to be an integer, a callable, or None') + + def decorating_function(user_function): + wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) + wrapper.cache_parameters = lambda : {'maxsize': maxsize, 'typed': typed} + return update_wrapper(wrapper, user_function) + + return decorating_function + +def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo): + # Constants shared by all lru cache instances: + sentinel = object() # unique object used to signal cache misses + make_key = _make_key # build a key from the function arguments + PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + + cache = {} + hits = misses = 0 + full = False + cache_get = cache.get # bound method to lookup a key or return None + cache_len = cache.__len__ # get cache size without calling len() + lock = RLock() # because linkedlist updates aren't threadsafe + root = [] # root of the circular doubly linked list + root[:] = [root, root, None, None] # initialize by pointing to self + + if maxsize == 0: + + def wrapper(*args, **kwds): + # No caching -- just a statistics update + nonlocal misses + misses += 1 + result = user_function(*args, **kwds) + return result + + elif maxsize is None: + + def wrapper(*args, **kwds): + # Simple caching without ordering or size limit + nonlocal hits, misses + key = make_key(args, kwds, typed) + result = cache_get(key, sentinel) + if result is not sentinel: + hits += 1 + return result + misses += 1 + result = user_function(*args, **kwds) + cache[key] = result + return result + + else: + + def wrapper(*args, **kwds): + # Size limited caching that tracks accesses by recency + nonlocal root, hits, misses, full + key = make_key(args, kwds, typed) + with lock: + link = cache_get(key) + if link is not None: + # Move the link to the front of the circular queue + link_prev, link_next, _key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = root[PREV] + last[NEXT] = root[PREV] = link + link[PREV] = last + link[NEXT] = root + hits += 1 + return result + misses += 1 + result = user_function(*args, **kwds) + with lock: + if key in cache: + # Getting here means that this same key was added to the + # cache while the lock was released. Since the link + # update is already done, we need only return the + # computed result and update the count of misses. + pass + elif full: + # Use the old root to store the new key and result. + oldroot = root + oldroot[KEY] = key + oldroot[RESULT] = result + # Empty the oldest link and make it the new root. + # Keep a reference to the old key and old result to + # prevent their ref counts from going to zero during the + # update. That will prevent potentially arbitrary object + # clean-up code (i.e. __del__) from running while we're + # still adjusting the links. + root = oldroot[NEXT] + oldkey = root[KEY] + oldresult = root[RESULT] + root[KEY] = root[RESULT] = None + # Now update the cache dictionary. + del cache[oldkey] + # Save the potentially reentrant cache[key] assignment + # for last, after the root and links have been put in + # a consistent state. + cache[key] = oldroot + else: + # Put result in a new link at the front of the queue. + last = root[PREV] + link = [last, root, key, result] + last[NEXT] = root[PREV] = cache[key] = link + # Use the cache_len bound method instead of the len() function + # which could potentially be wrapped in an lru_cache itself. + full = (cache_len() >= maxsize) + return result + + def cache_info(): + """Report cache statistics""" + with lock: + return _CacheInfo(hits, misses, maxsize, cache_len()) + + def cache_clear(): + """Clear the cache and cache statistics""" + nonlocal hits, misses, full + with lock: + cache.clear() + root[:] = [root, root, None, None] + hits = misses = 0 + full = False + + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return wrapper + +try: + from _functools import _lru_cache_wrapper +except ImportError: + pass + + +################################################################################ +### cache -- simplified access to the infinity cache +################################################################################ + +def cache(user_function, /): + 'Simple lightweight unbounded cache. Sometimes called "memoize".' + return lru_cache(maxsize=None)(user_function) + + +################################################################################ +### singledispatch() - single-dispatch generic function decorator +################################################################################ + +def _c3_merge(sequences): + """Merges MROs in *sequences* to a single MRO using the C3 algorithm. + + Adapted from https://docs.python.org/3/howto/mro.html. + + """ + result = [] + while True: + sequences = [s for s in sequences if s] # purge empty sequences + if not sequences: + return result + for s1 in sequences: # find merge candidates among seq heads + candidate = s1[0] + for s2 in sequences: + if candidate in s2[1:]: + candidate = None + break # reject the current head, it appears later + else: + break + if candidate is None: + raise RuntimeError("Inconsistent hierarchy") + result.append(candidate) + # remove the chosen candidate + for seq in sequences: + if seq[0] == candidate: + del seq[0] + +def _c3_mro(cls, abcs=None): + """Computes the method resolution order using extended C3 linearization. + + If no *abcs* are given, the algorithm works exactly like the built-in C3 + linearization used for method resolution. + + If given, *abcs* is a list of abstract base classes that should be inserted + into the resulting MRO. Unrelated ABCs are ignored and don't end up in the + result. The algorithm inserts ABCs where their functionality is introduced, + i.e. issubclass(cls, abc) returns True for the class itself but returns + False for all its direct base classes. Implicit ABCs for a given class + (either registered or inferred from the presence of a special method like + __len__) are inserted directly after the last ABC explicitly listed in the + MRO of said class. If two implicit ABCs end up next to each other in the + resulting MRO, their ordering depends on the order of types in *abcs*. + + """ + for i, base in enumerate(reversed(cls.__bases__)): + if hasattr(base, '__abstractmethods__'): + boundary = len(cls.__bases__) - i + break # Bases up to the last explicit ABC are considered first. + else: + boundary = 0 + abcs = list(abcs) if abcs else [] + explicit_bases = list(cls.__bases__[:boundary]) + abstract_bases = [] + other_bases = list(cls.__bases__[boundary:]) + for base in abcs: + if issubclass(cls, base) and not any( + issubclass(b, base) for b in cls.__bases__ + ): + # If *cls* is the class that introduces behaviour described by + # an ABC *base*, insert said ABC to its MRO. + abstract_bases.append(base) + for base in abstract_bases: + abcs.remove(base) + explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases] + abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases] + other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases] + return _c3_merge( + [[cls]] + + explicit_c3_mros + abstract_c3_mros + other_c3_mros + + [explicit_bases] + [abstract_bases] + [other_bases] + ) + +def _compose_mro(cls, types): + """Calculates the method resolution order for a given class *cls*. + + Includes relevant abstract base classes (with their respective bases) from + the *types* iterable. Uses a modified C3 linearization algorithm. + + """ + bases = set(cls.__mro__) + # Remove entries which are already present in the __mro__ or unrelated. + def is_related(typ): + return (typ not in bases and hasattr(typ, '__mro__') + and not isinstance(typ, GenericAlias) + and issubclass(cls, typ)) + types = [n for n in types if is_related(n)] + # Remove entries which are strict bases of other entries (they will end up + # in the MRO anyway. + def is_strict_base(typ): + for other in types: + if typ != other and typ in other.__mro__: + return True + return False + types = [n for n in types if not is_strict_base(n)] + # Subclasses of the ABCs in *types* which are also implemented by + # *cls* can be used to stabilize ABC ordering. + type_set = set(types) + mro = [] + for typ in types: + found = [] + for sub in typ.__subclasses__(): + if sub not in bases and issubclass(cls, sub): + found.append([s for s in sub.__mro__ if s in type_set]) + if not found: + mro.append(typ) + continue + # Favor subclasses with the biggest number of useful bases + found.sort(key=len, reverse=True) + for sub in found: + for subcls in sub: + if subcls not in mro: + mro.append(subcls) + return _c3_mro(cls, abcs=mro) + +def _find_impl(cls, registry): + """Returns the best matching implementation from *registry* for type *cls*. + + Where there is no registered implementation for a specific type, its method + resolution order is used to find a more generic implementation. + + Note: if *registry* does not contain an implementation for the base + *object* type, this function may return None. + + """ + mro = _compose_mro(cls, registry.keys()) + match = None + for t in mro: + if match is not None: + # If *match* is an implicit ABC but there is another unrelated, + # equally matching implicit ABC, refuse the temptation to guess. + if (t in registry and t not in cls.__mro__ + and match not in cls.__mro__ + and not issubclass(match, t)): + raise RuntimeError("Ambiguous dispatch: {} or {}".format( + match, t)) + break + if t in registry: + match = t + return registry.get(match) def singledispatch(func): - """Single-dispatch generic-function decorator. - - Mirrors :func:`functools.singledispatch`. Subsequent calls to the - returned callable dispatch on the *runtime* type of the first - argument; alternative implementations are registered with - ``@my_func.register(type)``. - - Notes: - - We don't honour the C-extension's caching of resolved types; - the linear walk over registered types is fast enough for our - target workloads. - - PEP 585 / annotation-based registration is omitted because we - don't yet have a stable ``get_type_hints`` story for module- - level functions defined in WeavePy. + """Single-dispatch generic function decorator. + + Transforms a function into a generic function, which can have different + behaviours depending upon the type of its first argument. The decorated + function acts as the default implementation, and additional + implementations can be registered using the register() attribute of the + generic function. """ - return _SingleDispatchCallable(func) + # There are many programs that use functools without singledispatch, so we + # trade-off making singledispatch marginally slower for the benefit of + # making start-up of such applications slightly faster. + import types, weakref + + registry = {} + dispatch_cache = weakref.WeakKeyDictionary() + cache_token = None + + def dispatch(cls): + """generic_func.dispatch(cls) -> + + Runs the dispatch algorithm to return the best available implementation + for the given *cls* registered on *generic_func*. + + """ + nonlocal cache_token + if cache_token is not None: + current_token = get_cache_token() + if cache_token != current_token: + dispatch_cache.clear() + cache_token = current_token + try: + impl = dispatch_cache[cls] + except KeyError: + try: + impl = registry[cls] + except KeyError: + impl = _find_impl(cls, registry) + dispatch_cache[cls] = impl + return impl + def _is_union_type(cls): + from typing import get_origin, Union + return get_origin(cls) in {Union, types.UnionType} -# ---- cached_property -------------------------------------------------------- + def _is_valid_dispatch_type(cls): + if isinstance(cls, type): + return True + from typing import get_args + return (_is_union_type(cls) and + all(isinstance(arg, type) for arg in get_args(cls))) + def register(cls, func=None): + """generic_func.register(cls, func) -> func -_MISSING = object() + Registers a new implementation for the given *cls* on a *generic_func*. + """ + nonlocal cache_token + if _is_valid_dispatch_type(cls): + if func is None: + return lambda f: register(cls, f) + else: + if func is not None: + raise TypeError( + f"Invalid first argument to `register()`. " + f"{cls!r} is not a class or union type." + ) + ann = getattr(cls, '__annotations__', {}) + if not ann: + raise TypeError( + f"Invalid first argument to `register()`: {cls!r}. " + f"Use either `@register(some_class)` or plain `@register` " + f"on an annotated function." + ) + func = cls + + # only import typing if annotation parsing is necessary + from typing import get_type_hints + argname, cls = next(iter(get_type_hints(func).items())) + if not _is_valid_dispatch_type(cls): + if _is_union_type(cls): + raise TypeError( + f"Invalid annotation for {argname!r}. " + f"{cls!r} not all arguments are classes." + ) + else: + raise TypeError( + f"Invalid annotation for {argname!r}. " + f"{cls!r} is not a class." + ) + + if _is_union_type(cls): + from typing import get_args + + for arg in get_args(cls): + registry[arg] = func + else: + registry[cls] = func + if cache_token is None and hasattr(cls, '__abstractmethods__'): + cache_token = get_cache_token() + dispatch_cache.clear() + return func -class cached_property: - """Method decorator turning ``self.foo`` into a once-computed attr. + def wrapper(*args, **kw): + if not args: + raise TypeError(f'{funcname} requires at least ' + '1 positional argument') + return dispatch(args[0].__class__)(*args, **kw) + + funcname = getattr(func, '__name__', 'singledispatch function') + registry[object] = func + wrapper.register = register + wrapper.dispatch = dispatch + wrapper.registry = types.MappingProxyType(registry) + wrapper._clear_cache = dispatch_cache.clear + update_wrapper(wrapper, func) + return wrapper + + +# Descriptor version +class singledispatchmethod: + """Single-dispatch generic method descriptor. - Compared to :class:`property`, the value produced by the wrapped - function is stored back onto the instance's ``__dict__`` under the - attribute's name, so subsequent accesses short-circuit the - descriptor and don't re-enter the wrapped function. + Supports wrapping existing descriptors. """ + def __init__(self, func): + if not callable(func) and not hasattr(func, "__get__"): + raise TypeError(f"{func!r} is not callable or a descriptor") + + self.dispatcher = singledispatch(func) + self.func = func + + def register(self, cls, method=None): + """generic_method.register(cls, func) -> func + + Registers a new implementation for the given *cls* on a *generic_method*. + """ + return self.dispatcher.register(cls, func=method) + + def __get__(self, obj, cls=None): + dispatch = self.dispatcher.dispatch + funcname = getattr(self.func, '__name__', 'singledispatchmethod method') + def _method(*args, **kwargs): + if not args: + raise TypeError(f'{funcname} requires at least ' + '1 positional argument') + return dispatch(args[0].__class__).__get__(obj, cls)(*args, **kwargs) + + _method.__isabstractmethod__ = self.__isabstractmethod__ + _method.register = self.register + update_wrapper(_method, self.func) + + return _method + + @property + def __isabstractmethod__(self): + return getattr(self.func, '__isabstractmethod__', False) + + +################################################################################ +### cached_property() - property result cached as instance attribute +################################################################################ + +_NOT_FOUND = object() + +class cached_property: def __init__(self, func): self.func = func self.attrname = None - self.__doc__ = getattr(func, "__doc__", None) + self.__doc__ = func.__doc__ + self.__module__ = func.__module__ def __set_name__(self, owner, name): if self.attrname is None: self.attrname = name elif name != self.attrname: raise TypeError( - "Cannot assign the same cached_property to two different names" + "Cannot assign the same cached_property to two different names " + f"({self.attrname!r} and {name!r})." ) def __get__(self, instance, owner=None): @@ -531,10 +1011,26 @@ def __get__(self, instance, owner=None): return self if self.attrname is None: raise TypeError( - "Cannot use cached_property instance without calling __set_name__" + "Cannot use cached_property instance without calling __set_name__ on it.") + try: + cache = instance.__dict__ + except AttributeError: # not all objects have __dict__ (e.g. class defines slots) + msg = ( + f"No '__dict__' attribute on {type(instance).__name__!r} " + f"instance to cache {self.attrname!r} property." ) - cached = instance.__dict__.get(self.attrname, _MISSING) - if cached is _MISSING: - cached = self.func(instance) - instance.__dict__[self.attrname] = cached - return cached + raise TypeError(msg) from None + val = cache.get(self.attrname, _NOT_FOUND) + if val is _NOT_FOUND: + val = self.func(instance) + try: + cache[self.attrname] = val + except TypeError: + msg = ( + f"The '__dict__' attribute on {type(instance).__name__!r} instance " + f"does not support item assignment for caching {self.attrname!r} property." + ) + raise TypeError(msg) from None + return val + + __class_getitem__ = classmethod(GenericAlias) diff --git a/crates/weavepy-vm/src/stdlib/python/pickle.py b/crates/weavepy-vm/src/stdlib/python/pickle.py index d7edd29..a00553f 100644 --- a/crates/weavepy-vm/src/stdlib/python/pickle.py +++ b/crates/weavepy-vm/src/stdlib/python/pickle.py @@ -49,6 +49,19 @@ FROZENSET = b"\x91" MARK = b"(" STOP = b"." +POP = b"0" +POP_MARK = b"1" +# Memo opcodes — preserve object identity/sharing and enable cyclic +# structures. PUT/GET use a textual index (protocol 0), BINPUT/BINGET a +# 1-byte index, LONG_BINPUT/LONG_BINGET a 4-byte index, and MEMOIZE +# (protocol 4+) appends the stack top to the memo with no explicit index. +PUT = b"p" +BINPUT = b"q" +LONG_BINPUT = b"r" +GET = b"g" +BINGET = b"h" +LONG_BINGET = b"j" +MEMOIZE = b"\x94" # Global reference + reduce opcodes used to serialize functions and # classes by their qualified name. CPython uses these for everything # from `pickle.dumps(int)` to `pickle.dumps(my_module.my_func)`. @@ -127,12 +140,41 @@ def __init__(self, buf, protocol): self.protocol = protocol self.bin = protocol >= 1 self.fast = False + # id(obj) -> (memo_index, obj). Keeping a reference to `obj` + # prevents its id from being reused mid-pickle. + self.memo = {} def dump(self, obj): self._buf.write(PROTO + bytes([self.protocol])) self._save(obj) self._buf.write(STOP) + def _memoize(self, obj): + """Record `obj` (already written / on the stack) in the memo and + emit the PUT opcode so a later occurrence can reference it.""" + if self.fast or id(obj) in self.memo: + return + idx = len(self.memo) + if self.protocol >= 4: + self._buf.write(MEMOIZE) + elif self.bin: + if idx < 256: + self._buf.write(BINPUT + bytes([idx])) + else: + self._buf.write(LONG_BINPUT + struct.pack("' + # and on-disk modules report a path. + if getattr(mod, "__file__", None) is None: + import types + fresh_mod = types.ModuleType(getattr(mod, "__name__", name)) + fresh_mod.__dict__.update(mod.__dict__) + return fresh_mod + return mod finally: if blocker is not None: try: diff --git a/crates/weavepy-vm/src/stdlib/python/traceback.py b/crates/weavepy-vm/src/stdlib/python/traceback.py index 53df880..fbfc8cb 100644 --- a/crates/weavepy-vm/src/stdlib/python/traceback.py +++ b/crates/weavepy-vm/src/stdlib/python/traceback.py @@ -110,6 +110,37 @@ def walk_tb(tb): tb = tb.tb_next +def _get_code_position(code, instruction_index): + """PEP-657 (lineno, end_lineno, colno, end_colno) for a bytecode offset. + + `co_positions()` yields one tuple per *code unit* (2 bytes), so the + instruction byte offset maps to entry `instruction_index // 2`. + """ + if instruction_index is None or instruction_index < 0: + return (None, None, None, None) + try: + positions = list(code.co_positions()) + except Exception: + return (None, None, None, None) + idx = instruction_index // 2 + if 0 <= idx < len(positions): + return positions[idx] + return (None, None, None, None) + + +def _walk_tb_with_full_positions(tb): + # Like walk_tb, but yields full code positions (end line + columns). + while tb is not None: + positions = _get_code_position(tb.tb_frame.f_code, tb.tb_lasti) + # Fall back to tb_lineno when co_positions has no line, matching + # walk_tb's behavior. + if positions[0] is None: + yield tb.tb_frame, (tb.tb_lineno,) + tuple(positions[1:]) + else: + yield tb.tb_frame, positions + tb = tb.tb_next + + class StackSummary: """A sequence of FrameSummary objects with extra formatting helpers. @@ -138,6 +169,22 @@ def __bool__(self): @classmethod def extract(cls, frame_gen, *, limit=None, lookup_lines=True, capture_locals=False): + # `frame_gen` yields plain (frame, lineno) pairs (no column info). + # Adapt to the extended generator the position-aware path consumes. + def extended_frame_gen(): + for f, lineno in frame_gen: + yield f, (lineno, None, None, None) + + return cls._extract_from_extended_frame_gen( + extended_frame_gen(), limit=limit, lookup_lines=lookup_lines, + capture_locals=capture_locals) + + @classmethod + def _extract_from_extended_frame_gen(cls, frame_gen, *, limit=None, + lookup_lines=True, capture_locals=False): + # Like `extract`, but consumes (frame, (lineno, end_lineno, colno, + # end_colno)) tuples so PEP-657 column anchors survive into each + # FrameSummary. Only lineno is required; the rest may be None. if limit is None: limit = getattr(sys, "tracebacklimit", None) if isinstance(limit, int) and limit < 0: @@ -146,7 +193,7 @@ def extract(cls, frame_gen, *, limit=None, lookup_lines=True, capture_locals=Fal frames = list(frame_gen) if isinstance(limit, int): frames = frames[-limit:] if limit else [] - for f, lineno in frames: + for f, (lineno, end_lineno, colno, end_colno) in frames: try: co = f.f_code filename = getattr(co, "co_filename", "") @@ -157,7 +204,8 @@ def extract(cls, frame_gen, *, limit=None, lookup_lines=True, capture_locals=Fal locals_ = f.f_locals if capture_locals else None result.append( FrameSummary(filename, lineno, name, lookup_line=lookup_lines, - locals=locals_) + locals=locals_, end_lineno=end_lineno, + colno=colno, end_colno=end_colno) ) return result @@ -210,7 +258,8 @@ def format_frame_summary(self, frame_summary): def extract_tb(tb, limit=None): - return StackSummary.extract(walk_tb(tb), limit=limit) + return StackSummary._extract_from_extended_frame_gen( + _walk_tb_with_full_positions(tb), limit=limit) def extract_stack(f=None, limit=None): @@ -319,9 +368,9 @@ def __init__(self, exc_type, exc_value, exc_tb, *, limit=None, max_group_width=15, max_group_depth=10, _seen=None): self.exc_type = exc_type self._str = str(exc_value) if exc_value is not None else "" - self.stack = StackSummary.extract(walk_tb(exc_tb), limit=limit, - lookup_lines=lookup_lines, - capture_locals=capture_locals) + self.stack = StackSummary._extract_from_extended_frame_gen( + _walk_tb_with_full_positions(exc_tb), limit=limit, + lookup_lines=lookup_lines, capture_locals=capture_locals) self.filename = getattr(exc_value, "filename", None) self.lineno = getattr(exc_value, "lineno", None) self.text = getattr(exc_value, "text", None) diff --git a/crates/weavepy-vm/src/stdlib/python/typing.py b/crates/weavepy-vm/src/stdlib/python/typing.py index 7bb36b6..02c00dc 100644 --- a/crates/weavepy-vm/src/stdlib/python/typing.py +++ b/crates/weavepy-vm/src/stdlib/python/typing.py @@ -57,6 +57,10 @@ def __call__(self, *args, **kwargs): InitVar = _SpecialForm("InitVar") Union = _SpecialForm("Union") Literal = _SpecialForm("Literal") +# PEP 646: ``Unpack[Ts]`` / ``*Ts``. Iterating a PEP 585 generic alias +# (``tuple[int]``) yields ``Unpack[self]`` exactly once, mirroring +# CPython's ``ga_iternext`` (which lazily reaches ``typing.Unpack``). +Unpack = _SpecialForm("Unpack") def Optional(*params): diff --git a/crates/weavepy-vm/src/trace.rs b/crates/weavepy-vm/src/trace.rs index 82e9b06..675d1ca 100644 --- a/crates/weavepy-vm/src/trace.rs +++ b/crates/weavepy-vm/src/trace.rs @@ -80,6 +80,16 @@ impl MonitoringTools { } } +/// Which observer slot a hook invocation belongs to. Used so that +/// when a hook raises on a non-`exception` event we can disable the +/// *right* hook (CPython turns off the offending trace/profile +/// function and re-raises). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum HookKind { + Trace, + Profile, +} + pub fn set_trace_hook(hook: Object) { TRACE_HOOK.with(|cell| { *cell.borrow_mut() = match hook { diff --git a/crates/weavepy/src/lib.rs b/crates/weavepy/src/lib.rs index 95592ae..596abfe 100644 --- a/crates/weavepy/src/lib.rs +++ b/crates/weavepy/src/lib.rs @@ -262,7 +262,14 @@ pub fn run_source_with_options(source: &str, opts: &RunOptions) -> Result<(), Er } else { Some(opts.filename.as_str()) }; - let _ = interpreter.run_module_as(&code, "__main__", file_for_main)?; + let result = interpreter.run_module_as(&code, "__main__", file_for_main); + // CPython runs finalizers for everything still alive during + // interpreter shutdown — including a module-global object whose + // `__del__` raises (which is reported via `sys.unraisablehook`). + // Do this whether the module returned normally or via `SystemExit`, + // before the caller turns a `SystemExit` into a process exit. + interpreter.run_shutdown_finalizers(); + result?; Ok(()) } diff --git a/tests/regrtest/expectations.toml b/tests/regrtest/expectations.toml index a36af42..c02a718 100644 --- a/tests/regrtest/expectations.toml +++ b/tests/regrtest/expectations.toml @@ -316,8 +316,8 @@ status = "fail" reason = "measured: object.__module__ + unbound type-methods now resolve (RFC 0037 WS6); the suite still reports a large mix of errors/failures across the descriptor-protocol subtests (no single root cause — slots, __set_name__, metaclass corners)." [tests."cpython/Lib/test/test_iter.py"] -status = "skip" -reason = "measured: now that the test.support helpers + collections.abc resolve (RFC 0037 WS8/WS9) the suite runs far enough to reach the same gc.collect() reachable-hang as test_set (iterator/collection reference cycles + weakrefs). Marked skip so CI doesn't stall 30s per run; tracked with test_set as a GC reachable-hang to revisit in wave 3." +status = "pass" +reason = "RFC 0037 WS7: full iterator-protocol fidelity. The prior gc-reachable hang is gone — the legacy __getitem__ sequence protocol and iter(callable, sentinel) now build *lazy* iterators (frozen _seqtools _SeqIter/_CallableIter) instead of eagerly materialising, so an unbounded sequence iterates on demand. Built-in iterators gained a faithful __reduce__ ((iter, (remaining,)) / (reversed, (fwd,), idx)) that resolves the iter/reversed builtin through the live builtins module dict so a hash-colliding custom __eq__ exhausts the iterator before its state is snapshotted (gh-101765); a PEP 585 generic alias iterates by yielding typing.Unpack[self] once (matching CPython ga_iternext). Plus: file objects are iterable (for line in f / x in f / list(f)) and writelines accepts any iterable; pickle gained memoisation so co-referenced instances unpickle shared; closures over enclosing-function locals resolve in nested methods; and tracebacks carry PEP 657 column offsets (co_positions/f_lasti/tb_lasti translated to CPython byte offsets). (skipped=2 are @cpython_only refcount subtests.)" [tests."cpython/Lib/test/test_generators.py"] status = "fail" @@ -444,8 +444,8 @@ status = "skip" reason = "RFC 0036: name/lookup now use the full UCD table (unicode_names2), but the full NormalizationTest sweep exceeds the per-test budget and residual failures track the UCD-version skew (engine ships 16.0.0, CPython 3.13 pins 15.1.0)" [tests."cpython/Lib/test/test_struct.py"] -status = "fail" -reason = "measured: first remaining failure is a Rust panic (RUST_BACKTRACE note in output) inside the struct module path — a pack/unpack edge case aborts rather than raising struct.error; needs a panic-to-exception guard in the struct builtin." +status = "pass" +reason = "RFC 0037 WS8: faithful struct port (buffer-protocol pack_into/unpack_from, half-float round-half-to-even with OverflowError, embedded-NUL/format validation, UnicodeEncodeError on non-ASCII formats, real unpack_iterator). The reference-cycle and runtime-shutdown subtests pass via new interpreter features: import_fresh_module hands back a collectable copy of native singletons, and shutdown finalization runs __del__ for live objects with the default sys.unraisablehook printing 'Exception ignored in:'. (skipped=4 are CPython _testcapi/refcount-only subtests.)" [tests."cpython/Lib/test/test_codecs.py"] status = "fail" @@ -582,8 +582,8 @@ reason = "RFC 0036: passes end-to-end (bigaddrspacetest fixtures skip cleanly wi # Genuine WeavePy gaps — run far enough to fail with a specific cause. [tests."cpython/Lib/test/test_bdb.py"] -status = "fail" -reason = "measured: patch_list + the other test.support helpers now import (RFC 0037 WS9); first remaining failure is 'type object Breakpoint has no attribute clearBreakpoints' — the bundled bdb module is missing Breakpoint classmethods." +status = "pass" +reason = "RFC 0037 WS9: passes end-to-end (36 tests). Required faithful sys.settrace event fidelity in the VM dispatch loop: the frame-entry RESUME no longer emits a spurious 'line' event; an exception raised *inside* a trace callback propagates into the traced program (and disables the offending hook) for non-'exception' events while being swallowed on 'exception' events, matching CPython's call_trace_protected; Object::Frame compares by identity so bdb's `frame == self.stopframe`/returnframe checks work; generator/coroutine frames reuse one cached PyFrame snapshot across suspensions so frame identity is stable for set_next/until/return; FOR_ITER and SEND (`yield from`) surface a generator's terminating StopIteration to the 'exception' hook before swallowing it; END_FOR is attributed to the `for` line (not the loop body) so an exhausted loop emits no spurious body-line event; a frame popped by a propagating exception fires a 'return' event with arg None (sys.monitoring PY_UNWIND); and f_trace_lines / f_trace_opcodes are real per-frame flags driving 'opcode' events for bdb/pdb instruction stepping (set_stepinstr)." [tests."cpython/Lib/test/test_contextlib_async.py"] status = "fail" From 6f29b1a7359706f38967dbd1687166112e50534c Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Mon, 8 Jun 2026 19:58:21 -0700 Subject: [PATCH 4/9] feat: advance CPython Lib/test conformance wave 2 --- crates/weavepy-vm/src/builtins.rs | 24 + crates/weavepy-vm/src/lib.rs | 86 + crates/weavepy-vm/src/stdlib/mod.rs | 8 + .../weavepy-vm/src/stdlib/python/_seqtools.py | 13 + .../weavepy-vm/src/stdlib/python/asyncio.py | 55 + .../src/stdlib/python/dataclasses.py | 2025 +++++++++++++---- .../weavepy-vm/src/stdlib/python/inspect.py | 96 + .../weavepy-vm/src/stdlib/python/keyword.py | 58 + crates/weavepy-vm/src/stdlib/python/typing.py | 29 + 9 files changed, 1912 insertions(+), 482 deletions(-) create mode 100644 crates/weavepy-vm/src/stdlib/python/keyword.py diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 2de2d93..e6e36e2 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -184,6 +184,8 @@ pub fn default_builtins() -> DictData { reg!("input", b_input_unsupported); reg!("next", b_next); reg!("iter", b_iter); + reg!("aiter", b_aiter); + reg!("anext", b_anext); reg!("divmod", b_divmod); reg!("round", b_round); reg!("format", b_format); @@ -518,6 +520,11 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "symmetric_difference_update", set_symmetric_difference_update, )), + // Membership dunder exposed as a bound method: CPython's + // `keyword.iskeyword = frozenset(kwlist).__contains__` grabs it + // directly, and `hasattr(s, '__contains__')` must hold. + "__contains__" => Some(method("__contains__", obj_contains)), + "__len__" => Some(method("__len__", obj_len)), _ => None, }, Object::Bytes(_) | Object::ByteArray(_) => match name { @@ -559,6 +566,10 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "reverse" if matches!(obj, Object::ByteArray(_)) => { Some(method("reverse", bytearray_reverse)) } + // Sequence dunders so direct calls / `hasattr` parity hold. + "__contains__" => Some(method("__contains__", obj_contains)), + "__len__" => Some(method("__len__", obj_len)), + "__getitem__" => Some(method("__getitem__", seq_getitem)), _ => None, }, Object::File(_) => match name { @@ -4803,6 +4814,19 @@ fn b_iter(args: &[Object]) -> Result { Ok(Object::Iter(Rc::new(RefCell::new(it)))) } +/// `aiter(async_iterable)` — return its async iterator (PEP 525 builtin, +/// 3.10+). VM-routed through [`crate::Vm::get_aiter`] so `__aiter__` +/// dispatch runs; this fallback only fires if invoked outside the VM. +fn b_aiter(_args: &[Object]) -> Result { + Err(type_error("aiter() must be called through the VM")) +} + +/// `anext(async_iterator[, default])` — return the awaitable from +/// `__anext__` (3.10+). VM-routed through [`crate::Vm::get_anext`]. +fn b_anext(_args: &[Object]) -> Result { + Err(type_error("anext() must be called through the VM")) +} + pub(crate) fn b_divmod(args: &[Object]) -> Result { if args.len() != 2 { return Err(type_error("divmod expected 2 arguments")); diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index aaf017b..1255f8b 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -5379,6 +5379,39 @@ impl Interpreter { } } + /// `anext(aiter, default)` — wrap the `__anext__` awaitable in a frozen + /// coroutine that returns `default` on `StopAsyncIteration`. Returns + /// `None` if the helper module is unavailable so the caller can fall + /// back to the bare (no-default) awaitable. + fn make_anext_with_default( + &mut self, + awaitable: &Object, + default: &Object, + globals: &Rc>, + ) -> Result, RuntimeError> { + let module = match self.do_import("_seqtools", &Object::None, 0, globals) { + Ok(m) => m, + Err(_) => return Ok(None), + }; + let func = match &module { + Object::Module(m) => m + .dict + .borrow() + .get(&DictKey(Object::from_static("_anext_with_default"))) + .cloned(), + _ => None, + }; + match func { + Some(func) => Ok(Some(self.call( + &func, + &[awaitable.clone(), default.clone()], + &[], + globals, + )?)), + None => Ok(None), + } + } + fn do_list_or_tuple_call( &mut self, name: &str, @@ -9051,6 +9084,26 @@ impl Interpreter { // for-loop machinery just works. return self.do_iter_callable_sentinel(args, outer_globals); } + // ``aiter(async_iterable)`` / ``anext(async_iterator[, default])`` + // — the PEP 525 async builtins (3.10+). Routed through the + // VM so `__aiter__` / `__anext__` dispatch runs (including + // for native async generators that don't carry the dunders + // as ordinary methods). + if b.name == "aiter" && args.len() == 1 { + return self.get_aiter(args[0].clone(), outer_globals); + } + if b.name == "anext" && args.len() == 1 { + return self.get_anext(&args[0], outer_globals); + } + if b.name == "anext" && args.len() == 2 { + let awaitable = self.get_anext(&args[0], outer_globals)?; + if let Some(coro) = + self.make_anext_with_default(&awaitable, &args[1], outer_globals)? + { + return Ok(coro); + } + return Ok(awaitable); + } if (b.name == "list" || b.name == "tuple") && args.len() == 1 { return self.do_list_or_tuple_call(b.name, &args[0], outer_globals); } @@ -11488,6 +11541,17 @@ impl Interpreter { Some(Object::None) | None => outer_globals.clone(), _ => return Err(type_error("exec() globals must be a dict")), }; + // The optional third argument is the *locals* namespace. When a + // distinct mapping is supplied, top-level `STORE_NAME`/`def`/`class` + // must land there (not in globals) and bare-name lookups resolve + // it first — exactly the class-body scoping the VM already models + // via `class_namespace`. CPython drives `exec(src, g, l)` codegen + // (e.g. `dataclasses` building `__init__`) this way. + let exec_locals: Option>> = match args.get(2) { + Some(Object::Dict(d)) if !Rc::ptr_eq(d, &globals_dict) => Some(d.clone()), + Some(Object::Dict(_)) | Some(Object::None) | None => None, + _ => return Err(type_error("exec() locals must be a mapping")), + }; let code_rc = match source { Object::Code(c) => c, Object::Str(src) => { @@ -11521,6 +11585,10 @@ impl Interpreter { } } let mut frame = self.make_frame(code_rc, Vec::new(), Vec::new(), globals_dict, true); + // Run top-level names into the distinct locals mapping when present. + if let Some(locals) = exec_locals { + frame.class_namespace = Some(locals); + } self.run_frame(&mut frame)?; Ok(Object::None) } @@ -11991,6 +12059,24 @@ impl Interpreter { .insert(DictKey(Object::from_str(name)), sub.clone()); return Ok(sub); } + // PEP 562: `from module import name` consults a module-level + // `__getattr__(name)` before failing, mirroring CPython's + // `import_from`, which falls back to `getattr(module, name)`. + let getattr = m + .dict + .borrow() + .get(&DictKey(Object::from_str("__getattr__"))) + .cloned(); + if let Some(getattr) = getattr { + let globals = m.dict.clone(); + match self.call(&getattr, &[Object::from_str(name)], &[], &globals) { + Ok(v) => return Ok(v), + // An `AttributeError` from `__getattr__` becomes the + // canonical `ImportError`; anything else propagates. + Err(e) if self.is_attribute_error(&e) => {} + Err(e) => return Err(e), + } + } Err(import_error(format!( "cannot import name '{name}' from '{}'", m.name diff --git a/crates/weavepy-vm/src/stdlib/mod.rs b/crates/weavepy-vm/src/stdlib/mod.rs index d7cdbd6..65894dd 100644 --- a/crates/weavepy-vm/src/stdlib/mod.rs +++ b/crates/weavepy-vm/src/stdlib/mod.rs @@ -171,6 +171,14 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/builtins.py"), is_package: false, }, + // `keyword` — verbatim CPython keyword/soft-keyword lists + + // membership predicates. Imported by `dataclasses` (field-name + // validation) and `pydoc`/`inspect`-adjacent code. + FrozenSource { + name: "keyword", + source: include_str!("python/keyword.py"), + is_package: false, + }, // Internal: `_SeqIter`, the lazy legacy-`__getitem__` iterator // `iter(obj)` returns when *obj* has no `__iter__` (CPython's // built-in `iterator`/seqiterobject). Kept out of `builtins` to diff --git a/crates/weavepy-vm/src/stdlib/python/_seqtools.py b/crates/weavepy-vm/src/stdlib/python/_seqtools.py index addf716..ca2399e 100644 --- a/crates/weavepy-vm/src/stdlib/python/_seqtools.py +++ b/crates/weavepy-vm/src/stdlib/python/_seqtools.py @@ -14,6 +14,19 @@ """ +async def _anext_with_default(awaitable, default): + """Back the two-argument ``anext(aiter, default)`` builtin. + + The VM hands us the already-resolved ``__anext__`` awaitable; we + return ``default`` when the async iterator is exhausted, matching + CPython's ``anext`` C wrapper. + """ + try: + return await awaitable + except StopAsyncIteration: + return default + + def _builtin_iter(): """Fetch ``iter`` from the live ``builtins`` *module* namespace. diff --git a/crates/weavepy-vm/src/stdlib/python/asyncio.py b/crates/weavepy-vm/src/stdlib/python/asyncio.py index 5d96679..5078397 100644 --- a/crates/weavepy-vm/src/stdlib/python/asyncio.py +++ b/crates/weavepy-vm/src/stdlib/python/asyncio.py @@ -598,6 +598,57 @@ def get_running_loop(): return _current_loop +# ---- event loop policy -------------------------------------------- +# +# The policy layer is deprecated in 3.12+ but the stdlib and its test +# suite (e.g. ``IsolatedAsyncioTestCase`` helpers in test_contextlib_async) +# still reach for ``get_event_loop_policy().get_event_loop()``. We ship a +# faithful default policy that simply delegates to the module-level loop +# accessors above. + + +class AbstractEventLoopPolicy: + def get_event_loop(self): + raise NotImplementedError + + def set_event_loop(self, loop): + raise NotImplementedError + + def new_event_loop(self): + raise NotImplementedError + + +class DefaultEventLoopPolicy(AbstractEventLoopPolicy): + def get_event_loop(self): + return get_event_loop() + + def set_event_loop(self, loop): + set_event_loop(loop) + + def new_event_loop(self): + return new_event_loop() + + +_event_loop_policy = None + + +def get_event_loop_policy(): + global _event_loop_policy + if _event_loop_policy is None: + _event_loop_policy = DefaultEventLoopPolicy() + return _event_loop_policy + + +def set_event_loop_policy(policy): + global _event_loop_policy + if policy is not None and not isinstance(policy, AbstractEventLoopPolicy): + raise TypeError( + f"policy must be an instance of AbstractEventLoopPolicy or None, " + f"not '{type(policy).__name__}'" + ) + _event_loop_policy = policy + + # ---- run / ensure_future ------------------------------------------ @@ -1786,6 +1837,10 @@ def run_coroutine_threadsafe(coro, loop=None): "new_event_loop", "set_event_loop", "get_running_loop", + "get_event_loop_policy", + "set_event_loop_policy", + "AbstractEventLoopPolicy", + "DefaultEventLoopPolicy", "Lock", "Event", "Semaphore", diff --git a/crates/weavepy-vm/src/stdlib/python/dataclasses.py b/crates/weavepy-vm/src/stdlib/python/dataclasses.py index c67b2f3..7883ce7 100644 --- a/crates/weavepy-vm/src/stdlib/python/dataclasses.py +++ b/crates/weavepy-vm/src/stdlib/python/dataclasses.py @@ -1,569 +1,1630 @@ -"""Dataclasses, the WeavePy edition. - -Implements the surface most code reaches for: - -- ``@dataclass`` (with ``init``, ``repr``, ``eq``, ``order``, ``frozen``, - ``slots``, ``kw_only``) -- ``field(default=..., default_factory=..., repr=..., compare=..., - init=..., kw_only=...)`` -- ``fields(cls_or_instance)`` -- ``asdict(obj)`` / ``astuple(obj)`` -- ``replace(obj, **changes)`` -- ``make_dataclass(name, fields, ...)`` -- ``is_dataclass(obj)`` - -Notable omissions (deferred): +import re +import sys +import copy +import types +import inspect +import keyword +import itertools +import abc +from reprlib import recursive_repr + + +__all__ = ['dataclass', + 'field', + 'Field', + 'FrozenInstanceError', + 'InitVar', + 'KW_ONLY', + 'MISSING', + + # Helper functions. + 'fields', + 'asdict', + 'astuple', + 'make_dataclass', + 'replace', + 'is_dataclass', + ] + +# Conditions for adding methods. The boxes indicate what action the +# dataclass decorator takes. For all of these tables, when I talk +# about init=, repr=, eq=, order=, unsafe_hash=, or frozen=, I'm +# referring to the arguments to the @dataclass decorator. When +# checking if a dunder method already exists, I mean check for an +# entry in the class's __dict__. I never check to see if an attribute +# is defined in a base class. + +# Key: +# +=========+=========================================+ +# + Value | Meaning | +# +=========+=========================================+ +# | | No action: no method is added. | +# +---------+-----------------------------------------+ +# | add | Generated method is added. | +# +---------+-----------------------------------------+ +# | raise | TypeError is raised. | +# +---------+-----------------------------------------+ +# | None | Attribute is set to None. | +# +=========+=========================================+ + +# __init__ +# +# +--- init= parameter +# | +# v | | | +# | no | yes | <--- class has __init__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + +# __repr__ +# +# +--- repr= parameter +# | +# v | | | +# | no | yes | <--- class has __repr__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + + +# __setattr__ +# __delattr__ +# +# +--- frozen= parameter +# | +# v | | | +# | no | yes | <--- class has __setattr__ or __delattr__ in __dict__? +# +=======+=======+=======+ +# | False | | | <- the default +# +-------+-------+-------+ +# | True | add | raise | +# +=======+=======+=======+ +# Raise because not adding these methods would break the "frozen-ness" +# of the class. + +# __eq__ +# +# +--- eq= parameter +# | +# v | | | +# | no | yes | <--- class has __eq__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + +# __lt__ +# __le__ +# __gt__ +# __ge__ +# +# +--- order= parameter +# | +# v | | | +# | no | yes | <--- class has any comparison method in __dict__? +# +=======+=======+=======+ +# | False | | | <- the default +# +-------+-------+-------+ +# | True | add | raise | +# +=======+=======+=======+ +# Raise because to allow this case would interfere with using +# functools.total_ordering. + +# __hash__ + +# +------------------- unsafe_hash= parameter +# | +----------- eq= parameter +# | | +--- frozen= parameter +# | | | +# v v v | | | +# | no | yes | <--- class has explicitly defined __hash__ +# +=======+=======+=======+========+========+ +# | False | False | False | | | No __eq__, use the base class __hash__ +# +-------+-------+-------+--------+--------+ +# | False | False | True | | | No __eq__, use the base class __hash__ +# +-------+-------+-------+--------+--------+ +# | False | True | False | None | | <-- the default, not hashable +# +-------+-------+-------+--------+--------+ +# | False | True | True | add | | Frozen, so hashable, allows override +# +-------+-------+-------+--------+--------+ +# | True | False | False | add | raise | Has no __eq__, but hashable +# +-------+-------+-------+--------+--------+ +# | True | False | True | add | raise | Has no __eq__, but hashable +# +-------+-------+-------+--------+--------+ +# | True | True | False | add | raise | Not frozen, but hashable +# +-------+-------+-------+--------+--------+ +# | True | True | True | add | raise | Frozen, so hashable +# +=======+=======+=======+========+========+ +# For boxes that are blank, __hash__ is untouched and therefore +# inherited from the base class. If the base is object, then +# id-based hashing is used. +# +# Note that a class may already have __hash__=None if it specified an +# __eq__ method in the class body (not one that was created by +# @dataclass). +# +# See _hash_action (below) for a coded version of this table. + +# __match_args__ +# +# +--- match_args= parameter +# | +# v | | | +# | no | yes | <--- class has __match_args__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ +# __match_args__ is always added unless the class already defines it. It is a +# tuple of __init__ parameter names; non-init fields must be matched by keyword. + + +# Raised when an attempt is made to modify a frozen class. +class FrozenInstanceError(AttributeError): pass + +# A sentinel object for default values to signal that a default +# factory will be used. This is given a nice repr() which will appear +# in the function signature of dataclasses' constructors. +class _HAS_DEFAULT_FACTORY_CLASS: + def __repr__(self): + return '' +_HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() -- ``__set_name__`` integration with descriptor fields (we still - honour ``__set_name__`` on user descriptors, just don't *route* - field defaults through them). -- ``InitVar`` / ``ClassVar`` annotation introspection — both are - recognised as marker objects and excluded from the generated - ``__init__``, but the runtime doesn't enforce ``ClassVar`` access - control beyond the dataclass machinery. -- ``__match_args__`` synthesis (the AST already supports structural - patterns; we leave the field list to user code). -""" +# A sentinel object to detect if a parameter is supplied or not. Use +# a class to give it a better repr. +class _MISSING_TYPE: + pass +MISSING = _MISSING_TYPE() +# A sentinel object to indicate that following fields are keyword-only by +# default. Use a class to give it a better repr. +class _KW_ONLY_TYPE: + pass +KW_ONLY = _KW_ONLY_TYPE() -MISSING = object() -_HAS_DEFAULT_FACTORY = object() +# Since most per-field metadata will be unused, create an empty +# read-only proxy that can be shared among all fields. +_EMPTY_METADATA = types.MappingProxyType({}) +# Markers for the various kinds of fields and pseudo-fields. +class _FIELD_BASE: + def __init__(self, name): + self.name = name + def __repr__(self): + return self.name +_FIELD = _FIELD_BASE('_FIELD') +_FIELD_CLASSVAR = _FIELD_BASE('_FIELD_CLASSVAR') +_FIELD_INITVAR = _FIELD_BASE('_FIELD_INITVAR') + +# The name of an attribute on the class where we store the Field +# objects. Also used to check if a class is a Data Class. +_FIELDS = '__dataclass_fields__' + +# The name of an attribute on the class that stores the parameters to +# @dataclass. +_PARAMS = '__dataclass_params__' + +# The name of the function, that if it exists, is called at the end of +# __init__. +_POST_INIT_NAME = '__post_init__' + +# String regex that string annotations for ClassVar or InitVar must match. +# Allows "identifier.identifier[" or "identifier[". +# https://bugs.python.org/issue33453 for details. +_MODULE_IDENTIFIER_RE = re.compile(r'^(?:\s*(\w+)\s*\.)?\s*(\w+)') + +# Atomic immutable types which don't require any recursive handling and for which deepcopy +# returns the same object. We can provide a fast-path for these types in asdict and astuple. +_ATOMIC_TYPES = frozenset({ + # Common JSON Serializable types + types.NoneType, + bool, + int, + float, + str, + # Other common types + complex, + bytes, + # Other types that are also unaffected by deepcopy + types.EllipsisType, + types.NotImplementedType, + types.CodeType, + types.BuiltinFunctionType, + types.FunctionType, + type, + range, + property, +}) + + +class InitVar: + __slots__ = ('type', ) + + def __init__(self, type): + self.type = type + def __repr__(self): + if isinstance(self.type, type): + type_name = self.type.__name__ + else: + # typing objects, e.g. List[int] + type_name = repr(self.type) + return f'dataclasses.InitVar[{type_name}]' + + def __class_getitem__(cls, type): + return InitVar(type) + +# Instances of Field are only ever created from within this module, +# and only from the field() function, although Field instances are +# exposed externally as (conceptually) read-only objects. +# +# name and type are filled in after the fact, not in __init__. +# They're not known at the time this class is instantiated, but it's +# convenient if they're available later. +# +# When cls._FIELDS is filled in with a list of Field objects, the name +# and type fields will have been populated. class Field: - """Descriptor-friendly record carrying a single dataclass field's - metadata. Created by :func:`field`. Mirrors CPython's name and - attribute list closely so introspecting tools (`dataclasses.fields` - plus user code) keep working.""" - - __slots__ = ( - "name", - "type", - "default", - "default_factory", - "repr", - "hash", - "init", - "compare", - "metadata", - "kw_only", - "_field_type", - ) - - def __init__( - self, - default=MISSING, - default_factory=MISSING, - repr=True, - hash=None, - init=True, - compare=True, - metadata=None, - kw_only=False, - ): + __slots__ = ('name', + 'type', + 'default', + 'default_factory', + 'repr', + 'hash', + 'init', + 'compare', + 'metadata', + 'kw_only', + '_field_type', # Private: not to be used by user code. + ) + + def __init__(self, default, default_factory, init, repr, hash, compare, + metadata, kw_only): self.name = None self.type = None self.default = default self.default_factory = default_factory + self.init = init self.repr = repr self.hash = hash - self.init = init self.compare = compare - self.metadata = metadata or {} + self.metadata = (_EMPTY_METADATA + if metadata is None else + types.MappingProxyType(metadata)) self.kw_only = kw_only - self._field_type = "_FIELD" + self._field_type = None + @recursive_repr() def __repr__(self): - return ( - f"Field(name={self.name!r},type={self.type!r}," - f"default={self.default!r},default_factory={self.default_factory!r}," - f"repr={self.repr!r},compare={self.compare!r},init={self.init!r}," - f"kw_only={self.kw_only!r})" - ) - + return ('Field(' + f'name={self.name!r},' + f'type={self.type!r},' + f'default={self.default!r},' + f'default_factory={self.default_factory!r},' + f'init={self.init!r},' + f'repr={self.repr!r},' + f'hash={self.hash!r},' + f'compare={self.compare!r},' + f'metadata={self.metadata!r},' + f'kw_only={self.kw_only!r},' + f'_field_type={self._field_type}' + ')') + + # This is used to support the PEP 487 __set_name__ protocol in the + # case where we're using a field that contains a descriptor as a + # default value. For details on __set_name__, see + # https://peps.python.org/pep-0487/#implementation-details. + # + # Note that in _process_class, this Field object is overwritten + # with the default value, so the end result is a descriptor that + # had __set_name__ called on it at the right time. def __set_name__(self, owner, name): - self.name = name + func = getattr(type(self.default), '__set_name__', None) + if func: + # There is a __set_name__ method on the descriptor, call + # it. + func(self.default, owner, name) + __class_getitem__ = classmethod(types.GenericAlias) -def field( - *, - default=MISSING, - default_factory=MISSING, - repr=True, - hash=None, - init=True, - compare=True, - metadata=None, - kw_only=False, -): - """Marker used inside a dataclass body to control a field's - behaviour. Mirrors :func:`dataclasses.field` from CPython.""" - if default is not MISSING and default_factory is not MISSING: - raise ValueError("cannot specify both default and default_factory") - return Field( - default=default, - default_factory=default_factory, - repr=repr, - hash=hash, - init=init, - compare=compare, - metadata=metadata, - kw_only=kw_only, - ) - - -def _is_classvar(annotation): - # We accept either the stringly-typed `typing.ClassVar` marker or - # a runtime ClassVar instance (typing.py exposes the latter). - if annotation is None: - return False - if isinstance(annotation, str): - return annotation.startswith("ClassVar") - name = getattr(annotation, "__name__", "") - return name == "ClassVar" +class _DataclassParams: + __slots__ = ('init', + 'repr', + 'eq', + 'order', + 'unsafe_hash', + 'frozen', + 'match_args', + 'kw_only', + 'slots', + 'weakref_slot', + ) + + def __init__(self, + init, repr, eq, order, unsafe_hash, frozen, + match_args, kw_only, slots, weakref_slot): + self.init = init + self.repr = repr + self.eq = eq + self.order = order + self.unsafe_hash = unsafe_hash + self.frozen = frozen + self.match_args = match_args + self.kw_only = kw_only + self.slots = slots + self.weakref_slot = weakref_slot -def _is_initvar(annotation): - if annotation is None: - return False - if isinstance(annotation, str): - return annotation.startswith("InitVar") - name = getattr(annotation, "__name__", "") - return name == "InitVar" + def __repr__(self): + return ('_DataclassParams(' + f'init={self.init!r},' + f'repr={self.repr!r},' + f'eq={self.eq!r},' + f'order={self.order!r},' + f'unsafe_hash={self.unsafe_hash!r},' + f'frozen={self.frozen!r},' + f'match_args={self.match_args!r},' + f'kw_only={self.kw_only!r},' + f'slots={self.slots!r},' + f'weakref_slot={self.weakref_slot!r}' + ')') + + +# This function is used instead of exposing Field creation directly, +# so that a type checker can be told (via overloads) that this is a +# function whose type depends on its parameters. +def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, + hash=None, compare=True, metadata=None, kw_only=MISSING): + """Return an object to identify dataclass fields. + + default is the default value of the field. default_factory is a + 0-argument function called to initialize a field's value. If init + is true, the field will be a parameter to the class's __init__() + function. If repr is true, the field will be included in the + object's repr(). If hash is true, the field will be included in the + object's hash(). If compare is true, the field will be used in + comparison functions. metadata, if specified, must be a mapping + which is stored but not otherwise examined by dataclass. If kw_only + is true, the field will become a keyword-only parameter to + __init__(). + + It is an error to specify both default and default_factory. + """ + if default is not MISSING and default_factory is not MISSING: + raise ValueError('cannot specify both default and default_factory') + return Field(default, default_factory, init, repr, hash, compare, + metadata, kw_only) -def _collect_fields(cls, kw_only_at_this_class=False): - """Walk the MRO bottom-up and gather every declared field in - declaration order, with subclass fields overriding base ones. - ``kw_only_at_this_class`` flips ``Field.kw_only`` to ``True`` on - fields declared at *this* class only — matching CPython's - semantics where ``@dataclass(kw_only=True)`` applies only to the - locally-declared annotations, not inherited ones. - """ - fields_seen = {} - own_annotations = getattr(cls, "__annotations__", {}) or {} - for base in reversed(cls.__mro__): - annotations = getattr(base, "__annotations__", {}) or {} - for name, annotation in annotations.items(): - if _is_classvar(annotation): - continue - init_only = _is_initvar(annotation) - default = getattr(base, name, MISSING) - if isinstance(default, Field): - f = default - f.name = name - f.type = annotation - else: - f = Field(default=default) - f.name = name - f.type = annotation - if init_only: - f._field_type = "_FIELD_INITVAR" - # `kw_only_at_this_class` only flips fields whose - # annotation lives on ``cls`` directly. Inherited fields - # keep their original `kw_only` flag — exactly what - # CPython's `dataclass` does. - if kw_only_at_this_class and base is cls and name in own_annotations: - if not isinstance(default, Field): - f.kw_only = True - elif not f.kw_only: - f.kw_only = True - fields_seen[name] = f - return list(fields_seen.values()) - - -def _make_init(fields, frozen): - """Build the synthesised ``__init__`` as a closure over the - field list — no source-string compilation, so it works in the - WeavePy runtime which does not implement :func:`exec`. - - Each field's own ``kw_only`` flag controls whether it is - positional or keyword-only; the global ``kw_only`` decorator - argument has already been folded into the per-field flags by - :func:`_collect_fields`. - """ +def _fields_in_init_order(fields): + # Returns the fields as __init__ will output them. It returns 2 tuples: + # the first for normal args, and the second for keyword args. - init_fields = [f for f in fields if f.init] - pos_fields = [f for f in init_fields if not f.kw_only] - kw_fields = [f for f in init_fields if f.kw_only] - non_init_fields = [f for f in fields if not f.init] - kw_only_names = {f.name for f in kw_fields} - - def __init__(self, *args, **kwargs): - if len(args) > len(pos_fields): - raise TypeError( - f"__init__() takes {len(pos_fields) + 1} positional arguments " - f"but {len(args) + 1} were given" + return (tuple(f for f in fields if f.init and not f.kw_only), + tuple(f for f in fields if f.init and f.kw_only) ) - init_field_names = {fld.name for fld in init_fields} - provided = {} - for f, value in zip(pos_fields, args): - provided[f.name] = value - for key, value in kwargs.items(): - # An explicit-parameter `__init__` (CPython) rejects names that - # aren't init fields; mirror that so `C(unknown=…)` (and thus - # `copy.replace(obj, unknown=…)`) raises instead of silently - # dropping the value. - if key not in init_field_names: - raise TypeError( - f"__init__() got an unexpected keyword argument {key!r}" - ) - if key in provided: - raise TypeError( - f"__init__() got multiple values for argument {key!r}" - ) - provided[key] = value - # Fill in defaults for missing fields and validate required. - for f in init_fields: - if f.name in provided: - continue - if f.default is not MISSING: - provided[f.name] = f.default - elif f.default_factory is not MISSING: - provided[f.name] = f.default_factory() - else: - raise TypeError( - f"__init__() missing required argument: {f.name!r}" - ) - # Apply attribute writes; honour `frozen` by bypassing the - # class's __setattr__ via `object.__setattr__`. - for f in init_fields: - if f._field_type == "_FIELD_INITVAR": - continue - value = provided[f.name] - if frozen: - object.__setattr__(self, f.name, value) - else: - setattr(self, f.name, value) - # Non-init fields with defaults/factories. - for f in non_init_fields: - if f.default is not MISSING: - value = f.default - elif f.default_factory is not MISSING: - value = f.default_factory() - else: - continue - if frozen: - object.__setattr__(self, f.name, value) - else: - setattr(self, f.name, value) - # Post-init hook. - post = getattr(self, "__post_init__", None) - if post is not None: - post() - - return __init__ - -def _make_repr(fields, cls_name): - repr_fields = [f for f in fields if f.repr] - def __repr__(self): - parts = [f"{f.name}={getattr(self, f.name)!r}" for f in repr_fields] - return f"{cls_name}({', '.join(parts)})" +def _tuple_str(obj_name, fields): + # Return a string representing each field of obj_name as a tuple + # member. So, if fields is ['x', 'y'] and obj_name is "self", + # return "(self.x,self.y)". + + # Special case for the 0-tuple. + if not fields: + return '()' + # Note the trailing comma, needed if this turns out to be a 1-tuple. + return f'({",".join([f"{obj_name}.{f.name}" for f in fields])},)' + + +class _FuncBuilder: + def __init__(self, globals): + self.names = [] + self.src = [] + self.globals = globals + self.locals = {} + self.overwrite_errors = {} + self.unconditional_adds = {} + + def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, + overwrite_error=False, unconditional_add=False, decorator=None): + if locals is not None: + self.locals.update(locals) + + # Keep track if this method is allowed to be overwritten if it already + # exists in the class. The error is method-specific, so keep it with + # the name. We'll use this when we generate all of the functions in + # the add_fns_to_class call. overwrite_error is either True, in which + # case we'll raise an error, or it's a string, in which case we'll + # raise an error and append this string. + if overwrite_error: + self.overwrite_errors[name] = overwrite_error + + # Should this function always overwrite anything that's already in the + # class? The default is to not overwrite a function that already + # exists. + if unconditional_add: + self.unconditional_adds[name] = True + + self.names.append(name) + + if return_type is not MISSING: + self.locals[f'__dataclass_{name}_return_type__'] = return_type + return_annotation = f'->__dataclass_{name}_return_type__' + else: + return_annotation = '' + args = ','.join(args) + body = '\n'.join(body) - return __repr__ + # Compute the text of the entire function, add it to the text we're generating. + self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}){return_annotation}:\n{body}') + def add_fns_to_class(self, cls): + # The source to all of the functions we're generating. + fns_src = '\n'.join(self.src) -def _make_eq(fields): - cmp_fields = [f for f in fields if f.compare] + # The locals they use. + local_vars = ','.join(self.locals.keys()) - def __eq__(self, other): - if type(self) is not type(other): - return NotImplemented - for f in cmp_fields: - if getattr(self, f.name) != getattr(other, f.name): - return False + # The names of all of the functions, used for the return value of the + # outer function. Need to handle the 0-tuple specially. + if len(self.names) == 0: + return_names = '()' + else: + return_names =f'({",".join(self.names)},)' + + # txt is the entire function we're going to execute, including the + # bodies of the functions we're defining. Here's a greatly simplified + # version: + # def __create_fn__(): + # def __init__(self, x, y): + # self.x = x + # self.y = y + # @recursive_repr + # def __repr__(self): + # return f"cls(x={self.x!r},y={self.y!r})" + # return __init__,__repr__ + + txt = f"def __create_fn__({local_vars}):\n{fns_src}\n return {return_names}" + ns = {} + exec(txt, self.globals, ns) + fns = ns['__create_fn__'](**self.locals) + + # Now that we've generated the functions, assign them into cls. + for name, fn in zip(self.names, fns): + fn.__qualname__ = f"{cls.__qualname__}.{fn.__name__}" + if self.unconditional_adds.get(name, False): + setattr(cls, name, fn) + else: + already_exists = _set_new_attribute(cls, name, fn) + + # See if it's an error to overwrite this particular function. + if already_exists and (msg_extra := self.overwrite_errors.get(name)): + error_msg = (f'Cannot overwrite attribute {fn.__name__} ' + f'in class {cls.__name__}') + if not msg_extra is True: + error_msg = f'{error_msg} {msg_extra}' + + raise TypeError(error_msg) + + +def _field_assign(frozen, name, value, self_name): + # If we're a frozen class, then assign to our fields in __init__ + # via object.__setattr__. Otherwise, just use a simple + # assignment. + # + # self_name is what "self" is called in this function: don't + # hard-code "self", since that might be a field name. + if frozen: + return f' __dataclass_builtins_object__.__setattr__({self_name},{name!r},{value})' + return f' {self_name}.{name}={value}' + + +def _field_init(f, frozen, globals, self_name, slots): + # Return the text of the line in the body of __init__ that will + # initialize this field. + + default_name = f'__dataclass_dflt_{f.name}__' + if f.default_factory is not MISSING: + if f.init: + # This field has a default factory. If a parameter is + # given, use it. If not, call the factory. + globals[default_name] = f.default_factory + value = (f'{default_name}() ' + f'if {f.name} is __dataclass_HAS_DEFAULT_FACTORY__ ' + f'else {f.name}') + else: + # This is a field that's not in the __init__ params, but + # has a default factory function. It needs to be + # initialized here by calling the factory function, + # because there's no other way to initialize it. + + # For a field initialized with a default=defaultvalue, the + # class dict just has the default value + # (cls.fieldname=defaultvalue). But that won't work for a + # default factory, the factory must be called in __init__ + # and we must assign that to self.fieldname. We can't + # fall back to the class dict's value, both because it's + # not set, and because it might be different per-class + # (which, after all, is why we have a factory function!). + + globals[default_name] = f.default_factory + value = f'{default_name}()' + else: + # No default factory. + if f.init: + if f.default is MISSING: + # There's no default, just do an assignment. + value = f.name + elif f.default is not MISSING: + globals[default_name] = f.default + value = f.name + else: + # If the class has slots, then initialize this field. + if slots and f.default is not MISSING: + globals[default_name] = f.default + value = default_name + else: + # This field does not need initialization: reading from it will + # just use the class attribute that contains the default. + # Signify that to the caller by returning None. + return None + + # Only test this now, so that we can create variables for the + # default. However, return None to signify that we're not going + # to actually do the assignment statement for InitVars. + if f._field_type is _FIELD_INITVAR: + return None + + # Now, actually generate the field assignment. + return _field_assign(frozen, f.name, value, self_name) + + +def _init_param(f): + # Return the __init__ parameter string for this field. For + # example, the equivalent of 'x:int=3' (except instead of 'int', + # reference a variable set to int, and instead of '3', reference a + # variable set to 3). + if f.default is MISSING and f.default_factory is MISSING: + # There's no default, and no default_factory, just output the + # variable name and type. + default = '' + elif f.default is not MISSING: + # There's a default, this will be the name that's used to look + # it up. + default = f'=__dataclass_dflt_{f.name}__' + elif f.default_factory is not MISSING: + # There's a factory function. Set a marker. + default = '=__dataclass_HAS_DEFAULT_FACTORY__' + return f'{f.name}:__dataclass_type_{f.name}__{default}' + + +def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, + self_name, func_builder, slots): + # fields contains both real fields and InitVar pseudo-fields. + + # Make sure we don't have fields without defaults following fields + # with defaults. This actually would be caught when exec-ing the + # function source code, but catching it here gives a better error + # message, and future-proofs us in case we build up the function + # using ast. + + seen_default = None + for f in std_fields: + # Only consider the non-kw-only fields in the __init__ call. + if f.init: + if not (f.default is MISSING and f.default_factory is MISSING): + seen_default = f + elif seen_default: + raise TypeError(f'non-default argument {f.name!r} ' + f'follows default argument {seen_default.name!r}') + + locals = {**{f'__dataclass_type_{f.name}__': f.type for f in fields}, + **{'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object, + } + } + + body_lines = [] + for f in fields: + line = _field_init(f, frozen, locals, self_name, slots) + # line is None means that this field doesn't require + # initialization (it's a pseudo-field). Just skip it. + if line: + body_lines.append(line) + + # Does this class have a post-init function? + if has_post_init: + params_str = ','.join(f.name for f in fields + if f._field_type is _FIELD_INITVAR) + body_lines.append(f' {self_name}.{_POST_INIT_NAME}({params_str})') + + # If no body lines, use 'pass'. + if not body_lines: + body_lines = [' pass'] + + _init_params = [_init_param(f) for f in std_fields] + if kw_only_fields: + # Add the keyword-only args. Because the * can only be added if + # there's at least one keyword-only arg, there needs to be a test here + # (instead of just concatenting the lists together). + _init_params += ['*'] + _init_params += [_init_param(f) for f in kw_only_fields] + func_builder.add_fn('__init__', + [self_name] + _init_params, + body_lines, + locals=locals, + return_type=None) + + +def _frozen_get_del_attr(cls, fields, func_builder): + locals = {'cls': cls, + 'FrozenInstanceError': FrozenInstanceError} + condition = 'type(self) is cls' + if fields: + condition += ' or name in {' + ', '.join(repr(f.name) for f in fields) + '}' + + func_builder.add_fn('__setattr__', + ('self', 'name', 'value'), + (f' if {condition}:', + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f' super(cls, self).__setattr__(name, value)'), + locals=locals, + overwrite_error=True) + func_builder.add_fn('__delattr__', + ('self', 'name'), + (f' if {condition}:', + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f' super(cls, self).__delattr__(name)'), + locals=locals, + overwrite_error=True) + + +def _is_classvar(a_type, typing): + # This test uses a typing internal class, but it's the best way to + # test if this is a ClassVar. + return (a_type is typing.ClassVar + or (type(a_type) is typing._GenericAlias + and a_type.__origin__ is typing.ClassVar)) + + +def _is_initvar(a_type, dataclasses): + # The module we're checking against is the module we're + # currently in (dataclasses.py). + return (a_type is dataclasses.InitVar + or type(a_type) is dataclasses.InitVar) + +def _is_kw_only(a_type, dataclasses): + return a_type is dataclasses.KW_ONLY + + +def _is_type(annotation, cls, a_module, a_type, is_type_predicate): + # Given a type annotation string, does it refer to a_type in + # a_module? For example, when checking that annotation denotes a + # ClassVar, then a_module is typing, and a_type is + # typing.ClassVar. + + # It's possible to look up a_module given a_type, but it involves + # looking in sys.modules (again!), and seems like a waste since + # the caller already knows a_module. + + # - annotation is a string type annotation + # - cls is the class that this annotation was found in + # - a_module is the module we want to match + # - a_type is the type in that module we want to match + # - is_type_predicate is a function called with (obj, a_module) + # that determines if obj is of the desired type. + + # Since this test does not do a local namespace lookup (and + # instead only a module (global) lookup), there are some things it + # gets wrong. + + # With string annotations, cv0 will be detected as a ClassVar: + # CV = ClassVar + # @dataclass + # class C0: + # cv0: CV + + # But in this example cv1 will not be detected as a ClassVar: + # @dataclass + # class C1: + # CV = ClassVar + # cv1: CV + + # In C1, the code in this function (_is_type) will look up "CV" in + # the module and not find it, so it will not consider cv1 as a + # ClassVar. This is a fairly obscure corner case, and the best + # way to fix it would be to eval() the string "CV" with the + # correct global and local namespaces. However that would involve + # a eval() penalty for every single field of every dataclass + # that's defined. It was judged not worth it. + + match = _MODULE_IDENTIFIER_RE.match(annotation) + if match: + ns = None + module_name = match.group(1) + if not module_name: + # No module name, assume the class's module did + # "from dataclasses import InitVar". + ns = sys.modules.get(cls.__module__).__dict__ + else: + # Look up module_name in the class's module. + module = sys.modules.get(cls.__module__) + if module and module.__dict__.get(module_name) is a_module: + ns = sys.modules.get(a_type.__module__).__dict__ + if ns and is_type_predicate(ns.get(match.group(2)), a_module): + return True + return False + + +def _get_field(cls, a_name, a_type, default_kw_only): + # Return a Field object for this field name and type. ClassVars and + # InitVars are also returned, but marked as such (see f._field_type). + # default_kw_only is the value of kw_only to use if there isn't a field() + # that defines it. + + # If the default value isn't derived from Field, then it's only a + # normal default value. Convert it to a Field(). + default = getattr(cls, a_name, MISSING) + if isinstance(default, Field): + f = default + else: + if isinstance(default, types.MemberDescriptorType): + # This is a field in __slots__, so it has no default value. + default = MISSING + f = field(default=default) + + # Only at this point do we know the name and the type. Set them. + f.name = a_name + f.type = a_type + + # Assume it's a normal field until proven otherwise. We're next + # going to decide if it's a ClassVar or InitVar, everything else + # is just a normal field. + f._field_type = _FIELD + + # In addition to checking for actual types here, also check for + # string annotations. get_type_hints() won't always work for us + # (see https://github.com/python/typing/issues/508 for example), + # plus it's expensive and would require an eval for every string + # annotation. So, make a best effort to see if this is a ClassVar + # or InitVar using regex's and checking that the thing referenced + # is actually of the correct type. + + # For the complete discussion, see https://bugs.python.org/issue33453 + + # If typing has not been imported, then it's impossible for any + # annotation to be a ClassVar. So, only look for ClassVar if + # typing has been imported by any module (not necessarily cls's + # module). + typing = sys.modules.get('typing') + if typing: + if (_is_classvar(a_type, typing) + or (isinstance(f.type, str) + and _is_type(f.type, cls, typing, typing.ClassVar, + _is_classvar))): + f._field_type = _FIELD_CLASSVAR + + # If the type is InitVar, or if it's a matching string annotation, + # then it's an InitVar. + if f._field_type is _FIELD: + # The module we're checking against is the module we're + # currently in (dataclasses.py). + dataclasses = sys.modules[__name__] + if (_is_initvar(a_type, dataclasses) + or (isinstance(f.type, str) + and _is_type(f.type, cls, dataclasses, dataclasses.InitVar, + _is_initvar))): + f._field_type = _FIELD_INITVAR + + # Validations for individual fields. This is delayed until now, + # instead of in the Field() constructor, since only here do we + # know the field name, which allows for better error reporting. + + # Special restrictions for ClassVar and InitVar. + if f._field_type in (_FIELD_CLASSVAR, _FIELD_INITVAR): + if f.default_factory is not MISSING: + raise TypeError(f'field {f.name} cannot have a ' + 'default factory') + # Should I check for other field settings? default_factory + # seems the most serious to check for. Maybe add others. For + # example, how about init=False (or really, + # init=)? It makes no sense for + # ClassVar and InitVar to specify init=. + + # kw_only validation and assignment. + if f._field_type in (_FIELD, _FIELD_INITVAR): + # For real and InitVar fields, if kw_only wasn't specified use the + # default value. + if f.kw_only is MISSING: + f.kw_only = default_kw_only + else: + # Make sure kw_only isn't set for ClassVars + assert f._field_type is _FIELD_CLASSVAR + if f.kw_only is not MISSING: + raise TypeError(f'field {f.name} is a ClassVar but specifies ' + 'kw_only') + + # For real fields, disallow mutable defaults. Use unhashable as a proxy + # indicator for mutability. Read the __hash__ attribute from the class, + # not the instance. + if f._field_type is _FIELD and f.default.__class__.__hash__ is None: + raise ValueError(f'mutable default {type(f.default)} for field ' + f'{f.name} is not allowed: use default_factory') + + return f + +def _set_new_attribute(cls, name, value): + # Never overwrites an existing attribute. Returns True if the + # attribute already exists. + if name in cls.__dict__: return True + setattr(cls, name, value) + return False + + +# Decide if/how we're going to create a hash function. Key is +# (unsafe_hash, eq, frozen, does-hash-exist). Value is the action to +# take. The common case is to do nothing, so instead of providing a +# function that is a no-op, use None to signify that. + +def _hash_set_none(cls, fields, func_builder): + # It's sort of a hack that I'm setting this here, instead of at + # func_builder.add_fns_to_class time, but since this is an exceptional case + # (it's not setting an attribute to a function, but to a scalar value), + # just do it directly here. I might come to regret this. + cls.__hash__ = None + +def _hash_add(cls, fields, func_builder): + flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] + self_tuple = _tuple_str('self', flds) + func_builder.add_fn('__hash__', + ('self',), + [f' return hash({self_tuple})'], + unconditional_add=True) + +def _hash_exception(cls, fields, func_builder): + # Raise an exception. + raise TypeError(f'Cannot overwrite attribute __hash__ ' + f'in class {cls.__name__}') + +# +# +-------------------------------------- unsafe_hash? +# | +------------------------------- eq? +# | | +------------------------ frozen? +# | | | +---------------- has-explicit-hash? +# | | | | +# | | | | +------- action +# | | | | | +# v v v v v +_hash_action = {(False, False, False, False): None, + (False, False, False, True ): None, + (False, False, True, False): None, + (False, False, True, True ): None, + (False, True, False, False): _hash_set_none, + (False, True, False, True ): None, + (False, True, True, False): _hash_add, + (False, True, True, True ): None, + (True, False, False, False): _hash_add, + (True, False, False, True ): _hash_exception, + (True, False, True, False): _hash_add, + (True, False, True, True ): _hash_exception, + (True, True, False, False): _hash_add, + (True, True, False, True ): _hash_exception, + (True, True, True, False): _hash_add, + (True, True, True, True ): _hash_exception, + } +# See https://bugs.python.org/issue32929#msg312829 for an if-statement +# version of this table. + + +def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, + match_args, kw_only, slots, weakref_slot): + # Now that dicts retain insertion order, there's no reason to use + # an ordered dict. I am leveraging that ordering here, because + # derived class fields overwrite base class fields, but the order + # is defined by the base class, which is found first. + fields = {} + + if cls.__module__ in sys.modules: + globals = sys.modules[cls.__module__].__dict__ + else: + # Theoretically this can happen if someone writes + # a custom string to cls.__module__. In which case + # such dataclass won't be fully introspectable + # (w.r.t. typing.get_type_hints) but will still function + # correctly. + globals = {} + + setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, + unsafe_hash, frozen, + match_args, kw_only, + slots, weakref_slot)) + + # Find our base classes in reverse MRO order, and exclude + # ourselves. In reversed order so that more derived classes + # override earlier field definitions in base classes. As long as + # we're iterating over them, see if all or any of them are frozen. + any_frozen_base = False + # By default `all_frozen_bases` is `None` to represent a case, + # where some dataclasses does not have any bases with `_FIELDS` + all_frozen_bases = None + has_dataclass_bases = False + for b in cls.__mro__[-1:0:-1]: + # Only process classes that have been processed by our + # decorator. That is, they have a _FIELDS attribute. + base_fields = getattr(b, _FIELDS, None) + if base_fields is not None: + has_dataclass_bases = True + for f in base_fields.values(): + fields[f.name] = f + if all_frozen_bases is None: + all_frozen_bases = True + current_frozen = getattr(b, _PARAMS).frozen + all_frozen_bases = all_frozen_bases and current_frozen + any_frozen_base = any_frozen_base or current_frozen + + # Annotations defined specifically in this class (not in base classes). + # + # Fields are found from cls_annotations, which is guaranteed to be + # ordered. Default values are from class attributes, if a field + # has a default. If the default value is a Field(), then it + # contains additional info beyond (and possibly including) the + # actual default value. Pseudo-fields ClassVars and InitVars are + # included, despite the fact that they're not real fields. That's + # dealt with later. + cls_annotations = inspect.get_annotations(cls) + + # Now find fields in our class. While doing so, validate some + # things, and set the default values (as class attributes) where + # we can. + cls_fields = [] + # Get a reference to this module for the _is_kw_only() test. + KW_ONLY_seen = False + dataclasses = sys.modules[__name__] + for name, type in cls_annotations.items(): + # See if this is a marker to change the value of kw_only. + if (_is_kw_only(type, dataclasses) + or (isinstance(type, str) + and _is_type(type, cls, dataclasses, dataclasses.KW_ONLY, + _is_kw_only))): + # Switch the default to kw_only=True, and ignore this + # annotation: it's not a real field. + if KW_ONLY_seen: + raise TypeError(f'{name!r} is KW_ONLY, but KW_ONLY ' + 'has already been specified') + KW_ONLY_seen = True + kw_only = True + else: + # Otherwise it's a field of some type. + cls_fields.append(_get_field(cls, name, type, kw_only)) + + for f in cls_fields: + fields[f.name] = f + + # If the class attribute (which is the default value for this + # field) exists and is of type 'Field', replace it with the + # real default. This is so that normal class introspection + # sees a real default value, not a Field. + if isinstance(getattr(cls, f.name, None), Field): + if f.default is MISSING: + # If there's no default, delete the class attribute. + # This happens if we specify field(repr=False), for + # example (that is, we specified a field object, but + # no default value). Also if we're using a default + # factory. The class attribute should not be set at + # all in the post-processed class. + delattr(cls, f.name) + else: + setattr(cls, f.name, f.default) + + # Do we have any Field members that don't also have annotations? + for name, value in cls.__dict__.items(): + if isinstance(value, Field) and not name in cls_annotations: + raise TypeError(f'{name!r} is a field but has no type annotation') + + # Check rules that apply if we are derived from any dataclasses. + if has_dataclass_bases: + # Raise an exception if any of our bases are frozen, but we're not. + if any_frozen_base and not frozen: + raise TypeError('cannot inherit non-frozen dataclass from a ' + 'frozen one') + + # Raise an exception if we're frozen, but none of our bases are. + if all_frozen_bases is False and frozen: + raise TypeError('cannot inherit frozen dataclass from a ' + 'non-frozen one') + + # Remember all of the fields on our class (including bases). This + # also marks this class as being a dataclass. + setattr(cls, _FIELDS, fields) + + # Was this class defined with an explicit __hash__? Note that if + # __eq__ is defined in this class, then python will automatically + # set __hash__ to None. This is a heuristic, as it's possible + # that such a __hash__ == None was not auto-generated, but it's + # close enough. + class_hash = cls.__dict__.get('__hash__', MISSING) + has_explicit_hash = not (class_hash is MISSING or + (class_hash is None and '__eq__' in cls.__dict__)) + + # If we're generating ordering methods, we must be generating the + # eq methods. + if order and not eq: + raise ValueError('eq must be true if order is true') + + # Include InitVars and regular fields (so, not ClassVars). This is + # initialized here, outside of the "if init:" test, because std_init_fields + # is used with match_args, below. + all_init_fields = [f for f in fields.values() + if f._field_type in (_FIELD, _FIELD_INITVAR)] + (std_init_fields, + kw_only_init_fields) = _fields_in_init_order(all_init_fields) + + func_builder = _FuncBuilder(globals) + + if init: + # Does this class have a post-init function? + has_post_init = hasattr(cls, _POST_INIT_NAME) + + _init_fn(all_init_fields, + std_init_fields, + kw_only_init_fields, + frozen, + has_post_init, + # The name to use for the "self" + # param in __init__. Use "self" + # if possible. + '__dataclass_self__' if 'self' in fields + else 'self', + func_builder, + slots, + ) + + _set_new_attribute(cls, '__replace__', _replace) + + # Get the fields as a list, and include only real fields. This is + # used in all of the following methods. + field_list = [f for f in fields.values() if f._field_type is _FIELD] + + if repr: + flds = [f for f in field_list if f.repr] + func_builder.add_fn('__repr__', + ('self',), + [' return f"{self.__class__.__qualname__}(' + + ', '.join([f"{f.name}={{self.{f.name}!r}}" + for f in flds]) + ')"'], + locals={'__dataclasses_recursive_repr': recursive_repr}, + decorator="@__dataclasses_recursive_repr()") + + if eq: + # Create __eq__ method. There's no need for a __ne__ method, + # since python will call __eq__ and negate it. + cmp_fields = (field for field in field_list if field.compare) + terms = [f'self.{field.name}==other.{field.name}' for field in cmp_fields] + field_comparisons = ' and '.join(terms) or 'True' + func_builder.add_fn('__eq__', + ('self', 'other'), + [ ' if self is other:', + ' return True', + ' if other.__class__ is self.__class__:', + f' return {field_comparisons}', + ' return NotImplemented']) - return __eq__ + if order: + # Create and set the ordering methods. + flds = [f for f in field_list if f.compare] + self_tuple = _tuple_str('self', flds) + other_tuple = _tuple_str('other', flds) + for name, op in [('__lt__', '<'), + ('__le__', '<='), + ('__gt__', '>'), + ('__ge__', '>='), + ]: + # Create a comparison function. If the fields in the object are + # named 'x' and 'y', then self_tuple is the string + # '(self.x,self.y)' and other_tuple is the string + # '(other.x,other.y)'. + func_builder.add_fn(name, + ('self', 'other'), + [ ' if other.__class__ is self.__class__:', + f' return {self_tuple}{op}{other_tuple}', + ' return NotImplemented'], + overwrite_error='Consider using functools.total_ordering') + + if frozen: + _frozen_get_del_attr(cls, field_list, func_builder) + + # Decide if/how we're going to create a hash function. + hash_action = _hash_action[bool(unsafe_hash), + bool(eq), + bool(frozen), + has_explicit_hash] + if hash_action: + cls.__hash__ = hash_action(cls, field_list, func_builder) + + # Generate the methods and add them to the class. This needs to be done + # before the __doc__ logic below, since inspect will look at the __init__ + # signature. + func_builder.add_fns_to_class(cls) + + if not getattr(cls, '__doc__'): + # Create a class doc-string. + try: + # In some cases fetching a signature is not possible. + # But, we surely should not fail in this case. + text_sig = str(inspect.signature(cls)).replace(' -> None', '') + except (TypeError, ValueError): + text_sig = '' + cls.__doc__ = (cls.__name__ + text_sig) + + if match_args: + # I could probably compute this once. + _set_new_attribute(cls, '__match_args__', + tuple(f.name for f in std_init_fields)) + + # It's an error to specify weakref_slot if slots is False. + if weakref_slot and not slots: + raise TypeError('weakref_slot is True but slots is False') + if slots: + cls = _add_slots(cls, frozen, weakref_slot) + abc.update_abstractmethods(cls) -def _make_order(fields, op_name, op): - cmp_fields = [f for f in fields if f.compare] + return cls - def __cmp__(self, other): - if type(self) is not type(other): - return NotImplemented - self_tuple = tuple(getattr(self, f.name) for f in cmp_fields) - other_tuple = tuple(getattr(other, f.name) for f in cmp_fields) - return op(self_tuple, other_tuple) - __cmp__.__name__ = op_name - return __cmp__ +# _dataclass_getstate and _dataclass_setstate are needed for pickling frozen +# classes with slots. These could be slightly more performant if we generated +# the code instead of iterating over fields. But that can be a project for +# another day, if performance becomes an issue. +def _dataclass_getstate(self): + return [getattr(self, f.name) for f in fields(self)] + + +def _dataclass_setstate(self, state): + for field, value in zip(fields(self), state): + # use setattr because dataclass may be frozen + object.__setattr__(self, field.name, value) + + +def _get_slots(cls): + match cls.__dict__.get('__slots__'): + # `__dictoffset__` and `__weakrefoffset__` can tell us whether + # the base type has dict/weakref slots, in a way that works correctly + # for both Python classes and C extension types. Extension types + # don't use `__slots__` for slot creation + case None: + slots = [] + if getattr(cls, '__weakrefoffset__', -1) != 0: + slots.append('__weakref__') + if getattr(cls, '__dictoffset__', -1) != 0: + slots.append('__dict__') + yield from slots + case str(slot): + yield slot + # Slots may be any iterable, but we cannot handle an iterator + # because it will already be (partially) consumed. + case iterable if not hasattr(iterable, '__next__'): + yield from iterable + case _: + raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") + + +def _add_slots(cls, is_frozen, weakref_slot): + # Need to create a new class, since we can't set __slots__ + # after a class has been created. + + # Make sure __slots__ isn't already set. + if '__slots__' in cls.__dict__: + raise TypeError(f'{cls.__name__} already specifies __slots__') + + # Create a new dict for our new class. + cls_dict = dict(cls.__dict__) + field_names = tuple(f.name for f in fields(cls)) + # Make sure slots don't overlap with those in base classes. + inherited_slots = set( + itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) + ) + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + cls_dict["__slots__"] = tuple( + itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ), + ) + for field_name in field_names: + # Remove our attributes, if present. They'll still be + # available in _MARKER. + cls_dict.pop(field_name, None) -def _make_hash(fields): - cmp_fields = [f for f in fields if f.compare] + # Remove __dict__ itself. + cls_dict.pop('__dict__', None) - def __hash__(self): - return hash(tuple(getattr(self, f.name) for f in cmp_fields)) + # Clear existing `__weakref__` descriptor, it belongs to a previous type: + cls_dict.pop('__weakref__', None) # gh-102069 - return __hash__ + # And finally create the class. + qualname = getattr(cls, '__qualname__', None) + cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + if qualname is not None: + cls.__qualname__ = qualname + if is_frozen: + # Need this for pickling frozen classes with slots. + if '__getstate__' not in cls_dict: + cls.__getstate__ = _dataclass_getstate + if '__setstate__' not in cls_dict: + cls.__setstate__ = _dataclass_setstate -def _process_class(cls, init, repr, eq, order, frozen, slots, kw_only): - fields = _collect_fields(cls, kw_only_at_this_class=kw_only) - # When `kw_only=True` is in effect, kw-only fields must follow - # positional fields in the synthesised __init__ signature even - # if the user declared them in a different order. We re-stable- - # sort here so positional fields appear first and kw-only fields - # last; declaration order is preserved within each group. - fields = sorted(fields, key=lambda f: 1 if f.kw_only else 0) - setattr(cls, "__dataclass_fields__", {f.name: f for f in fields}) - setattr(cls, "__dataclass_params__", _DataclassParams(init, repr, eq, order, frozen)) + return cls - if init and "__init__" not in cls.__dict__: - cls.__init__ = _make_init(fields, frozen=frozen) - if repr and "__repr__" not in cls.__dict__: - cls.__repr__ = _make_repr(fields, cls.__name__) +def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, + unsafe_hash=False, frozen=False, match_args=True, + kw_only=False, slots=False, weakref_slot=False): + """Add dunder methods based on the fields defined in the class. - if eq and "__eq__" not in cls.__dict__: - cls.__eq__ = _make_eq(fields) - if "__hash__" not in cls.__dict__: - if frozen: - cls.__hash__ = _make_hash(fields) - else: - cls.__hash__ = None + Examines PEP 526 __annotations__ to determine fields. - if order: - ops = [ - ("__lt__", lambda a, b: a < b), - ("__le__", lambda a, b: a <= b), - ("__gt__", lambda a, b: a > b), - ("__ge__", lambda a, b: a >= b), - ] - for op_name, op in ops: - if op_name not in cls.__dict__: - setattr(cls, op_name, _make_order(fields, op_name, op)) - - if frozen and "__setattr__" not in cls.__dict__: - def _frozen_setattr(self, key, value): - raise FrozenInstanceError(f"cannot assign to field {key!r}") - - def _frozen_delattr(self, key): - raise FrozenInstanceError(f"cannot delete field {key!r}") - - cls.__setattr__ = _frozen_setattr - cls.__delattr__ = _frozen_delattr - - # `copy.replace(obj)` (Python 3.13+) dispatches through `__replace__`. - if "__replace__" not in cls.__dict__: - cls.__replace__ = _replace + If init is true, an __init__() method is added to the class. If repr + is true, a __repr__() method is added. If order is true, rich + comparison dunder methods are added. If unsafe_hash is true, a + __hash__() method is added. If frozen is true, fields may not be + assigned to after instance creation. If match_args is true, the + __match_args__ tuple is added. If kw_only is true, then by default + all fields are keyword-only. If slots is true, a new class with a + __slots__ attribute is returned. + """ - if slots: - # CPython rebuilds the class so ``__slots__`` is in effect at - # construction time; assigning ``cls.__slots__ = ...`` after - # the fact does not give the type slot storage. We mirror the - # CPython logic here: collect inherited slot names, exclude - # them from the new tuple, and re-create the class via the - # original metaclass. - cls = _add_slots(cls, fields, frozen) + def wrap(cls): + return _process_class(cls, init, repr, eq, order, unsafe_hash, + frozen, match_args, kw_only, slots, + weakref_slot) - return cls + # See if we're being called as @dataclass or @dataclass(). + if cls is None: + # We're called with parens. + return wrap + # We're called as @dataclass without parens. + return wrap(cls) -def _add_slots(cls, dc_fields, is_frozen): - field_names = tuple(f.name for f in dc_fields) - inherited_slots = set() - for c in cls.__mro__[1:-1]: - inherited_slots.update(getattr(c, "__slots__", ()) or ()) - - # Materialise the existing class dict into a fresh dict via the - # public attribute API. Going through `dir(cls)` + `getattr` is - # safer than `dict(cls.__dict__)` because the latter risks - # holding overlapping borrows of the underlying dict storage on - # runtimes that share the dict by reference between class and - # mappingproxy. - cls_dict = {} - for key in list(cls.__dict__.keys()): - if key == "__dict__" or key == "__weakref__": - continue - if key in field_names: - continue - cls_dict[key] = cls.__dict__[key] - new_slots = tuple(n for n in field_names if n not in inherited_slots) - cls_dict["__slots__"] = new_slots - qualname = getattr(cls, "__qualname__", None) - new_cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) - if qualname is not None: - try: - new_cls.__qualname__ = qualname - except (AttributeError, TypeError): - pass - return new_cls +def fields(class_or_instance): + """Return a tuple describing the fields of this dataclass. -class _DataclassParams: - __slots__ = ("init", "repr", "eq", "order", "frozen") + Accepts a dataclass or an instance of one. Tuple elements are of + type Field. + """ - def __init__(self, init, repr, eq, order, frozen): - self.init = init - self.repr = repr - self.eq = eq - self.order = order - self.frozen = frozen + # Might it be worth caching this, per class? + try: + fields = getattr(class_or_instance, _FIELDS) + except AttributeError: + raise TypeError('must be called with a dataclass type or instance') from None + # Exclude pseudo-fields. Note that fields is sorted by insertion + # order, so the order of the tuple is as the fields were defined. + return tuple(f for f in fields.values() if f._field_type is _FIELD) -class FrozenInstanceError(AttributeError): - pass +def _is_dataclass_instance(obj): + """Returns True if obj is an instance of a dataclass.""" + return hasattr(type(obj), _FIELDS) -def dataclass( - cls=None, - /, - *, - init=True, - repr=True, - eq=True, - order=False, - unsafe_hash=False, - frozen=False, - match_args=True, - kw_only=False, - slots=False, - weakref_slot=False, -): - """The ``@dataclass`` class decorator. Accepts the same keyword - arguments as CPython's dataclass; ``match_args`` and - ``weakref_slot`` are accepted but ignored (no behaviour - difference in the current runtime).""" - _ = unsafe_hash, match_args, weakref_slot - - def wrap(c): - return _process_class(c, init, repr, eq, order, frozen, slots, kw_only) - if cls is None: - return wrap - return wrap(cls) +def is_dataclass(obj): + """Returns True if obj is a dataclass or an instance of a + dataclass.""" + cls = obj if isinstance(obj, type) else type(obj) + return hasattr(cls, _FIELDS) -def fields(class_or_instance): - """Return a tuple of the dataclass fields for the given class or - instance, in declaration order.""" - try: - flds = class_or_instance.__dataclass_fields__ - except AttributeError: - raise TypeError("fields() argument must be a dataclass or instance") - return tuple(flds.values()) +def asdict(obj, *, dict_factory=dict): + """Return the fields of a dataclass instance as a new dictionary mapping + field names to field values. + Example usage:: -def is_dataclass(obj): - """``True`` if *obj* is a dataclass *or* a dataclass instance.""" - return hasattr(obj, "__dataclass_fields__") + @dataclass + class C: + x: int + y: int + c = C(1, 2) + assert asdict(c) == {'x': 1, 'y': 2} -def asdict(obj, *, dict_factory=dict): - """Recursively convert a dataclass instance to a dict, mirroring - each dataclass field's value.""" - if not is_dataclass(obj) or isinstance(obj, type): - raise TypeError("asdict() expects a dataclass instance") + If given, 'dict_factory' will be used instead of built-in dict. + The function applies recursively to field values that are + dataclass instances. This will also look into built-in containers: + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. + """ + if not _is_dataclass_instance(obj): + raise TypeError("asdict() should be called on dataclass instances") return _asdict_inner(obj, dict_factory) def _asdict_inner(obj, dict_factory): - if is_dataclass(obj) and not isinstance(obj, type): - result = [] - for f in fields(obj): - value = _asdict_inner(getattr(obj, f.name), dict_factory) - result.append((f.name, value)) - return dict_factory(result) - if isinstance(obj, (list, tuple)): - kind = type(obj) - return kind(_asdict_inner(v, dict_factory) for v in obj) - if isinstance(obj, dict): - return type(obj)( - (_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) + obj_type = type(obj) + if obj_type in _ATOMIC_TYPES: + return obj + elif hasattr(obj_type, _FIELDS): + # dataclass instance: fast path for the common case + if dict_factory is dict: + return { + f.name: _asdict_inner(getattr(obj, f.name), dict) + for f in fields(obj) + } + else: + return dict_factory([ + (f.name, _asdict_inner(getattr(obj, f.name), dict_factory)) + for f in fields(obj) + ]) + # handle the builtin types first for speed; subclasses handled below + elif obj_type is list: + return [_asdict_inner(v, dict_factory) for v in obj] + elif obj_type is dict: + return { + _asdict_inner(k, dict_factory): _asdict_inner(v, dict_factory) for k, v in obj.items() - ) - return obj + } + elif obj_type is tuple: + return tuple([_asdict_inner(v, dict_factory) for v in obj]) + elif issubclass(obj_type, tuple): + if hasattr(obj, '_fields'): + # obj is a namedtuple. Recurse into it, but the returned + # object is another namedtuple of the same type. This is + # similar to how other list- or tuple-derived classes are + # treated (see below), but we just need to create them + # differently because a namedtuple's __init__ needs to be + # called differently (see bpo-34363). + + # I'm not using namedtuple's _asdict() + # method, because: + # - it does not recurse in to the namedtuple fields and + # convert them to dicts (using dict_factory). + # - I don't actually want to return a dict here. The main + # use case here is json.dumps, and it handles converting + # namedtuples to lists. Admittedly we're losing some + # information here when we produce a json list instead of a + # dict. Note that if we returned dicts here instead of + # namedtuples, we could no longer call asdict() on a data + # structure where a namedtuple was used as a dict key. + return obj_type(*[_asdict_inner(v, dict_factory) for v in obj]) + else: + return obj_type(_asdict_inner(v, dict_factory) for v in obj) + elif issubclass(obj_type, dict): + if hasattr(obj_type, 'default_factory'): + # obj is a defaultdict, which has a different constructor from + # dict as it requires the default_factory as its first arg. + result = obj_type(obj.default_factory) + for k, v in obj.items(): + result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) + return result + return obj_type((_asdict_inner(k, dict_factory), + _asdict_inner(v, dict_factory)) + for k, v in obj.items()) + elif issubclass(obj_type, list): + # Assume we can create an object of this type by passing in a + # generator + return obj_type(_asdict_inner(v, dict_factory) for v in obj) + else: + return copy.deepcopy(obj) def astuple(obj, *, tuple_factory=tuple): - """Recursively convert a dataclass instance to a tuple.""" - if not is_dataclass(obj) or isinstance(obj, type): - raise TypeError("astuple() expects a dataclass instance") + """Return the fields of a dataclass instance as a new tuple of field values. + + Example usage:: + + @dataclass + class C: + x: int + y: int + + c = C(1, 2) + assert astuple(c) == (1, 2) + + If given, 'tuple_factory' will be used instead of built-in tuple. + The function applies recursively to field values that are + dataclass instances. This will also look into built-in containers: + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. + """ + + if not _is_dataclass_instance(obj): + raise TypeError("astuple() should be called on dataclass instances") return _astuple_inner(obj, tuple_factory) def _astuple_inner(obj, tuple_factory): - if is_dataclass(obj) and not isinstance(obj, type): - return tuple_factory( - _astuple_inner(getattr(obj, f.name), tuple_factory) for f in fields(obj) - ) - if isinstance(obj, (list, tuple)): - kind = type(obj) - return kind(_astuple_inner(v, tuple_factory) for v in obj) - if isinstance(obj, dict): - return type(obj)( - (_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) - for k, v in obj.items() - ) - return obj + if type(obj) in _ATOMIC_TYPES: + return obj + elif _is_dataclass_instance(obj): + return tuple_factory([ + _astuple_inner(getattr(obj, f.name), tuple_factory) + for f in fields(obj) + ]) + elif isinstance(obj, tuple) and hasattr(obj, '_fields'): + # obj is a namedtuple. Recurse into it, but the returned + # object is another namedtuple of the same type. This is + # similar to how other list- or tuple-derived classes are + # treated (see below), but we just need to create them + # differently because a namedtuple's __init__ needs to be + # called differently (see bpo-34363). + return type(obj)(*[_astuple_inner(v, tuple_factory) for v in obj]) + elif isinstance(obj, (list, tuple)): + # Assume we can create an object of this type by passing in a + # generator (which is not true for namedtuples, handled + # above). + return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) + elif isinstance(obj, dict): + obj_type = type(obj) + if hasattr(obj_type, 'default_factory'): + # obj is a defaultdict, which has a different constructor from + # dict as it requires the default_factory as its first arg. + result = obj_type(getattr(obj, 'default_factory')) + for k, v in obj.items(): + result[_astuple_inner(k, tuple_factory)] = _astuple_inner(v, tuple_factory) + return result + return obj_type((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) + for k, v in obj.items()) + else: + return copy.deepcopy(obj) + + +def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, + repr=True, eq=True, order=False, unsafe_hash=False, + frozen=False, match_args=True, kw_only=False, slots=False, + weakref_slot=False, module=None): + """Return a new dynamically created dataclass. + + The dataclass name will be 'cls_name'. 'fields' is an iterable + of either (name), (name, type) or (name, type, Field) objects. If type is + omitted, use the string 'typing.Any'. Field objects are created by + the equivalent of calling 'field(name, type [, Field-info])'.:: + + C = make_dataclass('C', ['x', ('y', int), ('z', int, field(init=False))], bases=(Base,)) + + is equivalent to:: + + @dataclass + class C(Base): + x: 'typing.Any' + y: int + z: int = field(init=False) + + For the bases and namespace parameters, see the builtin type() function. + + The parameters init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, + slots, and weakref_slot are passed to dataclass(). + + If module parameter is defined, the '__module__' attribute of the dataclass is + set to that value. + """ + + if namespace is None: + namespace = {} + + # While we're looking through the field names, validate that they + # are identifiers, are not keywords, and not duplicates. + seen = set() + annotations = {} + defaults = {} + for item in fields: + if isinstance(item, str): + name = item + tp = 'typing.Any' + elif len(item) == 2: + name, tp, = item + elif len(item) == 3: + name, tp, spec = item + defaults[name] = spec + else: + raise TypeError(f'Invalid field: {item!r}') + + if not isinstance(name, str) or not name.isidentifier(): + raise TypeError(f'Field names must be valid identifiers: {name!r}') + if keyword.iskeyword(name): + raise TypeError(f'Field names must not be keywords: {name!r}') + if name in seen: + raise TypeError(f'Field name duplicated: {name!r}') + + seen.add(name) + annotations[name] = tp + + # Update 'ns' with the user-supplied namespace plus our calculated values. + def exec_body_callback(ns): + ns.update(namespace) + ns.update(defaults) + ns['__annotations__'] = annotations + + # We use `types.new_class()` instead of simply `type()` to allow dynamic creation + # of generic dataclasses. + cls = types.new_class(cls_name, bases, {}, exec_body_callback) + + # For pickling to work, the __module__ variable needs to be set to the frame + # where the dataclass is created. + if module is None: + try: + module = sys._getframemodulename(1) or '__main__' + except AttributeError: + try: + module = sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + if module is not None: + cls.__module__ = module + + # Apply the normal decorator. + return dataclass(cls, init=init, repr=repr, eq=eq, order=order, + unsafe_hash=unsafe_hash, frozen=frozen, + match_args=match_args, kw_only=kw_only, slots=slots, + weakref_slot=weakref_slot) + + +def replace(obj, /, **changes): + """Return a new object replacing specified fields with new values. + + This is especially useful for frozen classes. Example usage:: + + @dataclass(frozen=True) + class C: + x: int + y: int + + c = C(1, 2) + c1 = replace(c, x=3) + assert c1.x == 3 and c1.y == 2 + """ + if not _is_dataclass_instance(obj): + raise TypeError("replace() should be called on dataclass instances") + return _replace(obj, **changes) def _replace(self, /, **changes): - """`__replace__` implementation bound on each dataclass — delegates - to `replace` so `copy.replace(obj, **changes)` works (Python 3.13+).""" - return replace(self, **changes) + # We're going to mutate 'changes', but that's okay because it's a + # new dict, even if called with 'replace(self, **my_changes)'. + # It's an error to have init=False fields in 'changes'. + # If a field is not in 'changes', read its value from the provided 'self'. + + for f in getattr(self, _FIELDS).values(): + # Only consider normal fields or InitVars. + if f._field_type is _FIELD_CLASSVAR: + continue -def replace(obj, /, **changes): - """Return a new dataclass instance with `changes` applied, all - other fields copied from `obj`.""" - if not is_dataclass(obj) or isinstance(obj, type): - raise TypeError("replace() expects a dataclass instance") - # Fill in field values not being changed, mutating `changes` in place - # (CPython semantics). Any leftover keys that aren't init fields stay in - # `changes` and reach `__init__`, which rejects them with `TypeError` — - # so `replace(obj, not_a_field=…)` raises, as CPython requires. - for f in fields(obj): if not f.init: + # Error if this field is specified in changes. if f.name in changes: - raise ValueError( - f"cannot replace non-init field {f.name!r}" - ) + raise TypeError(f'field {f.name} is declared with ' + f'init=False, it cannot be specified with ' + f'replace()') continue - if f.name not in changes: - changes[f.name] = getattr(obj, f.name) - return type(obj)(**changes) - -def make_dataclass(cls_name, fields_spec, *, bases=(), namespace=None, **kwargs): - """Dynamically create a dataclass. - - Each entry in ``fields_spec`` is either ``name``, ``(name, type)``, - or ``(name, type, field_descriptor)`` — matching CPython. - """ - ns = dict(namespace or {}) - annotations = ns.setdefault("__annotations__", {}) - for entry in fields_spec: - if isinstance(entry, str): - name, type_, default = entry, "typing.Any", MISSING - elif len(entry) == 2: - name, type_ = entry - default = MISSING - else: - name, type_, default = entry - annotations[name] = type_ - if default is not MISSING: - ns[name] = default - new_cls = type(cls_name, bases, ns) - return dataclass(new_cls, **kwargs) - - -__all__ = [ - "dataclass", - "field", - "Field", - "FrozenInstanceError", - "MISSING", - "fields", - "is_dataclass", - "asdict", - "astuple", - "replace", - "make_dataclass", -] + if f.name not in changes: + if f._field_type is _FIELD_INITVAR and f.default is MISSING: + raise TypeError(f"InitVar {f.name!r} " + f'must be specified with replace()') + changes[f.name] = getattr(self, f.name) + + # Create the new object, which calls __init__() and + # __post_init__() (if defined), using all of the init fields we've + # added and/or left in 'changes'. If there are values supplied in + # changes that aren't fields, this will correctly raise a + # TypeError. + return self.__class__(**changes) diff --git a/crates/weavepy-vm/src/stdlib/python/inspect.py b/crates/weavepy-vm/src/stdlib/python/inspect.py index 1c2c1f8..b3c2c6e 100644 --- a/crates/weavepy-vm/src/stdlib/python/inspect.py +++ b/crates/weavepy-vm/src/stdlib/python/inspect.py @@ -10,6 +10,8 @@ import sys import linecache +import types +import functools __all__ = [ @@ -62,6 +64,7 @@ "CO_COROUTINE", "CO_ITERABLE_COROUTINE", "CO_ASYNC_GENERATOR", + "get_annotations", ] @@ -82,6 +85,99 @@ def _safe_type_name(t): return getattr(t, "__name__", repr(t)) +def get_annotations(obj, *, globals=None, locals=None, eval_str=False): + """Compute the annotations dict for an object. + + Verbatim port of CPython 3.13's ``inspect.get_annotations``: ``obj`` + may be a callable, class, or module, and the result is always a + freshly-created dict. ``dataclasses`` relies on this to read a + class's own ``__annotations__`` while ignoring inherited ones. + """ + if isinstance(obj, type): + # class + obj_dict = getattr(obj, '__dict__', None) + if obj_dict and hasattr(obj_dict, 'get'): + ann = obj_dict.get('__annotations__', None) + if isinstance(ann, types.GetSetDescriptorType): + ann = None + else: + ann = None + + obj_globals = None + module_name = getattr(obj, '__module__', None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, '__dict__', None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + ann = getattr(obj, '__annotations__', None) + obj_globals = getattr(obj, '__dict__') + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + ann = getattr(obj, '__annotations__', None) + obj_globals = getattr(obj, '__globals__', None) + obj_locals = None + unwrap = obj + else: + raise TypeError(f"{obj!r} is not a module, class, or callable.") + + if ann is None: + return {} + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if unwrap is not None: + while True: + if hasattr(unwrap, '__wrapped__'): + unwrap = unwrap.__wrapped__ + continue + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals or {} + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + locals = {param.__name__: param for param in type_params} | locals + + # PEP 646 star-unpack rewriting lives in `typing` on CPython 3.13; fall + # back to a no-op when that internal helper isn't available. + try: + from typing import _rewrite_star_unpack as _rewrite + except ImportError: + def _rewrite(value): + return value + + return_value = { + key: value if not isinstance(value, str) + else eval(_rewrite(value), globals, locals) + for key, value in ann.items() } + return return_value + + # ---------------- predicates ---------------- # def _has_attrs(obj, *names): diff --git a/crates/weavepy-vm/src/stdlib/python/keyword.py b/crates/weavepy-vm/src/stdlib/python/keyword.py new file mode 100644 index 0000000..5fd76d6 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/keyword.py @@ -0,0 +1,58 @@ +"""Keywords (from "Grammar/python.gram") + +This file is intentionally a verbatim port of CPython 3.13's +auto-generated ``Lib/keyword.py``: the literal keyword and +soft-keyword lists plus the two membership predicates. Modules such +as ``dataclasses`` import ``keyword.iskeyword`` to validate field +names. +""" + +__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"] + +kwlist = [ + 'False', + 'None', + 'True', + 'and', + 'as', + 'assert', + 'async', + 'await', + 'break', + 'class', + 'continue', + 'def', + 'del', + 'elif', + 'else', + 'except', + 'finally', + 'for', + 'from', + 'global', + 'if', + 'import', + 'in', + 'is', + 'lambda', + 'nonlocal', + 'not', + 'or', + 'pass', + 'raise', + 'return', + 'try', + 'while', + 'with', + 'yield' +] + +softkwlist = [ + '_', + 'case', + 'match', + 'type' +] + +iskeyword = frozenset(kwlist).__contains__ +issoftkeyword = frozenset(softkwlist).__contains__ diff --git a/crates/weavepy-vm/src/stdlib/python/typing.py b/crates/weavepy-vm/src/stdlib/python/typing.py index 02c00dc..7ffc923 100644 --- a/crates/weavepy-vm/src/stdlib/python/typing.py +++ b/crates/weavepy-vm/src/stdlib/python/typing.py @@ -752,4 +752,33 @@ def _namedtuple_mro_entries(bases): "get_args", "NewType", "TYPE_CHECKING", + "Deque", + "DefaultDict", + "OrderedDict", + "Counter", + "ChainMap", ] + + +# Container aliases backed by the ``collections`` module (the legacy +# ``typing.Deque`` / ``typing.DefaultDict`` spellings). Resolved lazily via +# PEP 562 so importing ``typing`` never forces ``collections`` during +# interpreter bootstrap (avoids an import cycle). +_LAZY_COLLECTION_ALIASES = { + "Deque": "deque", + "DefaultDict": "defaultdict", + "OrderedDict": "OrderedDict", + "Counter": "Counter", + "ChainMap": "ChainMap", +} + + +def __getattr__(name): + target = _LAZY_COLLECTION_ALIASES.get(name) + if target is not None: + import collections + + alias = _OriginAlias(name, getattr(collections, target)) + globals()[name] = alias + return alias + raise AttributeError(f"module 'typing' has no attribute {name!r}") From a8d373ac8419d854784baf73050bbc9294714f68 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Mon, 8 Jun 2026 21:21:42 -0700 Subject: [PATCH 5/9] feat: advance CPython Lib/test conformance wave 2 --- crates/weavepy-compiler/src/lib.rs | 44 +++++++++- crates/weavepy-lexer/src/lib.rs | 38 +++++++++ crates/weavepy-lexer/src/scanner.rs | 35 ++++++++ crates/weavepy-vm/src/lib.rs | 93 +++++++++++++++++---- crates/weavepy-vm/src/object.rs | 4 +- crates/weavepy-vm/src/stdlib/marshal_mod.rs | 16 ++-- 6 files changed, 202 insertions(+), 28 deletions(-) diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index dd9b6e1..971148a 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -70,6 +70,12 @@ pub enum CompileError { #[derive(Debug, Clone, Default, PartialEq)] pub struct CodeObject { pub name: String, + /// Dotted qualified name (PEP 3155), computed at compile time from the + /// lexical scope nesting: `outer..inner` for a function nested + /// in `outer`, `C.method` for a method of class `C`. Equals `name` for + /// module-level definitions. Drives `function.__qualname__` / + /// `type.__qualname__` (and thus reprs, error messages, and pickling). + pub qualname: String, /// Source filename or ``. Used for diagnostics only. pub filename: String, pub instructions: Vec, @@ -631,6 +637,9 @@ impl Compiler { future_annotations: bool, ) -> Self { let mut co = CodeObject::default(); + // Default qualname == name; nested scopes overwrite this via + // `compute_child_qualname` once the parent context is known. + co.qualname = name.clone(); co.name = name; co.filename = filename; co.is_class_body = matches!(kind, CodeKind::Class); @@ -657,6 +666,29 @@ impl Compiler { } } + /// Compute the PEP 3155 `__qualname__` for a function/class named + /// `name` defined directly inside *this* (the parent) scope. Mirrors + /// CPython's `compiler_set_qualname` (`Python/compile.c`): + /// + /// - A definition whose parent is the module gets the bare `name`. + /// - Otherwise the parent's qualname is the base, with `.` + /// appended when the parent is a function/lambda scope (so a nested + /// `def`/`class` reads `outer..inner`), and just the parent + /// qualname when the parent is a class body (so a method reads + /// `C.method`). The child name is then dotted onto that base. + fn compute_child_qualname(&self, name: &str) -> String { + if matches!(self.kind, CodeKind::Module) { + return name.to_owned(); + } + let mut base = self.co.qualname.clone(); + if matches!(self.kind, CodeKind::Function) { + base.push_str("."); + } + base.push('.'); + base.push_str(name); + base + } + fn finish(mut self) -> CodeObject { // Always terminate the code object with an implicit `return None`, // matching CPython's "fall off the end of the function" shape. @@ -1720,6 +1752,7 @@ impl Compiler { self.source.clone(), self.future_annotations, ); + inner.co.qualname = self.compute_child_qualname(name); inner.co.arg_count = arg_count; inner.co.posonly_count = posonly_count; inner.co.kwonly_count = kwonly_count; @@ -1933,6 +1966,7 @@ impl Compiler { self.source.clone(), self.future_annotations, ); + inner.co.qualname = self.compute_child_qualname(name); inner.current_line = self.current_line; // Every class body carries a `__class__` cell so methods can // close over it. `__build_class__` patches the cell with the @@ -1981,10 +2015,14 @@ impl Compiler { } inner.emit(OpCode::Resume, 0); - // `__module__ = __name__` and `__qualname__ = name` boilerplate. - let name_const = inner.co.intern_constant(Constant::Str(name.to_owned())); + // `__module__ = __name__` and `__qualname__ = ` + // boilerplate. The class body stores its full PEP 3155 qualname + // (e.g. `Outer.method..C`), not the bare name, so + // `C.__qualname__` and `repr`s built from it match CPython. + let qualname_str = inner.co.qualname.clone(); + let qualname_const = inner.co.intern_constant(Constant::Str(qualname_str)); let qualname_idx = inner.co.intern_name("__qualname__"); - inner.emit(OpCode::LoadConst, name_const); + inner.emit(OpCode::LoadConst, qualname_const); inner.emit(OpCode::StoreName, qualname_idx); // CPython stores a class body's leading string literal as diff --git a/crates/weavepy-lexer/src/lib.rs b/crates/weavepy-lexer/src/lib.rs index f5b3811..5f54fdb 100644 --- a/crates/weavepy-lexer/src/lib.rs +++ b/crates/weavepy-lexer/src/lib.rs @@ -60,6 +60,44 @@ mod tests { ); } + #[test] + fn eof_without_trailing_newline_terminates_indented_block() { + // CPython injects an implicit NEWLINE before the closing DEDENTs + // when the final logical line lacks a trailing newline. This is the + // exact shape `dataclasses`/`namedtuple` codegen feeds to `exec`. + let k = kinds("def f():\n return (x,)"); + assert_eq!( + k, + vec![ + TokenKind::Keyword(Keyword::Def), + TokenKind::Name, + TokenKind::LPar, + TokenKind::RPar, + TokenKind::Colon, + TokenKind::Newline, + TokenKind::Indent, + TokenKind::Keyword(Keyword::Return), + TokenKind::LPar, + TokenKind::Name, + TokenKind::Comma, + TokenKind::RPar, + TokenKind::Newline, + TokenKind::Dedent, + TokenKind::Endmarker, + ] + ); + } + + #[test] + fn eof_bare_identifier_no_newline_gets_newline() { + // `foo` with no trailing newline still terminates with NEWLINE. + let k = kinds("foo"); + assert_eq!( + k, + vec![TokenKind::Name, TokenKind::Newline, TokenKind::Endmarker] + ); + } + #[test] fn integer_kinds() { assert_eq!(kinds("42")[0], TokenKind::Number); diff --git a/crates/weavepy-lexer/src/scanner.rs b/crates/weavepy-lexer/src/scanner.rs index 554e877..86a6b0e 100644 --- a/crates/weavepy-lexer/src/scanner.rs +++ b/crates/weavepy-lexer/src/scanner.rs @@ -35,6 +35,20 @@ pub fn tokenize_with_escapes(source: &str) -> (Result, LexError>, Vec match scanner.next_token() { Ok(Some(tok)) => { let is_endmarker = matches!(tok.kind, TokenKind::Endmarker); + // Track whether the most recent token leaves a logical line + // "open" (i.e. needs a NEWLINE to terminate it). The EOF + // branch of `next_token` consults this to synthesize the + // implicit final NEWLINE CPython emits for source lacking a + // trailing newline. Structural/trivia tokens don't open a + // logical line. + scanner.last_was_content = !matches!( + tok.kind, + TokenKind::Newline + | TokenKind::Nl + | TokenKind::Indent + | TokenKind::Dedent + | TokenKind::Endmarker + ); out.push(tok); if is_endmarker { break Ok(out); @@ -66,6 +80,11 @@ struct Scanner<'src> { pending_indent: bool, /// True after we emitted ENDMARKER; further calls return None. finished: bool, + /// True when the most recently emitted token leaves a logical line + /// "open" — any token other than NEWLINE/NL/INDENT/DEDENT/ENDMARKER. + /// Drives the implicit final-NEWLINE synthesis in `next_token`'s EOF + /// branch (CPython terminates an unterminated last line this way). + last_was_content: bool, /// Invalid-escape `SyntaxWarning`s gathered while scanning string and /// bytes literals, in source order (the first invalid escape *per /// literal*, matching CPython's `first_invalid_escape` tracking). @@ -83,6 +102,7 @@ impl<'src> Scanner<'src> { pending_dedents: 0, pending_indent: false, finished: false, + last_was_content: false, escape_warnings: Vec::new(), } } @@ -201,6 +221,21 @@ impl<'src> Scanner<'src> { } let Some(b) = self.peek() else { + // CPython's tokenizer implicitly terminates a final logical line + // that lacks a trailing newline with a NEWLINE token *before* + // emitting the closing DEDENTs. Without it, source whose last + // line sits inside an indented block — e.g. + // `compile("def f():\n return (x,)", ...)`, exactly the shape + // `dataclasses`/`namedtuple`/`functools` codegen produces via + // `exec` — fails to parse, because the parser never sees the + // NEWLINE that closes the statement and the suite. We mirror + // CPython here. `last_was_content` is the reliable signal — + // `at_line_start` is cleared by `handle_line_start` at EOF even + // for newline-terminated input, which would double-emit. + if self.paren_depth == 0 && self.last_was_content { + self.last_was_content = false; + return Ok(Some(self.token(TokenKind::Newline, self.pos, self.pos))); + } return Ok(Some(self.emit_endmarker())); }; diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 1255f8b..8ae13b7 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -2577,9 +2577,12 @@ impl Interpreter { DictKey(Object::from_static("__name__")), name_obj.clone(), ); + // `__qualname__` is the code object's PEP 3155 dotted name + // (computed at compile time from lexical nesting), not the + // bare `__name__`. Pinned as a stable object like `__name__`. attrs.borrow_mut().insert( DictKey(Object::from_static("__qualname__")), - name_obj, + Object::from_str(code.qualname.clone()), ); if let Some(ann) = annotations_obj { attrs @@ -3524,7 +3527,11 @@ impl Interpreter { } match name { "__name__" => return Ok(Object::from_str(&f.name)), - "__qualname__" => return Ok(Object::from_str(&f.name)), + // PEP 3155 qualname comes from the code object (computed + // at compile time from lexical nesting), unless user code + // has overridden it via `f.__qualname__ = …` (handled by + // the `f.attrs` lookup above). + "__qualname__" => return Ok(Object::from_str(&f.code.qualname)), "__doc__" => { // CPython convention: the first statement of // the function body, if it is a string @@ -9963,10 +9970,21 @@ impl Interpreter { _ => return Err(type_error("type() arg 2 must be tuple of bases")), }; let ns_dict_obj = args[2].clone(); - let ns = match &args[2] { + let mut ns = match &args[2] { Object::Dict(d) => d.borrow().clone(), _ => return Err(type_error("type() arg 3 must be a dict")), }; + // CPython's `type.__new__` defaults `__doc__` to `None` when the + // namespace doesn't define one, so `Cls.__doc__` reads `None` + // rather than raising `AttributeError`. The `class` statement path + // does this in `build_class`; the dynamic `type(name, bases, ns)` / + // `types.new_class` / `dataclasses.make_dataclass` path must match. + { + let key = DictKey(Object::from_static("__doc__")); + if ns.get(&key).is_none() { + ns.insert(key, Object::None); + } + } let mut effective_bases = bases.clone(); if effective_bases.is_empty() { effective_bases.push(builtin_types().object_.clone()); @@ -10908,26 +10926,39 @@ impl Interpreter { } } } - for (i, was_filled) in filled.iter().take(total_args).enumerate() { - if !was_filled { - return Err(type_error(format!( - "{}() missing required argument: '{}'", - f.name, code.varnames[i] - ))); - } + // CPython renders missing arguments as e.g. + // `f() missing 1 required positional argument: 'x'` or + // `f() missing 2 required positional arguments: 'x' and 'y'`, + // listing *all* of them (Oxford-comma joined for 3+). Many stdlib + // tests assertRaisesRegex against this exact wording. + let missing_positional: Vec<&str> = filled + .iter() + .take(total_args) + .enumerate() + .filter(|(_, was_filled)| !**was_filled) + .map(|(i, _)| code.varnames[i].as_str()) + .collect(); + if !missing_positional.is_empty() { + return Err(type_error(format_missing_arguments( + &f.name, + "positional", + &missing_positional, + ))); } - for (i, was_filled) in filled + let missing_kwonly: Vec<&str> = filled .iter() .enumerate() .skip(kwonly_start) .take(kwonly_end - kwonly_start) - { - if !was_filled { - return Err(type_error(format!( - "{}() missing required keyword-only argument: '{}'", - f.name, code.varnames[i] - ))); - } + .filter(|(_, was_filled)| !**was_filled) + .map(|(i, _)| code.varnames[i].as_str()) + .collect(); + if !missing_kwonly.is_empty() { + return Err(type_error(format_missing_arguments( + &f.name, + "keyword-only", + &missing_kwonly, + ))); } let mut frame = self.make_frame( code.clone(), @@ -13678,6 +13709,32 @@ fn ascii_escape(r: &str) -> String { out } +/// Render a "missing required argument(s)" `TypeError` message in +/// CPython's exact phrasing (`Python/ceval.c` `format_missing`): +/// +/// - `f() missing 1 required positional argument: 'x'` +/// - `f() missing 2 required positional arguments: 'x' and 'y'` +/// - `f() missing 3 required positional arguments: 'x', 'y', and 'z'` +/// +/// `kind` is `"positional"` or `"keyword-only"`. `names` must be non-empty. +fn format_missing_arguments(func_name: &str, kind: &str, names: &[&str]) -> String { + let count = names.len(); + let joined = match names { + [one] => format!("'{one}'"), + [a, b] => format!("'{a}' and '{b}'"), + _ => { + let head = names[..count - 1] + .iter() + .map(|n| format!("'{n}'")) + .collect::>() + .join(", "); + format!("{head}, and '{}'", names[count - 1]) + } + }; + let plural = if count == 1 { "" } else { "s" }; + format!("{func_name}() missing {count} required {kind} argument{plural}: {joined}") +} + /// The user-visible `__name__`/`__qualname__` for a builtin. Internal /// builtins registered under a dotted sentinel (e.g. the `str.format` /// method as `.format`, or `gc.collect` as `.gc.collect`) report their diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 28929e7..3672a09 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -1677,8 +1677,8 @@ impl Object { } _ => Err(type_error(format!( "'<' not supported between instances of '{}' and '{}'", - self.type_name(), - other.type_name() + self.type_name_owned(), + other.type_name_owned() ))), } } diff --git a/crates/weavepy-vm/src/stdlib/marshal_mod.rs b/crates/weavepy-vm/src/stdlib/marshal_mod.rs index fdde30f..4237ecc 100644 --- a/crates/weavepy-vm/src/stdlib/marshal_mod.rs +++ b/crates/weavepy-vm/src/stdlib/marshal_mod.rs @@ -337,9 +337,10 @@ impl MarshalWriter { self.write_value(&Object::new_bytes(cp.localspluskinds))?; self.write_value(&Object::from_str(co.filename.clone()))?; self.write_value(&Object::from_str(co.name.clone()))?; - // We don't track a separate qualified name; the plain name is a - // faithful stand-in for top-level defs and is what `dis` prints. - self.write_value(&Object::from_str(co.name.clone()))?; + // PEP 3155 qualified name, computed at compile time from lexical + // nesting (`outer..inner`, `C.method`). Round-trips so an + // unmarshalled function/class keeps a faithful `__qualname__`. + self.write_value(&Object::from_str(co.qualname.clone()))?; self.write_int(cp.firstlineno as i32); self.write_value(&Object::new_bytes(cp.co_linetable))?; self.write_value(&Object::new_bytes(cp.co_exceptiontable))?; @@ -624,7 +625,7 @@ impl<'a> MarshalReader<'a> { let localspluskinds = self.read_value()?; let filename = self.read_value()?; let name = self.read_value()?; - let _qualname = self.read_value()?; + let qualname = self.read_value()?; let firstlineno = self.read_int()? as u32; let linetable = self.read_value()?; let exceptiontable = self.read_value()?; @@ -645,8 +646,13 @@ impl<'a> MarshalReader<'a> { ) .ok_or_else(|| value_error("marshal: code object uses an unsupported opcode"))?; + let co_name = string_of(&name, "co_name")?; + // Fall back to the bare name when the producer didn't record a + // qualname (e.g. older marshal payloads); CPython always writes one. + let co_qualname = string_of(&qualname, "co_qualname").unwrap_or_else(|_| co_name.clone()); let co = CodeObject { - name: string_of(&name, "co_name")?, + name: co_name, + qualname: co_qualname, filename: string_of(&filename, "co_filename")?, caches: CacheTable::with_len(decoded.instructions.len()), instructions: decoded.instructions, From c0107d13d00879afb80a70a577dceeac1730b289 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Tue, 9 Jun 2026 12:13:13 -0700 Subject: [PATCH 6/9] feat: advance CPython Lib/test conformance wave 2 --- crates/weavepy-compiler/src/lib.rs | 157 +- crates/weavepy-conformance/src/regrtest.rs | 13 + crates/weavepy-vm/src/builtin_types.rs | 227 +- crates/weavepy-vm/src/builtins.rs | 260 +- crates/weavepy-vm/src/error.rs | 9 +- crates/weavepy-vm/src/lib.rs | 2024 +++++++++++- crates/weavepy-vm/src/object.rs | 135 +- crates/weavepy-vm/src/stdlib/mod.rs | 54 + .../src/stdlib/python/_pyrepl_init.py | 6 + .../src/stdlib/python/_pyrepl_pager.py | 175 + .../weavepy-vm/src/stdlib/python/_seqtools.py | 118 + .../src/stdlib/python/_testlimitedcapi.py | 24 + crates/weavepy-vm/src/stdlib/python/enum.py | 2434 ++++++++++++-- .../src/stdlib/python/importlib_bootstrap.py | 41 + .../python/importlib_bootstrap_external.py | 26 + .../weavepy-vm/src/stdlib/python/inspect.py | 347 +- .../weavepy-vm/src/stdlib/python/itertools.py | 63 +- crates/weavepy-vm/src/stdlib/python/pydoc.py | 2859 +++++++++++++++++ .../weavepy-vm/src/stdlib/python/sysconfig.py | 184 ++ .../src/stdlib/python/test_support_init.py | 1 + crates/weavepy-vm/src/stdlib/python/token.py | 141 + .../weavepy-vm/src/stdlib/python/tokenize.py | 613 ++++ crates/weavepy-vm/src/stdlib/sys.rs | 7 + .../src/stdlib/testinternalcapi_mod.rs | 67 + crates/weavepy-vm/src/stdlib/thread_real.rs | 2 + crates/weavepy-vm/src/stdlib/weakref_real.rs | 1 + crates/weavepy-vm/src/types.rs | 118 + crates/weavepy-vm/src/weakref_registry.rs | 1 + 28 files changed, 9524 insertions(+), 583 deletions(-) create mode 100644 crates/weavepy-vm/src/stdlib/python/_pyrepl_init.py create mode 100644 crates/weavepy-vm/src/stdlib/python/_pyrepl_pager.py create mode 100644 crates/weavepy-vm/src/stdlib/python/_testlimitedcapi.py create mode 100644 crates/weavepy-vm/src/stdlib/python/importlib_bootstrap.py create mode 100644 crates/weavepy-vm/src/stdlib/python/importlib_bootstrap_external.py create mode 100755 crates/weavepy-vm/src/stdlib/python/pydoc.py create mode 100644 crates/weavepy-vm/src/stdlib/python/sysconfig.py create mode 100644 crates/weavepy-vm/src/stdlib/python/token.py create mode 100644 crates/weavepy-vm/src/stdlib/python/tokenize.py create mode 100644 crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index 971148a..2fb4e2c 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -616,6 +616,11 @@ enum FinallyKind { /// Name node because the synthetic local name (".with_cm0") /// isn't a valid identifier and would fail name resolution. WithExit { cm_idx: u32 }, + /// Synthetic frame for an `async with` block: emit + /// `await (None, None, None)`. Mirrors `WithExit` + /// but awaits the `__aexit__` coroutine, so a `return`/`break`/ + /// `continue` out of an `async with` body still runs the exit. + AsyncWithExit { aexit_idx: u32 }, } struct FinallyFrame { @@ -2139,6 +2144,19 @@ impl Compiler { self.emit(OpCode::PopTop, 0); Ok(()) } + FinallyKind::AsyncWithExit { aexit_idx } => { + // `await (None, None, None)`. The bound coroutine + // method was stashed at `aexit_idx` by `compile_async_with`. + self.emit(OpCode::LoadFast, *aexit_idx); + let none_idx = self.co.intern_constant(Constant::None); + self.emit(OpCode::LoadConst, none_idx); + self.emit(OpCode::LoadConst, none_idx); + self.emit(OpCode::LoadConst, none_idx); + self.emit(OpCode::Call, 3); + self.compile_await_dance(2); + self.emit(OpCode::PopTop, 0); + Ok(()) + } } } @@ -2424,6 +2442,14 @@ impl Compiler { handler: handlers_start, depth: body_depth, }); + // Record the propagating exception as the active handled + // exception for the duration of the finally body. Without + // this a `raise` inside `finally` (e.g. a `@contextmanager` + // generator's `finally: raise`) gets no implicit + // `__context__`, breaking PEP 3134 chaining. `PUSH_EXC_INFO` + // only peeks the value-stack top in this VM, so the + // exception stays put for the trailing `RERAISE 0`. + let push_exc_site = self.emit(OpCode::PushExcInfo, 0); let saved = self.finally_stack.pop(); for s in finalbody { self.compile_stmt(s)?; @@ -2432,6 +2458,12 @@ impl Compiler { self.finally_stack.push(f); } self.emit(OpCode::Reraise, 0); + // Tag the active-handler entry with the pc just past the + // RERAISE so the unwinder drops it when a `raise` inside the + // finally escapes to an enclosing `try` (mirrors the + // except-handler path above). + let end = self.next_offset(); + self.co.instructions[push_exc_site as usize].arg = end; } // Patch normal exit jump to land after handlers/finally. if has_handlers || has_finally { @@ -2536,7 +2568,15 @@ impl Compiler { handler: handler_start, depth: body_depth, }); - // Stack: [exc] + // Stack: [exc]. Record the propagating exception as the active + // handled exception for the duration of the `__exit__` call so a + // `raise` inside `__exit__` chains it as the new exception's + // implicit `__context__` (PEP 3134). This is what makes + // `contextlib.ExitStack`'s `_fix_exception_context` work — it + // walks each callback exception's context back to + // `sys.exc_info()[1]`. `PUSH_EXC_INFO` only peeks the value-stack + // top in this VM, so `[exc]` is preserved for `WITH_EXCEPT_START`. + let push_exc_site = self.emit(OpCode::PushExcInfo, 0); self.emit(OpCode::LoadFast, cm_idx); self.emit(OpCode::LoadAttr, exit_name); // Stack: [exc, __exit__] @@ -2551,11 +2591,17 @@ impl Compiler { self.emit(OpCode::RaiseVarargs, 1); let swallow_target = self.next_offset(); self.patch_jump(swallow, swallow_target); - // Swallowed: Stack: [__exit__, exc] + // Swallowed: Stack: [__exit__, exc]. Drop the active handled-exc + // entry now that the suppressing `__exit__` returned cleanly. + self.emit(OpCode::PopExcept, 0); self.emit(OpCode::PopTop, 0); self.emit(OpCode::PopTop, 0); let end = self.next_offset(); self.patch_jump(end_jump, end); + // Tag the active-handler entry with the pc just past the handler + // so the unwinder drops it if `__exit__` raises and the new + // exception escapes to an enclosing `try`. + self.co.instructions[push_exc_site as usize].arg = end; Ok(()) } @@ -3282,7 +3328,15 @@ impl Compiler { let idx = self.co.intern_constant(Constant::None); self.emit(OpCode::LoadConst, idx); } - self.emit(OpCode::YieldValue, 0); + // An async generator's *own* `yield` produces a value for the + // consumer (`__anext__`), distinct from the `YIELD_VALUE` the + // `await`/`yield from` dance emits to pass an inner + // suspension's value through (oparg 0). The runtime uses this + // marker (CPython's `PyAsyncGenWrappedValue`) to tell "the + // agen yielded X" from "the agen is suspended on an inner + // await that yielded X". + let yield_arg = u32::from(self.co.is_async_generator); + self.emit(OpCode::YieldValue, yield_arg); } ExprKind::YieldFrom(iter) => { if self.kind != CodeKind::Function { @@ -3427,12 +3481,27 @@ impl Compiler { self.emit(OpCode::PopTop, 0); } // Stash aexit in a synthetic local so we can recover it on - // the exit path. (We don't have a full exception table for - // async with yet — this is enough for the no-exception path.) + // both the normal-exit and the exception-cleanup paths. let slot = format!(".aexit{}", self.with_counter); self.with_counter += 1; let slot_idx = self.var_index_or_add(&slot); self.emit(OpCode::StoreFast, slot_idx); + + // Synthetic finally frame so `return`/`break`/`continue` out of + // the body still `await __aexit__(None, None, None)`. Mirrors the + // `WithExit` frame `compile_with` pushes; without it an early exit + // from an `async with` body skipped the exit entirely (e.g. an + // `@asynccontextmanager` used as a decorator never ran its + // post-`yield` cleanup). + let awith_loop_depth = self.loop_stack.len(); + self.finally_stack.push(FinallyFrame { + kind: FinallyKind::AsyncWithExit { + aexit_idx: slot_idx, + }, + loop_depth_at_push: awith_loop_depth, + }); + + let body_start = self.next_offset(); if rest.is_empty() { for s in body { self.compile_stmt(s)?; @@ -3440,15 +3509,75 @@ impl Compiler { } else { self.compile_async_with(rest, body)?; } - // Normal exit: push aexit, call with (None, None, None), await. + let body_end = self.next_offset(); + + // Pop the synthetic frame; the explicit normal-exit and + // exception-cleanup paths below emit their own `__aexit__` call. + self.finally_stack.pop(); + + // Normal exit: `await aexit(None, None, None)`. self.emit(OpCode::LoadFast, slot_idx); let none_idx = self.co.intern_constant(Constant::None); self.emit(OpCode::LoadConst, none_idx); self.emit(OpCode::LoadConst, none_idx); self.emit(OpCode::LoadConst, none_idx); self.emit(OpCode::Call, 3); - self.compile_await_dance(0); + self.compile_await_dance(2); self.emit(OpCode::PopTop, 0); + let end_jump = self.emit(OpCode::JumpForward, 0); + + // Exception-cleanup path — the async counterpart of the handler + // emitted by `compile_with`: `result = await aexit(type(exc), exc, + // None)`; if `result` is truthy the exception is swallowed, + // otherwise it is re-raised. The previous codegen omitted this + // entirely, so an exception escaping an `async with` body never + // reached `__aexit__` and could not be suppressed (the `with` + // statement's `__exit__` already had this). + let handler_start = self.next_offset(); + // Preserve enclosing for-loop iterators on the operand stack, the + // same depth convention used by `try`/`except` and `compile_with`. + let body_depth = self.loop_stack.iter().filter(|fr| fr.is_for_loop).count() as u32; + self.co.exception_table.push(ExcHandler { + start: body_start, + end: body_end, + handler: handler_start, + depth: body_depth, + }); + // Stack: [exc]. Record the propagating exception as the active + // handled exception for the duration of the awaited `__aexit__`, + // exactly as the sync `with` handler does. Without it the body's + // exception isn't visible via `sys.exc_info()` inside `__aexit__` + // (a coroutine driven by the await dance below), so a `raise` + // there gets no implicit `__context__` and + // `contextlib.AsyncExitStack`'s `_fix_exception_context` (which + // walks each callback exception back to `sys.exc_info()[1]`) + // cannot reconstruct the chain. + let push_exc_site = self.emit(OpCode::PushExcInfo, 0); + self.emit(OpCode::LoadFast, slot_idx); + // Stack: [exc, aexit] + self.emit(OpCode::Swap, 2); + // Stack: [aexit, exc] + self.emit(OpCode::WithExceptStart, 0); + // Stack: [aexit, exc, awaitable] — await the `__aexit__` coroutine. + self.compile_await_dance(2); + // Stack: [aexit, exc, result] + let swallow = self.emit(OpCode::PopJumpIfTrue, 0); + // Falsy: re-raise. Stack: [aexit, exc] + self.emit(OpCode::Swap, 2); + self.emit(OpCode::PopTop, 0); + self.emit(OpCode::RaiseVarargs, 1); + let swallow_target = self.next_offset(); + self.patch_jump(swallow, swallow_target); + // Swallowed: Stack: [aexit, exc]. Drop the active handled-exc + // entry now that the suppressing `__aexit__` returned cleanly. + self.emit(OpCode::PopExcept, 0); + self.emit(OpCode::PopTop, 0); + self.emit(OpCode::PopTop, 0); + let end = self.next_offset(); + self.patch_jump(end_jump, end); + // Tag the active-handler entry with the pc just past the handler + // so the unwinder drops it if `__aexit__` raises a new exception. + self.co.instructions[push_exc_site as usize].arg = end; Ok(()) } @@ -3593,6 +3722,10 @@ impl Compiler { self.future_annotations, ); inner.current_line = self.current_line; + // PEP 3155: a comprehension scope gets a dotted qualname like any + // other nested scope (`C.m..`); CPython's + // `compiler_set_qualname` doesn't special-case comprehensions. + inner.co.qualname = self.compute_child_qualname(name); inner.co.arg_count = 1; inner.co.varnames.push(".0".to_owned()); inner.bindings.insert(".0".to_owned(), Binding::Local); @@ -3801,7 +3934,12 @@ fn compile_comp_body( } OpCode::YieldValue => { inner.compile_expr(elt)?; - inner.emit(OpCode::YieldValue, 0); + // An async-generator comprehension `(x async for x in xs)` + // yields a consumer value here; mark it (arg 1) like a plain + // async-gen `yield` so the runtime's passthrough machinery + // doesn't mistake it for an inner-await suspension. Sync + // genexps stay arg 0. + inner.emit(OpCode::YieldValue, u32::from(inner.co.is_async_generator)); inner.emit(OpCode::PopTop, 0); } _ => { @@ -3937,6 +4075,9 @@ fn clone_finally_frame(f: &FinallyFrame) -> FinallyFrame { let kind = match &f.kind { FinallyKind::Stmts(body) => FinallyKind::Stmts(body.clone()), FinallyKind::WithExit { cm_idx } => FinallyKind::WithExit { cm_idx: *cm_idx }, + FinallyKind::AsyncWithExit { aexit_idx } => { + FinallyKind::AsyncWithExit { aexit_idx: *aexit_idx } + } }; FinallyFrame { kind, diff --git a/crates/weavepy-conformance/src/regrtest.rs b/crates/weavepy-conformance/src/regrtest.rs index 3d6f53a..8e35ede 100644 --- a/crates/weavepy-conformance/src/regrtest.rs +++ b/crates/weavepy-conformance/src/regrtest.rs @@ -265,6 +265,19 @@ pub fn discover_with( path: p, label: format!("cpython/Lib/test/{name}"), }); + continue; + } + // Some regression tests are *packages* (`test_dataclasses/` + // with an `__init__.py`); keep the `.py` label but run the + // package entry point. + let pkg_init = dir + .join(name.trim_end_matches(".py")) + .join("__init__.py"); + if pkg_init.is_file() { + out.push(RegrtestFile { + path: pkg_init, + label: format!("cpython/Lib/test/{name}"), + }); } } } diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index 69c11f8..b96151c 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -53,6 +53,12 @@ pub struct BuiltinTypes { pub simple_namespace_: Rc, pub function_: Rc, pub method_: Rc, + /// `builtin_function_or_method` — the type of Rust-implemented + /// callables (`type(len)`), distinct from `function` as in CPython. + pub builtin_function_: Rc, + /// `method-wrapper` — the type of a slot wrapper bound to an + /// instance (`type(object().__str__)`). + pub method_wrapper_: Rc, pub generator_: Rc, pub coroutine_: Rc, pub async_generator_: Rc, @@ -185,6 +191,14 @@ impl BuiltinTypes { // `function` so `type(obj.meth)` is `method` (as in CPython) and // `types.MethodType(func, obj)` can construct a bound method. let method_ = mk("method", vec![object_.clone()]); + // `types.BuiltinFunctionType` — Rust-implemented callables. + // CPython keeps this distinct from `function` (`type(len) is not + // type(lambda: 0)`); `inspect`/`pydoc` classification relies on + // the distinction. + let builtin_function_ = mk("builtin_function_or_method", vec![object_.clone()]); + // `types.MethodWrapperType` — a slot-wrapper dunder bound to an + // instance (`object().__str__`). + let method_wrapper_ = mk("method-wrapper", vec![object_.clone()]); let generator_ = mk("generator", vec![object_.clone()]); let coroutine_ = mk("coroutine", vec![object_.clone()]); let async_generator_ = mk("async_generator", vec![object_.clone()]); @@ -342,6 +356,8 @@ impl BuiltinTypes { simple_namespace_, function_, method_, + builtin_function_, + method_wrapper_, generator_, coroutine_, async_generator_, @@ -419,6 +435,11 @@ impl BuiltinTypes { } // RFC 0019 — install numeric/bytes class methods. install_numeric_class_methods(&bt); + // Install `__new__` in each value/container type's own dict (CPython + // keeps a distinct `tp_new` per type). Needed so `'__new__' in + // int.__dict__` is True — `enum._find_data_type_` uses exactly this to + // recognise `int`/`str`/… as the mix-in data type. + install_value_type_new(&bt); bt } @@ -803,37 +824,52 @@ fn native_seed_for_new(cls: &Rc, value: Option<&Object>) -> Option Result { + use crate::types::PyInstance; + let cls = match args.first() { + Some(Object::Type(t)) => t.clone(), + _ => { + return Err(crate::error::type_error( + "object.__new__(): first arg must be a class".to_owned(), + )) + } + }; + // When `cls` derives from a value/container built-in (`int`, `float`, + // `str`, `tuple`, `list`, `dict`, …) capture the native payload the + // instance wraps so the inherited protocols keep firing through the + // subclass. `super().__new__(cls, value)` passes the seed value as the + // second positional argument (how `copyreg.__newobj__` reconstructs + // immutable subclasses); mutable containers start empty and are filled by + // `__init__` / `__setstate__` / the `_reconstruct` append-and-update loop. + if let Some(native) = native_seed_for_new(&cls, args.get(1)) { + return Ok(Object::Instance(Rc::new(PyInstance::with_native(cls, native)))); + } + Ok(Object::Instance(Rc::new(PyInstance::new(cls)))) +} + +/// A fresh `Object::StaticMethod(Builtin "__new__")` wrapping [`object_new`]. +/// Each call returns a *distinct* object so `int.__new__ is object.__new__` +/// is `False` (matching CPython) while the instantiation path still treats it +/// as the default allocator (it keys on the builtin's `"__new__"` name). +fn make_default_new() -> Object { + use crate::object::BuiltinFn; + Object::StaticMethod(Rc::new(Object::Builtin(Rc::new(BuiltinFn { + name: "__new__", + call: Box::new(object_new), + call_kw: None, + })))) +} + /// Install `object.__new__`, `object.__init__`, `object.__setattr__` /// and `object.__delattr__` on the root class. These are the implicit /// base methods every user class inherits. fn install_object_dunders(object_: &Rc) { use crate::object::BuiltinFn; - use crate::types::PyInstance; - fn object_new(args: &[Object]) -> Result { - // `object.__new__(cls, *args, **kwargs)` — args[0] is `cls`. - let cls = match args.first() { - Some(Object::Type(t)) => t.clone(), - _ => { - return Err(crate::error::type_error( - "object.__new__(): first arg must be a class".to_owned(), - )) - } - }; - // When `cls` derives from a value/container built-in (`int`, - // `float`, `str`, `tuple`, `list`, `dict`, …) capture the native - // payload the instance wraps so the inherited protocols keep - // firing through the subclass. `super().__new__(cls, value)` - // passes the seed value as the second positional argument (how - // `copyreg.__newobj__` reconstructs immutable subclasses); mutable - // containers start empty and are filled by `__init__` / - // `__setstate__` / the `_reconstruct` append-and-update loop. - if let Some(native) = native_seed_for_new(&cls, args.get(1)) { - return Ok(Object::Instance(Rc::new(PyInstance::with_native( - cls, native, - )))); - } - Ok(Object::Instance(Rc::new(PyInstance::new(cls)))) - } fn object_init(_args: &[Object]) -> Result { // No-op; honours `super().__init__()` chains. Ok(Object::None) @@ -848,23 +884,40 @@ fn install_object_dunders(object_: &Rc) { "object.__setattr__() takes 3 arguments".to_owned(), )); } - let inst = match &args[0] { - Object::Instance(i) => i.clone(), - other => { - return Err(crate::error::type_error(format!( - "object.__setattr__() requires an instance, got '{}'", - other.type_name() - ))) - } - }; let name = match &args[1] { Object::Str(s) => s.to_string(), _ => return Err(crate::error::type_error("attribute name must be str")), }; - inst.dict - .borrow_mut() - .insert(DictKey(Object::from_str(name)), args[2].clone()); - Ok(Object::None) + match &args[0] { + Object::Instance(inst) => { + inst.dict + .borrow_mut() + .insert(DictKey(Object::from_str(name)), args[2].clone()); + Ok(Object::None) + } + // `type.__setattr__` semantics for a class receiver — reached + // via `super().__setattr__(…)` inside a metaclass override + // (e.g. `EnumType.__setattr__` chaining to the default). + Object::Type(ty) => { + if ty.flags.is_builtin { + return Err(crate::error::type_error(format!( + "cannot set '{name}' attribute of immutable type '{}'", + ty.name + ))); + } + ty.dict + .borrow_mut() + .insert(DictKey(Object::from_str(&name)), args[2].clone()); + if name == "__getattribute__" { + ty.invalidate_getattribute_cache(); + } + Ok(Object::None) + } + other => Err(crate::error::type_error(format!( + "object.__setattr__() requires an instance, got '{}'", + other.type_name() + ))), + } } fn object_delattr(args: &[Object]) -> Result { if args.len() != 2 { @@ -872,31 +925,52 @@ fn install_object_dunders(object_: &Rc) { "object.__delattr__() takes 2 arguments".to_owned(), )); } - let inst = match &args[0] { - Object::Instance(i) => i.clone(), - other => { - return Err(crate::error::type_error(format!( - "object.__delattr__() requires an instance, got '{}'", - other.type_name() - ))) - } - }; let name = match &args[1] { Object::Str(s) => s.to_string(), _ => return Err(crate::error::type_error("attribute name must be str")), }; - let removed = inst - .dict - .borrow_mut() - .shift_remove(&DictKey(Object::from_str(&name))) - .is_some(); - if !removed { - return Err(crate::error::attribute_error(format!( - "'{}' object has no attribute '{}'", - inst.class.name, name - ))); + match &args[0] { + Object::Instance(inst) => { + let removed = inst + .dict + .borrow_mut() + .shift_remove(&DictKey(Object::from_str(&name))) + .is_some(); + if !removed { + return Err(crate::error::attribute_error(format!( + "'{}' object has no attribute '{}'", + inst.class.name, name + ))); + } + Ok(Object::None) + } + // `type.__delattr__` semantics for a class receiver (chained + // via `super().__delattr__(…)` in a metaclass override). + Object::Type(ty) => { + if ty.flags.is_builtin { + return Err(crate::error::type_error(format!( + "cannot delete '{name}' attribute of immutable type '{}'", + ty.name + ))); + } + let removed = ty + .dict + .borrow_mut() + .shift_remove(&DictKey(Object::from_str(&name))) + .is_some(); + if !removed { + return Err(crate::error::attribute_error(format!( + "type object '{}' has no attribute '{}'", + ty.name, name + ))); + } + Ok(Object::None) + } + other => Err(crate::error::type_error(format!( + "object.__delattr__() requires an instance, got '{}'", + other.type_name() + ))), } - Ok(Object::None) } fn object_hash(args: &[Object]) -> Result { // Default `object.__hash__`: the same canonical hash the `hash()` @@ -918,11 +992,7 @@ fn install_object_dunders(object_: &Rc) { ); dict.insert( DictKey(Object::from_static("__new__")), - Object::StaticMethod(Rc::new(Object::Builtin(Rc::new(BuiltinFn { - name: "__new__", - call: Box::new(object_new), - call_kw: None, - })))), + make_default_new(), ); dict.insert( DictKey(Object::from_static("__init__")), @@ -1933,6 +2003,35 @@ pub fn instance_is_subclass(obj: &Object, cls: &TypeObject) -> bool { } } +/// Install a distinct `__new__` in each value/container built-in's own dict. +/// +/// CPython exposes a per-type `tp_new` in `tp_dict`, so `'__new__' in +/// int.__dict__` is True and `int.__new__ is not object.__new__`. WeavePy's +/// instantiation path keys the "default allocator" check on the builtin's +/// `"__new__"` name (not its type), so these all route through the same +/// native-seeding allocator — only their *identity* differs, which is what +/// `enum`'s `_find_data_type_` / `_find_new_` inspect. +fn install_value_type_new(bt: &BuiltinTypes) { + for ty in [ + &bt.int_, + &bt.float_, + &bt.bool_, + &bt.complex_, + &bt.str_, + &bt.bytes_, + &bt.bytearray_, + &bt.tuple_, + &bt.list_, + &bt.dict_, + &bt.set_, + &bt.frozenset_, + ] { + ty.dict + .borrow_mut() + .insert(DictKey(Object::from_static("__new__")), make_default_new()); + } +} + /// RFC 0019 — install class methods on the numeric / bytes types. /// Adds `int.from_bytes`, `bytes.fromhex`, `bytearray.fromhex`, /// and `float.fromhex` as classmethod-shaped builtins so that diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index e6e36e2..f0342dd 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -618,11 +618,18 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { }, // `property` objects expose `getter`/`setter`/`deleter` // methods that return a *new* property carrying a patched - // function (the underlying decorator pattern). + // function (the underlying decorator pattern), plus the + // explicit descriptor-protocol slots — CPython's `property` is + // a data descriptor precisely because its *type* defines + // `__set__`/`__delete__`, and `inspect.isdatadescriptor` + // checks exactly that. Object::Property(_) => match name { "getter" => Some(method("getter", property_getter)), "setter" => Some(method("setter", property_setter)), "deleter" => Some(method("deleter", property_deleter)), + "__get__" => Some(method("__get__", property_dunder_get)), + "__set__" => Some(method("__set__", property_dunder_set)), + "__delete__" => Some(method("__delete__", property_dunder_delete)), "fget" | "fset" | "fdel" | "__doc__" => { // These are looked up via `lookup_attr` in the VM // rather than method dispatch; we don't return them @@ -631,6 +638,17 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { } _ => None, }, + // Non-data descriptor protocol slots, reachable both bound + // (`sm.__get__`) via `load_attr` and unbound + // (`staticmethod.__get__`) via the slot-wrapper table. + Object::StaticMethod(_) => match name { + "__get__" => Some(method("__get__", staticmethod_descr_get)), + _ => None, + }, + Object::ClassMethod(_) => match name { + "__get__" => Some(method("__get__", classmethod_descr_get)), + _ => None, + }, Object::Int(_) | Object::Long(_) | Object::Bool(_) => match name { "bit_length" => Some(method("bit_length", int_bit_length)), "bit_count" => Some(method("bit_count", int_bit_count)), @@ -1052,6 +1070,10 @@ fn numeric_dunder(self_repr: &Object, name: &str) -> Option { }), "__format__" => method("__format__", |args| { let value = args.first().cloned().unwrap_or(Object::None); + // A subclass instance (e.g. an `IntEnum` member) formats its + // native payload — `int.__format__(member, '')` is `'3'`, + // never the instance repr. + let value = value.native_value().unwrap_or(value); let spec = match args.get(1) { Some(Object::Str(s)) => s.to_string(), Some(other) => { @@ -1192,11 +1214,135 @@ pub fn unbound_method(type_name: &str, name: &str) -> Option { items: Rc::from(Vec::::new()), index: 0, }))), + // Descriptor types: expose their protocol slots + // (`property.__set__`, `staticmethod.__get__`, …) for + // type-level access; the call receives the real descriptor as + // `self` via `args[0]`. + "property" => Object::Property(Rc::new(crate::object::PyProperty::new( + Object::None, + Object::None, + Object::None, + Object::None, + ))), + "staticmethod" => Object::StaticMethod(Rc::new(Object::None)), + "classmethod" => Object::ClassMethod(Rc::new(Object::None)), _ => return None, }; lookup_method(&rep, name) } +// ---- universal object-protocol slot wrappers (`object.__repr__`, …) ---- +// +// CPython stores a slot wrapper for the object protocol in every type's +// `tp_dict` (`object.__repr__`, `int.__str__`, `str.__format__`, …). WeavePy +// synthesizes these on demand for *type-level* attribute access only (the +// instance path keeps using `repr_of` / `stringify`), and the caller caches +// the result per `(type, name)` so identity is stable — `enum`'s bootstrap +// compares `getattr(member_type, '__str__') is object.__str__` and +// `found_method in (data_type_method, object_method)`. + +/// `object.__repr__(self)` / `int.__repr__(self)` / … — the default repr of +/// `self`, unwrapping a built-in subclass's native payload first (so +/// `int.__repr__(IntEnumMember)` renders the wrapped integer). +fn slot_repr(args: &[Object]) -> Result { + let o = args + .first() + .ok_or_else(|| type_error("__repr__() takes exactly one argument (0 given)"))?; + let native = o.native_value(); + Ok(Object::from_str(native.as_ref().unwrap_or(o).repr())) +} + +/// `str.__str__(self)` / `object.__str__(self)` — `str()` of `self`. Mirrors +/// CPython: for a value that doesn't define its own `__str__`, this is the +/// `repr`-derived default; for `str`/`bytes` it returns the payload. +fn slot_str(args: &[Object]) -> Result { + let o = args + .first() + .ok_or_else(|| type_error("__str__() takes exactly one argument (0 given)"))?; + let native = o.native_value(); + Ok(Object::from_str(native.as_ref().unwrap_or(o).to_str())) +} + +/// `object.__format__(self, spec)` / `str.__format__(self, spec)` — format +/// `self` per `spec`, unwrapping a native payload first. An empty spec is +/// equivalent to `str(self)`. +fn slot_format(args: &[Object]) -> Result { + let o = args + .first() + .ok_or_else(|| type_error("__format__() takes exactly 2 arguments (0 given)"))?; + let spec = match args.get(1) { + Some(Object::Str(s)) => s.to_string(), + Some(Object::None) | None => String::new(), + Some(other) => { + return Err(type_error(format!( + "__format__() argument 1 must be str, not {}", + other.type_name() + ))) + } + }; + let native = o.native_value(); + crate::format_via_spec(native.as_ref().unwrap_or(o), &spec).map(Object::from_str) +} + +/// `type.__call__` / `function.__call__` / … — invoke `args[0]` with the +/// remaining arguments (CPython's `tp_call` slot exposed as a wrapper). +fn slot_call(args: &[Object], kwargs: &[(String, Object)]) -> Result { + let callee = args + .first() + .ok_or_else(|| type_error("__call__ needs an argument"))?; + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| crate::error::runtime_error("no running interpreter"))?; + // SAFETY: published by an enclosing VM frame still live on this + // thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + interp.call_object_with_globals(callee, &args[1..], kwargs, &globals) +} + +/// Resolve the slot wrapper a *built-in* type `base_name` contributes for the +/// dunder `name`, or `None` if that type does not define it (so the caller's +/// MRO walk falls through to the next built-in base). Reuses the canonical +/// value-type implementations ([`unbound_method`]) and adds the universal +/// object-protocol dunders (`__repr__`/`__str__`/`__format__`) that aren't +/// modeled there. +/// +/// `__str__` is intentionally restricted to the string-like built-ins; the +/// numeric/container types inherit `object.__str__` exactly as in CPython, so +/// `int.__str__ is object.__str__` holds and `IntEnum` correctly falls back to +/// `int.__repr__` for member stringification. +pub fn builtin_type_dunder(base_name: &str, name: &str) -> Option { + if let Some(o) = unbound_method(base_name, name) { + return Some(o); + } + // `__call__` lives only on the callable types (CPython: `tp_call` + // present on `type`, functions, methods — not on `object`). + if name == "__call__" + && matches!( + base_name, + "type" | "function" | "builtin_function_or_method" | "method" | "method-wrapper" + ) + { + return Some(Object::Builtin(Rc::new(method_kw("__call__", slot_call)))); + } + let f: fn(&[Object]) -> Result = match name { + "__repr__" => slot_repr, + "__format__" => slot_format, + // Every built-in value type has its own `tp_str` in CPython + // (`int.__str__ is not object.__str__` — enum's ReprEnum wiring + // tests that identity), and `slot_str` already stringifies the + // receiver's native payload per type. + "__str__" => slot_str, + _ => return None, + }; + let static_name = match name { + "__repr__" => "__repr__", + "__str__" => "__str__", + "__format__" => "__format__", + _ => return None, + }; + Some(Object::Builtin(Rc::new(method(static_name, f)))) +} + // ---------- free builtins ---------- fn one<'a>(args: &'a [Object], name: &str) -> Result<&'a Object, RuntimeError> { @@ -1544,6 +1690,77 @@ fn property_deleter(args: &[Object]) -> Result { property_with(args, crate::object::PropertyAttr::Del) } +/// Re-enter the running interpreter to call a Python-level callable from +/// builtin context. Shared by the explicit descriptor-protocol slots +/// (`property.__get__` / `__set__` / `__delete__`), whose accessors are +/// ordinary Python functions. +fn reentrant_call(callable: &Object, args: &[Object]) -> Result { + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| crate::error::runtime_error("no running interpreter"))?; + // SAFETY: the pointer was published by an enclosing VM frame still + // live on this thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + interp.call_object_with_globals(callable, args, &[], &globals) +} + +fn property_self(args: &[Object], op: &str) -> Result, RuntimeError> { + match args.first() { + Some(Object::Property(p)) => Ok(p.clone()), + _ => Err(type_error(format!( + "descriptor '{op}' requires a 'property' object" + ))), + } +} + +/// `property.__get__(self, obj, objtype=None)` — CPython's +/// `property_descr_get`: class access (obj is None) returns the property +/// itself; instance access invokes `fget`. +fn property_dunder_get(args: &[Object]) -> Result { + let p = property_self(args, "__get__")?; + match args.get(1) { + Some(obj) if !matches!(obj, Object::None) => { + if matches!(p.fget, Object::None) { + return Err(crate::error::attribute_error("unreadable attribute")); + } + reentrant_call(&p.fget, &[obj.clone()]) + } + _ => Ok(args[0].clone()), + } +} + +/// `property.__set__(self, obj, value)` — CPython's `property_descr_set`. +fn property_dunder_set(args: &[Object]) -> Result { + let p = property_self(args, "__set__")?; + let (obj, value) = match (args.get(1), args.get(2)) { + (Some(o), Some(v)) => (o.clone(), v.clone()), + _ => return Err(type_error("__set__() takes exactly 3 arguments")), + }; + if matches!(p.fset, Object::None) { + return Err(crate::error::attribute_error( + "property has no setter".to_owned(), + )); + } + reentrant_call(&p.fset, &[obj, value])?; + Ok(Object::None) +} + +/// `property.__delete__(self, obj)` — CPython's deleter slot. +fn property_dunder_delete(args: &[Object]) -> Result { + let p = property_self(args, "__delete__")?; + let obj = args + .get(1) + .cloned() + .ok_or_else(|| type_error("__delete__() takes exactly 2 arguments"))?; + if matches!(p.fdel, Object::None) { + return Err(crate::error::attribute_error( + "property has no deleter".to_owned(), + )); + } + reentrant_call(&p.fdel, &[obj])?; + Ok(Object::None) +} + fn b_getattr(args: &[Object]) -> Result { if args.len() < 2 { return Err(type_error("getattr() requires at least 2 arguments")); @@ -4101,9 +4318,20 @@ pub fn make_super(class: Rc, receiver: Object) -> Obje dict: Rc::new(RefCell::new({ let mut d = DictData::new(); d.insert(DictKey(Object::from_static("__self__")), receiver); + // CPython's `su->obj_type` — the class whose MRO is walked, + // passed as `owner` to descriptor `__get__`s. Also used to + // detect the class-bound form (`su->obj == starttype`), + // where descriptors get a NULL instance (so plain functions + // come back *unbound*: `super().__new__(cls, v)` must not + // prepend a second `cls`). + d.insert( + DictKey(Object::from_static("__obj_type__")), + Object::Type(receiver_class.clone()), + ); d })), native: None, + inline_values: crate::sync::Cell::new(true), }; Object::Instance(Rc::new(inst)) } @@ -4215,10 +4443,28 @@ pub fn class_of(obj: &Object) -> crate::sync::Rc { }, Object::SimpleNamespace(_) => bt.simple_namespace_.clone(), Object::Type(t) => t.metaclass_or_type(), - Object::Function(_) | Object::Builtin(_) => bt.function_.clone(), + Object::Function(_) => bt.function_.clone(), + // Rust-implemented callables are `builtin_function_or_method`, + // distinct from `function`, exactly as in CPython (`type(len)`). + Object::Builtin(_) => bt.builtin_function_.clone(), // A bound method is its own type in CPython (`type(o.m)` is `method`), // which also makes `types.MethodType(func, obj)` construct one. - Object::BoundMethod(_) => bt.method_.clone(), + // Distinguish what the method wraps, as CPython does: + // * Python function -> `method` + // * builtin slot dunder -> `method-wrapper` (`x.__add__`) + // * other builtin callable -> `builtin_function_or_method` + // (`[].append` — bound C methods share the C-function type) + Object::BoundMethod(bm) => match &bm.function { + Object::Builtin(b) => { + let n = b.name.trim_start_matches('.'); + if n.starts_with("__") && n.ends_with("__") { + bt.method_wrapper_.clone() + } else { + bt.builtin_function_.clone() + } + } + _ => bt.method_.clone(), + }, Object::Property(_) => bt.property_.clone(), Object::StaticMethod(_) => bt.staticmethod_.clone(), Object::ClassMethod(_) => bt.classmethod_.clone(), @@ -4230,6 +4476,11 @@ pub fn class_of(obj: &Object) -> crate::sync::Rc { Object::Generator(_) => bt.generator_.clone(), Object::Coroutine(_) => bt.coroutine_.clone(), Object::AsyncGenerator(_) => bt.async_generator_.clone(), + // The transient `asend`/`athrow`/`aclose` awaitables have no + // dedicated singleton type; treat them as plain objects for + // `type()` (their faithful CPython name is still surfaced by + // `repr`/error messages via `Object::type_name`). + Object::AsyncGenAwait(_) => bt.object_.clone(), Object::Module(_) => bt.module_.clone(), Object::Code(_) | Object::Cell(_) | Object::SlotDescriptor(_) | Object::File(_) => { bt.object_.clone() @@ -4310,6 +4561,7 @@ fn object_identity(obj: &Object) -> i64 { Object::Generator(g) => Rc::as_ptr(g) as usize as i64, Object::Coroutine(g) => Rc::as_ptr(g) as usize as i64, Object::AsyncGenerator(g) => Rc::as_ptr(g) as usize as i64, + Object::AsyncGenAwait(a) => Rc::as_ptr(a) as usize as i64, Object::File(f) => Rc::as_ptr(f) as usize as i64, Object::Property(p) => Rc::as_ptr(p) as usize as i64, Object::StaticMethod(m) => Rc::as_ptr(m) as usize as i64, @@ -4389,7 +4641,7 @@ fn b_hash(args: &[Object]) -> Result { /// fall back to a small list of dunder names. We deliberately keep /// this loose because runtime helpers (typing, dataclasses, abc) /// only need it to enumerate user attributes. -fn b_dir(args: &[Object]) -> Result { +pub(crate) fn b_dir(args: &[Object]) -> Result { use std::collections::BTreeSet; let mut names: BTreeSet = BTreeSet::new(); let obj = one(args, "dir")?; diff --git a/crates/weavepy-vm/src/error.rs b/crates/weavepy-vm/src/error.rs index c0b76f4..8340a8f 100644 --- a/crates/weavepy-vm/src/error.rs +++ b/crates/weavepy-vm/src/error.rs @@ -244,8 +244,15 @@ pub fn stop_iteration_with(value: Object) -> RuntimeError { if let Object::Instance(ref inst) = pe.instance { let key = crate::object::DictKey(Object::from_static("value")); inst.dict.borrow_mut().insert(key, value.clone()); + // A bare `return` (value None) raises `StopIteration()` with + // *empty* args, so `str(e)` renders bare and `e.args` is `()` — + // CPython's `gen_return` only packs non-None return values. let args_key = crate::object::DictKey(Object::from_static("args")); - let args = Object::new_tuple(vec![value]); + let args = if matches!(value, Object::None) { + Object::new_tuple(Vec::new()) + } else { + Object::new_tuple(vec![value]) + }; inst.dict.borrow_mut().insert(args_key, args); } RuntimeError::PyException(pe) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 8ae13b7..2619b85 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -79,12 +79,40 @@ struct Frame { /// For class-body frames, names are stored here instead of globals. /// `None` for ordinary function and module frames. class_namespace: Option>>, + /// PEP 3115: when a metaclass `__prepare__` returns a *custom* mapping + /// (not a plain `dict`), class-body `STORE_NAME`/`LOAD_NAME`/`DELETE_NAME` + /// route through this object's `__setitem__`/`__getitem__`/`__delitem__` + /// so the mapping observes every binding (e.g. `enum._EnumDict` detecting + /// members). `None` for the common plain-dict path, which keeps using the + /// fast `class_namespace` `DictData`. + class_namespace_obj: Option, /// Stack of currently-handled exceptions. `PUSH_EXC_INFO` pushes /// onto this; `POP_EXCEPT` pops; `RERAISE 1` re-raises the top. /// Each entry is tagged with the pc just past its handler body /// (the `PUSH_EXC_INFO` arg) so the unwinder can discard handlers an /// exception propagates *out of* (see `handle_exception`). exc_handlers: Vec<(u32, PyException)>, + /// A *generator/coroutine* frame's own handled-exception entries, + /// detached from the interpreter-wide `exc_info_stack` while it is + /// suspended at a `yield`. CPython stores this as the generator's + /// `exc_state`: on resume the entries are pushed back so an + /// `except`/`with` block the generator suspended inside is active + /// again (and `gen.throw` chains its `__context__`); on suspend they + /// are peeled back off so they don't leak into the *resumer's* + /// `sys.exc_info()`. Always empty for ordinary frames (they never + /// suspend). + saved_exc_info: Vec, + /// For an async-generator frame: was the most recent suspension caused + /// by the agen's *own* `yield` (a value for the consumer) rather than an + /// inner `await` passing a suspension's value through? Set by the + /// `YIELD_VALUE` handler from the opcode's arg (the compiler marks + /// async-gen yields with arg 1). Read after the frame re-suspends to + /// decide whether `__anext__`/`SEND` should report `StopIteration(value)` + /// (agen yielded) or yield the value onward (inner-await passthrough). + /// Mirrors CPython's `PyAsyncGenWrappedValue`. `true` by default so a + /// non-async-gen frame (or one that hasn't yielded) keeps the historical + /// "completes with value" behavior. + agen_yielded_value: bool, /// pc *before* the current instruction — used to look up the /// exception handler when an opcode raises. pc: u32, @@ -940,7 +968,10 @@ impl Interpreter { stack: Vec::with_capacity(16), globals, class_namespace: None, + class_namespace_obj: None, exc_handlers: Vec::new(), + saved_exc_info: Vec::new(), + agen_yielded_value: true, pc: 0, py_frame: None, } @@ -1006,6 +1037,15 @@ impl Interpreter { // activation leaves un-popped (see the reconciliation at the // function's exit). let exc_depth_on_entry = self.exc_info_stack.borrow().len(); + // Restore a resumed generator/coroutine's own handled-exception + // entries (detached when it last suspended) so an `except` / + // `with` block it yielded inside is the active handled exception + // again. Empty for ordinary frames and a generator's first run, + // so this is a no-op there. + if !frame.saved_exc_info.is_empty() { + let restored = std::mem::take(&mut frame.saved_exc_info); + self.exc_info_stack.borrow_mut().extend(restored); + } // Distinguish the three ways control can enter a frame here: // // * a fresh ordinary call (pc == 0, not gen code) @@ -1155,18 +1195,33 @@ impl Interpreter { // through a handler — the matching `POP_EXCEPT` may not run, so // `PUSH_EXC_INFO` entries this frame pushed can linger. Left in // place they leak into `sys.exc_info()` and wrongly become the - // implicit `__context__` of the next, unrelated `raise`. Drop - // anything this activation added once it truly completes - // (returned or raised). A *yield* keeps its entries: a generator - // suspended inside a handler must see the same exc state on - // resume. - if !matches!( - result, - Ok(FrameOutcome::Yielded(_)) | Ok(FrameOutcome::StartGenerator) - ) { - let mut stack = self.exc_info_stack.borrow_mut(); - if stack.len() > exc_depth_on_entry { - stack.truncate(exc_depth_on_entry); + // implicit `__context__` of the next, unrelated `raise`. + match &result { + Ok(FrameOutcome::Yielded(_)) => { + // A generator/coroutine suspended. Peel its own + // handled-exception entries back off the interpreter-wide + // stack and stash them on the frame: they must NOT be + // visible to whoever resumes us (CPython swaps the + // generator's `exc_state` out on suspend), yet must be + // restored on the next resume. Without this the + // generator's `except` exception leaks into the resumer's + // `sys.exc_info()`. + let mut stack = self.exc_info_stack.borrow_mut(); + if stack.len() > exc_depth_on_entry { + frame.saved_exc_info = stack.split_off(exc_depth_on_entry); + } + } + Ok(FrameOutcome::StartGenerator) => { + // Bootstrap only creates the suspended object; no user + // code ran, so leave the stack untouched. + } + _ => { + // Returned or raised: this activation is done. Drop any + // entries it left un-popped. + let mut stack = self.exc_info_stack.borrow_mut(); + if stack.len() > exc_depth_on_entry { + stack.truncate(exc_depth_on_entry); + } } } result @@ -1192,6 +1247,28 @@ impl Interpreter { self.frame_stack.borrow_mut().push(existing.clone()); return existing; } + let back = self.frame_stack.borrow().last().cloned(); + let py = self.build_py_frame(frame, back); + // Cache the snapshot on generator-family frames so the next + // resume re-pushes this very object (stable identity). Plain + // function frames run exactly once and are never re-entered, + // so caching them would only waste a clone. + if frame.code.is_generator || frame.code.is_coroutine || frame.code.is_async_generator { + frame.py_frame = Some(py.clone()); + } + self.frame_stack.borrow_mut().push(py.clone()); + py + } + + /// Construct a [`PyFrame`] snapshot for `frame` with the given + /// `back` pointer, *without* touching the interpreter's call stack + /// or caching it on the frame. [`Self::push_py_frame`] uses this for + /// live frames (passing the current stack top as `back`); generator + /// introspection (`gi_frame`/`cr_frame`/`ag_frame`) uses it to + /// materialise the frame of a not-yet-started generator on demand, + /// where `back` is `None` (a suspended/created generator frame has + /// no live caller). + fn build_py_frame(&self, frame: &Frame, back: Option>) -> Rc { let varnames = frame.code.varnames.clone(); let locals_snapshot = Rc::new(RefCell::new(frame.locals.clone())); let cell_names: Vec = frame @@ -1255,8 +1332,7 @@ impl Interpreter { } Object::Dict(Rc::new(RefCell::new(d))) }); - let back = self.frame_stack.borrow().last().cloned(); - let py = Rc::new(PyFrame { + Rc::new(PyFrame { code: frame.code.clone(), globals, builtins: self.builtins.clone(), @@ -1270,16 +1346,7 @@ impl Interpreter { last_line: Cell::new(None), trace_lines: Cell::new(true), trace_opcodes: Cell::new(false), - }); - // Cache the snapshot on generator-family frames so the next - // resume re-pushes this very object (stable identity). Plain - // function frames run exactly once and are never re-entered, - // so caching them would only waste a clone. - if frame.code.is_generator || frame.code.is_coroutine || frame.code.is_async_generator { - frame.py_frame = Some(py.clone()); - } - self.frame_stack.borrow_mut().push(py.clone()); - py + }) } /// Refresh the live-locals mirror on the current Python frame. @@ -1300,7 +1367,68 @@ impl Interpreter { } fn pop_py_frame(&self) { - self.frame_stack.borrow_mut().pop(); + let popped = self.frame_stack.borrow_mut().pop(); + // A generator-family frame that just suspended (yielded) or + // finished is no longer reachable from a live caller. CPython + // reports `gi_frame.f_back is None` whenever the generator is not + // currently executing, so drop the resumer link we set on entry. + // Ordinary function frames keep their `back` (tracebacks chain + // through it); only generator-family frames are re-entered and + // observed while suspended. + if let Some(popped) = popped { + if popped.code.is_generator + || popped.code.is_coroutine + || popped.code.is_async_generator + { + *popped.back.borrow_mut() = None; + } + } + } + + /// The code object backing a generator/coroutine/async-generator + /// (`gi_code`/`cr_code`/`ag_code`), read from its execution frame. + /// `None` while running (the frame is on the call stack) or finished + /// (the frame is gone); the suspended/created case covers the + /// introspection the conformance suite performs. + fn gen_code_object(&self, g: &Rc) -> Object { + match &*g.state.borrow() { + GeneratorState::Created(boxed) | GeneratorState::Suspended(boxed) => boxed + .downcast_ref::() + .map_or(Object::None, |f| Object::Code(f.code.clone())), + GeneratorState::Running | GeneratorState::Finished => Object::None, + } + } + + /// The stable Python-visible frame of a generator/coroutine/async + /// generator (`gi_frame`/`cr_frame`/`ag_frame`). Materialised on + /// demand for a not-yet-started generator (with `f_back is None`), + /// then reused across resumes for a stable identity (CPython keeps a + /// single `gi_frame` alive for the generator's lifetime). `None` once + /// the generator has finished. + fn gen_py_frame(&self, g: &Rc) -> Object { + let mut state = g.state.borrow_mut(); + match &mut *state { + GeneratorState::Created(boxed) | GeneratorState::Suspended(boxed) => { + match boxed.downcast_mut::() { + Some(frame) => { + if let Some(py) = frame.py_frame.clone() { + return Object::Frame(py); + } + // Not yet entered: build the frame snapshot now so + // `gi_frame` is observable before the first + // `next()`. `back` is None — a created/suspended + // generator frame has no live caller. + let py = self.build_py_frame(frame, None); + frame.py_frame = Some(py.clone()); + Object::Frame(py) + } + None => Object::None, + } + } + // Running: the frame is live on the interpreter call stack, + // not in the box. Finished: the frame has been dropped. + GeneratorState::Running | GeneratorState::Finished => Object::None, + } } // =========================================================== @@ -1689,10 +1817,14 @@ impl Interpreter { } OpCode::LoadName => { let name = self.name_at(&frame.code, ins.arg)?; - let from_ns = frame - .class_namespace - .as_ref() - .and_then(|ns| ns.borrow().get(&DictKey(Object::from_str(&name))).cloned()); + let from_ns = match &frame.class_namespace_obj { + // PEP 3115 custom namespace: read it before globals. + Some(ns_obj) => self.class_ns_load(ns_obj, &name), + None => frame + .class_namespace + .as_ref() + .and_then(|ns| ns.borrow().get(&DictKey(Object::from_str(&name))).cloned()), + }; let v = match from_ns { Some(v) => v, None => self.lookup_global_or_builtin(&frame.globals, &name)?, @@ -1738,7 +1870,12 @@ impl Interpreter { OpCode::StoreName => { let v = frame.pop()?; let name = self.name_at(&frame.code, ins.arg)?; - if let Some(ns) = &frame.class_namespace { + if let Some(ns_obj) = frame.class_namespace_obj.clone() { + // PEP 3115: a custom class namespace observes the binding + // through its `__setitem__` (e.g. `enum._EnumDict`). + let g = frame.globals.clone(); + self.class_ns_store(&ns_obj, &name, v, &g)?; + } else if let Some(ns) = &frame.class_namespace { ns.borrow_mut().insert(DictKey(Object::from_str(name)), v); } else { frame @@ -1757,6 +1894,11 @@ impl Interpreter { } OpCode::DeleteName => { let name = self.name_at(&frame.code, ins.arg)?; + if let Some(ns_obj) = frame.class_namespace_obj.clone() { + let g = frame.globals.clone(); + self.class_ns_delete(&ns_obj, &name, &g)?; + return Ok(StepOutcome::Continue); + } if let Some(ns) = &frame.class_namespace { if ns .borrow_mut() @@ -2002,7 +2144,9 @@ impl Interpreter { OpCode::ContainsOp => { let container = frame.pop()?; let item = frame.pop()?; - let found = if let Some(method) = instance_method(&container, "__contains__") { + let found = if let Some(method) = instance_method(&container, "__contains__") + .or_else(|| metaclass_method(&container, "__contains__")) + { let r = self.call( &method, std::slice::from_ref(&item), @@ -2826,6 +2970,11 @@ impl Interpreter { } OpCode::YieldValue => { let v = frame.pop()?; + // arg 1 = the async generator's own `yield` (a consumer + // value); arg 0 = the `await`/`yield from` dance passing an + // inner suspension's value through. `__anext__`/`SEND` read + // this after the frame re-suspends. See `Frame.agen_yielded_value`. + frame.agen_yielded_value = ins.arg == 1; return Ok(StepOutcome::Yield(v)); } OpCode::ReturnGenerator => { @@ -2855,20 +3004,27 @@ impl Interpreter { let result = match &iter { Object::Generator(g) | Object::Coroutine(g) => self.generator_send(g, value), Object::AsyncGenerator(g) => { - // Async-generator semantics under SEND - // (simple cooperative model — no support for - // `await` *inside* the agen body, which would - // require CPython's intermediate-value - // passthrough machinery): - // * `agen` yields `v` -> asend completes - // with value `v` (i.e. emulate - // `StopIteration(v)` so SEND short- - // circuits to `END_SEND`). - // * `agen` returns -> raise - // `StopAsyncIteration`. + // Async-generator semantics under SEND: + // * `agen` yields `v` via its own `yield` -> the + // asend completes with value `v` (emulate + // `StopIteration(v)` so SEND short-circuits to + // `END_SEND`; the async-for/await sees the item). + // * `agen` suspends on an inner `await` that yielded + // `v` -> pass `v` through (Ok), so the surrounding + // coroutine's following YIELD_VALUE re-suspends it + // and `v` reaches the event loop. The next SEND + // resumes the inner await. This is CPython's + // intermediate-value passthrough. + // * `agen` returns -> raise `StopAsyncIteration`. // * `agen` raises -> propagate. match self.generator_send(g, value) { - Ok(v) => Err(stop_iteration_with(v)), + Ok(v) => { + if Self::agen_yielded_a_value(g) { + Err(stop_iteration_with(v)) + } else { + Ok(v) + } + } Err(RuntimeError::PyException(exc)) if exc.type_name() == "StopIteration" => { @@ -2877,6 +3033,14 @@ impl Interpreter { other => other, } } + // Driving `await agen.asend()/athrow()/aclose()`: apply + // the deferred op. `step_agen_await` reports completion as + // `StopIteration(value)` and lets `StopAsyncIteration`/ + // exceptions propagate, exactly the shape SEND's result + // handler below expects. The first drive uses the op's + // fixed payload; the popped `value` is forwarded so a + // re-drive (after an inner-await passthrough) resumes it. + Object::AsyncGenAwait(a) => self.step_agen_await(a, value), Object::Iter(_) => { if !matches!(value, Object::None) { return Err(type_error( @@ -3145,32 +3309,115 @@ impl Interpreter { } /// If the most-recent handled exception is still active when - /// `raise X` runs without a `from` clause, attach it as the new - /// exception's `__context__` so chained tracebacks render - /// `During handling of the above exception, another exception - /// occurred:`. Mirrors PEP 3134 / CPython. + /// `raise X` runs, attach it as the new exception's `__context__` + /// so chained tracebacks render `During handling of the above + /// exception, another exception occurred:`. Mirrors PEP 3134 / + /// CPython `_PyErr_SetObject`. + /// + /// The implicit context is set even when an explicit `from` cause is + /// present: `raise X from Y` sets *both* `__cause__ = Y` and + /// `__context__ = `. The `from` clause only flips + /// `__suppress_context__` (handled in `sync_exc_attrs`), which + /// governs *display*, not whether `__context__` exists. fn attach_implicit_context(&self, exc: &mut PyException) { - if exc.cause.is_some() { - return; - } let stack = self.exc_info_stack.borrow(); let Some(ctx) = stack.last() else { return; }; + let Object::Instance(ctx_inst) = &ctx.instance else { + return; + }; + let Object::Instance(exc_inst) = &exc.instance else { + return; + }; // Don't self-reference if user code re-raises through `raise` // (the existing context-handler is the same exception). - if Rc::as_ptr(&match &ctx.instance { - Object::Instance(i) => i.clone(), - _ => return, - }) == Rc::as_ptr(&match &exc.instance { - Object::Instance(i) => i.clone(), - _ => return, - }) { + if Rc::ptr_eq(ctx_inst, exc_inst) { return; } + // bpo-27122: avoid building an implicit-context *cycle*. If the + // exception we're about to chain (`ctx`) already reaches the new + // exception via its own `__context__` chain, sever that link + // first so traceback walkers can't loop. CPython does the same + // before assigning the new `__context__`. + Self::break_implicit_context_cycle(ctx_inst, exc_inst); exc.context = Some(Box::new(ctx.clone())); } + /// Chain an exception injected by `gen.throw()` to whatever the + /// generator was handling at its suspend point. CPython's + /// `_gen_throw` calls `_PyErr_SetObject`, which sets the new + /// exception's `__context__` to the generator's current `exc_state`. + /// The suspended generator's active handled exception is the top of + /// the entries we detached on suspend (`frame.saved_exc_info`). + fn chain_thrown_context(exc: &mut PyException, frame: &Frame) { + let Some(active) = frame.saved_exc_info.last() else { + return; + }; + let Object::Instance(active_inst) = &active.instance else { + return; + }; + let Object::Instance(exc_inst) = &exc.instance else { + return; + }; + if Rc::ptr_eq(active_inst, exc_inst) { + return; + } + Self::break_implicit_context_cycle(active_inst, exc_inst); + exc.context = Some(Box::new(active.clone())); + // Mirror onto the instance dict so Python's `e.__context__` sees + // it (the throw path doesn't go through `RAISE_VARARGS`, which is + // where `sync_exc_attrs` normally runs). + Self::sync_exc_attrs(exc); + } + + /// Walk `holder`'s `__context__` chain (instance dicts, the canonical + /// chain Python walks) and, if it reaches `target`, drop the link + /// that points at `target`. Bounded by a Floyd tortoise/hare so a + /// pre-existing cycle can't hang us. + fn break_implicit_context_cycle( + holder: &Rc, + target: &Rc, + ) { + fn ctx_of(inst: &Rc) -> Option> { + let dict = inst.dict.borrow(); + match dict.get(&DictKey(Object::from_static("__context__"))) { + Some(Object::Instance(i)) => Some(i.clone()), + _ => None, + } + } + let mut slow = holder.clone(); + let mut fast = holder.clone(); + loop { + // Advance `fast` two steps, severing a link to `target`. + let Some(next) = ctx_of(&fast) else { return }; + if Rc::ptr_eq(&next, target) { + fast.dict.borrow_mut().insert( + DictKey(Object::from_static("__context__")), + Object::None, + ); + return; + } + fast = next; + let Some(next) = ctx_of(&fast) else { return }; + if Rc::ptr_eq(&next, target) { + fast.dict.borrow_mut().insert( + DictKey(Object::from_static("__context__")), + Object::None, + ); + return; + } + fast = next; + // Advance `slow` one step; if it meets `fast` we're on a + // pre-existing cycle with no `target` link — stop. + let Some(next_slow) = ctx_of(&slow) else { return }; + slow = next_slow; + if Rc::ptr_eq(&slow, &fast) { + return; + } + } + } + /// Mirror the `cause` / `context` chain onto the instance dict so /// Python code accessing `e.__cause__` / `e.__context__` sees /// the canonical values. Called right before raising. @@ -3325,6 +3572,29 @@ impl Interpreter { } fn load_attr(&mut self, obj: &Object, name: &str) -> Result { + let result = self.load_attr_inner(obj, name); + // CPython's `object.__getattribute__` final step: after the + // per-variant lookup misses, resolve the name through + // `type(obj)`'s MRO slot-wrapper table and bind it + // (`object().__str__`, `(1).__neg__`, `len.__call__`, + // `prop.__set__`, …). Types are excluded — `load_attr_type` + // already consults the slot table itself — as are dunder misses + // raised *from within* a successful lookup (we only rescue + // genuine AttributeErrors for `name` itself). + if let Err(err) = &result { + if !matches!(obj, Object::Type(_)) && self.is_attribute_error(err) { + if let Some(f) = builtin_slot_wrapper(&crate::builtins::class_of(obj), name) { + return Ok(Object::BoundMethod(Rc::new(BoundMethod { + receiver: obj.clone(), + function: f, + }))); + } + } + } + result + } + + fn load_attr_inner(&mut self, obj: &Object, name: &str) -> Result { // `__class__` is readable on *every* object and returns its // type. Instances and classes keep their dedicated handling // below (which honours `__class__` reassignment and the @@ -3335,6 +3605,16 @@ impl Interpreter { if name == "__class__" && !matches!(obj, Object::Instance(_) | Object::Type(_)) { return Ok(Object::Type(crate::builtins::class_of(obj))); } + // `__new__` is resolvable on *every* object via its type's MRO + // (it lives on `object` as a staticmethod). Our primitive arms below + // don't carry it, so resolve `obj.__new__` uniformly through + // `type(obj)` — `None.__new__`, `(1).__new__`, `"".__new__` all + // return the same function `type(obj).__new__` does. `enum`'s + // `_find_new_` builds the set `{None, None.__new__, object.__new__, + // ...}` and relies on this. + if name == "__new__" && !matches!(obj, Object::Instance(_) | Object::Type(_)) { + return self.load_attr_type(&crate::builtins::class_of(obj), "__new__"); + } match obj { Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { let allowed: &[&str] = match obj { @@ -3349,8 +3629,65 @@ impl Interpreter { let method = make_gen_method(name, obj); return Ok(method); } + // Frame/code introspection. Each flavour uses its own + // attribute prefix (`gi_` generator, `cr_` coroutine, + // `ag_` async generator) over the same underlying state. + let prefix = match obj { + Object::Coroutine(_) => "cr_", + Object::AsyncGenerator(_) => "ag_", + _ => "gi_", + }; + if let Some(suffix) = name.strip_prefix(prefix) { + match suffix { + // The code object backing the generator. + "code" => return Ok(self.gen_code_object(g)), + // Currently executing (illegal to re-enter)? + "running" => { + return Ok(Object::Bool(matches!( + &*g.state.borrow(), + GeneratorState::Running + ))) + } + // The (stable) Python-visible frame, or None once + // the generator has finished. + "frame" => return Ok(self.gen_py_frame(g)), + // PEP 580-era `gi_suspended` (generators only). + "suspended" if prefix == "gi_" => { + return Ok(Object::Bool(matches!( + &*g.state.borrow(), + GeneratorState::Suspended(_) + ))) + } + // The sub-iterator a `yield from` / `await` is + // currently delegating to. We don't track the + // active delegate yet, so report "not delegating". + "yieldfrom" if prefix == "gi_" => return Ok(Object::None), + "await" if prefix == "cr_" || prefix == "ag_" => { + return Ok(Object::None) + } + _ => {} + } + } + // `object.__reduce_ex__` exists on every CPython object; + // for generator-family objects invoking it raises + // `TypeError` ("cannot pickle ..."). `copy.copy` and + // `pickle` both rely on getting TypeError (not + // AttributeError) here. + if matches!(name, "__reduce__" | "__reduce_ex__") { + let type_name = obj.type_name().to_owned(); + return Ok(Object::Builtin(Rc::new(BuiltinFn { + name: "__reduce_ex__", + call: Box::new(move |_args| { + Err(type_error(format!( + "cannot pickle '{type_name}' object" + ))) + }), + call_kw: None, + }))); + } match name { - "__name__" | "__qualname__" => Ok(Object::from_str(&g.name)), + "__name__" => Ok(Object::from_str(g.name.borrow().clone())), + "__qualname__" => Ok(Object::from_str(g.qualname.borrow().clone())), _ => Err(attribute_error(format!( "'{}' object has no attribute '{}'", obj.type_name(), @@ -3358,6 +3695,21 @@ impl Interpreter { ))), } } + // The deferred `asend`/`athrow`/`aclose`/`__anext__` awaitable + // is its own iterator (CPython's `async_generator_asend`): + // `__next__`/`send` drive it one step, `__iter__`/`__await__` + // return it. The drive routes through `gen_method_send`, which + // dispatches `AsyncGenAwait` to `step_agen_await`. + Object::AsyncGenAwait(_) => match name { + "__next__" | "send" | "__iter__" | "__await__" | "throw" | "close" => { + Ok(make_gen_method(name, obj)) + } + _ => Err(attribute_error(format!( + "'{}' object has no attribute '{}'", + obj.type_name(), + name + ))), + }, Object::Instance(inst) => self.load_attr_instance(inst, obj, name), Object::Type(ty) => self.load_attr_type(ty, name), Object::Property(p) => match name { @@ -3889,12 +4241,33 @@ impl Interpreter { if name != "__self__" { if let Some(receiver) = super_receiver { if let Some(v) = inst.class.lookup(name) { - let owner = match &receiver { - Object::Type(t) => Object::Type(t.clone()), - Object::Instance(i) => Object::Type(i.class.clone()), + // CPython passes `su->obj_type` (the MRO-walk start + // class) as `owner`, and a NULL instance when + // `su->obj == starttype` — the class-bound form + // `super(C, cls)` inside `__new__`/classmethods — + // so plain functions come back unbound while + // classmethods still bind to the class. + let obj_type = inst + .dict + .borrow() + .get(&DictKey(Object::from_static("__obj_type__"))) + .cloned(); + let owner = match (&obj_type, &receiver) { + (Some(Object::Type(t)), _) => Object::Type(t.clone()), + (_, Object::Type(t)) => Object::Type(t.clone()), + (_, Object::Instance(i)) => Object::Type(i.class.clone()), _ => Object::Type(inst.class.clone()), }; - return self.descriptor_get(&v, &receiver, &owner); + let instance_for_get = match (&receiver, &owner) { + (Object::Type(r), Object::Type(o)) if Rc::ptr_eq(r, o) => { + Object::None + } + _ => receiver.clone(), + }; + let bound = self.descriptor_get(&v, &instance_for_get, &owner)?; + // `classmethod.__get__(NULL, starttype)` binds to the + // class; plain functions return themselves unbound. + return Ok(bound); } // The MRO beyond the starting class reaches a built-in base // (`dict`, `list`, …) whose methods aren't stored on the type @@ -4017,6 +4390,20 @@ impl Interpreter { /// classmethods bind to the class, plain functions stay /// unbound). fn load_attr_type(&mut self, ty: &Rc, name: &str) -> Result { + // (0) `__name__`/`__qualname__` are `type` getset descriptors in + // CPython — they never resolve through the MRO or the metaclass. + // Honour an *own*-dict entry (static class bodies record one, and + // assignment writes one), otherwise synthesize from the type + // itself. Without the own-only rule, a dynamically created class + // (`type('D', (A,), {})`, enum's functional API) would inherit + // its base's or metaclass's `__name__`. + if name == "__name__" || name == "__qualname__" { + if let Some(v) = ty.dict.borrow().get(&DictKey(Object::from_str(name))) { + return Ok(v.clone()); + } + return Ok(Object::from_str(&ty.name)); + } + let meta = ty.metaclass_or_type(); let owner = Object::Type(ty.clone()); let self_as_obj = Object::Type(ty.clone()); @@ -4061,6 +4448,7 @@ impl Interpreter { } "__class__" => return Ok(Object::Type(meta)), "__dict__" => return Ok(Object::Dict(ty.dict.clone())), + "__flags__" => return Ok(Object::Int(ty.flags_bits())), "__subclasses__" => { // `type.__subclasses__` is a bound method; the actual // work is done in `Interpreter::call` via the sentinel @@ -4098,6 +4486,24 @@ impl Interpreter { _ => {} } + // (4b) Generator/coroutine/async-generator methods are reachable as + // *unbound* functions through the type: `type(agen).__anext__(agen)`, + // which test_asyncgen's `py_anext` relies on. Synthesize the unbound + // builtin; `Interpreter::call` routes the `.u.*` sentinel using + // `args[0]` as the receiver. + if let Some(internal) = unbound_gen_method_sentinel(ty, name) { + return Ok(Object::Builtin(Rc::new(BuiltinFn { + name: internal, + call: Box::new(|_args| { + Err(RuntimeError::Internal( + "unbound generator method must be dispatched via Interpreter::call" + .to_owned(), + )) + }), + call_kw: None, + }))); + } + // (5) Built-in class methods not stored in ``ty.dict``: most // CPython classmethods/staticmethods (``str.maketrans``, // ``bytes.fromhex``, ``int.from_bytes``, ``dict.fromkeys``, @@ -4108,15 +4514,47 @@ impl Interpreter { return Ok(b); } - // (6) Unbound instance methods reached via the type: `str.upper`, - // `float.hex`, `list.append`, … CPython exposes every instance method - // on its type as a function taking `self` explicitly. The builtins - // already treat `args[0]` as the receiver, so we hand back the raw - // function object (unbound) rather than binding it to the class. - if let Some(b) = crate::builtins::unbound_method(&ty.name, name) { + // (6) Slot-wrapper dunders and unbound instance methods reached via + // the type: `str.upper`, `list.append`, `int.__add__`, and the + // object-protocol dunders `object.__repr__` / `int.__str__` / + // `str.__format__`. CPython exposes every instance method on its type + // as a function taking `self` explicitly (the builtins already treat + // `args[0]` as the receiver), and stores identity-stable slot wrappers + // in each type's `tp_dict`. [`builtin_slot_wrapper`] reproduces that: + // it walks the MRO so a subclass resolves to the *defining* built-in + // base's wrapper, and caches per `(type, name)` so repeated access + // returns the same object — identity the `enum` bootstrap relies on. + if let Some(b) = builtin_slot_wrapper(ty, name) { return Ok(b); } + // (7) `object.__dir__` / `type.__dir__` reached as an unbound + // method (`object.__dir__(self)` — enum's `Enum.__dir__` uses it + // to seed the interesting-name set). The default implementation + // is the same namespace walk `dir()` performs. + if name == "__dir__" { + return Ok(Object::Builtin(Rc::new(BuiltinFn { + name: "__dir__", + call: Box::new(crate::builtins::b_dir), + call_kw: None, + }))); + } + + // `type.mro()` — bound method on every class returning a copy + // of the MRO list (CPython exposes it as a method of `type`). + if name == "mro" { + let t = ty.clone(); + return Ok(Object::Builtin(Rc::new(BuiltinFn { + name: "mro", + call: Box::new(move |_args| { + Ok(Object::new_list( + t.mro.borrow().iter().cloned().map(Object::Type).collect(), + )) + }), + call_kw: None, + }))); + } + Err(attribute_error(format!( "type object '{}' has no attribute '{}'", ty.name, name @@ -4634,6 +5072,21 @@ impl Interpreter { Err(unsupported_format_string(value)) }; } + // `format(SomeClass)` — a class consults its *metaclass* + // `__format__`, defaulting to `object.__format__` (str(cls), which + // itself dispatches the metaclass `__str__`/`__repr__`). + if let Object::Type(_) = value { + if let Some(method) = metaclass_method(value, "__format__") { + let r = self.call(&method, &[Object::from_str(spec)], &[], globals)?; + return Ok(r.to_str()); + } + let s = self.stringify(value, globals)?; + return if spec.is_empty() { + Ok(s) + } else { + Err(unsupported_format_string(value)) + }; + } format_via_spec(value, spec) } @@ -4660,7 +5113,11 @@ impl Interpreter { v: &Object, globals: &Rc>, ) -> Result { - if let Some(method) = instance_method(v, "__len__") { + // `len(x)` is `type(x).__len__(x)` — for an *instance* that's the + // class's method; for a *class* it's the metaclass's + // (`len(SomeEnum)` → `EnumType.__len__`). + let method = instance_method(v, "__len__").or_else(|| metaclass_method(v, "__len__")); + if let Some(method) = method { let r = self.call(&method, &[], &[], globals)?; return match r { Object::Int(i) => Ok(Object::Int(i)), @@ -5530,15 +5987,55 @@ impl Interpreter { } } + /// Instantiate one of `_seqtools`'s lazy iterator classes + /// (`_FilterIter` / `_MapIter` / `_ZipIter`). Returns `Ok(None)` only + /// if the frozen helper module is somehow unavailable, letting the + /// caller fall back; in practice it is always frozen in. + fn make_seqtools_iter( + &mut self, + class_name: &'static str, + args: &[Object], + globals: &Rc>, + ) -> Result, RuntimeError> { + let module = match self.do_import("_seqtools", &Object::None, 0, globals) { + Ok(m) => m, + Err(_) => return Ok(None), + }; + let cls = match &module { + Object::Module(m) => m + .dict + .borrow() + .get(&DictKey(Object::from_static(class_name))) + .cloned(), + _ => None, + }; + match cls { + Some(cls) => Ok(Some(self.call(&cls, args, &[], globals)?)), + None => Ok(None), + } + } + /// `map(func, *iterables)` — VM-aware (the plain builtin can't call - /// back into the interpreter). Evaluated eagerly into an iterator so - /// generators and `next()` both work (RFC 0033). Stops at the - /// shortest iterable, matching CPython. + /// back into the interpreter). + /// + /// When any input is a VM-driven iterable (generator, instance with + /// `__next__`, …) the result is a *lazy* `_seqtools._MapIter` + /// (CPython's `map` object): `func` runs on demand, so mapping over + /// an unbounded source works. For plain native containers we keep + /// the eager fast path — observably equivalent for finite pure + /// inputs, and crucially the returned native iterator stays + /// consumable by native builtins (e.g. `dict.fromkeys(map(...))`). fn do_map_call( &mut self, args: &[Object], globals: &Rc>, ) -> Result { + if args[1..].iter().any(object_needs_vm_iter) { + return match self.make_seqtools_iter("_MapIter", args, globals)? { + Some(it) => Ok(it), + None => Err(runtime_error("internal: _seqtools._MapIter unavailable")), + }; + } let func = args[0].clone(); let mut cols: Vec> = Vec::with_capacity(args.len() - 1); for it in &args[1..] { @@ -5556,12 +6053,22 @@ impl Interpreter { /// `filter(func_or_None, iterable)` — VM-aware. `None` keeps truthy /// items; otherwise an item is kept when `func(item)` is truthy. - /// Returns an iterator (RFC 0033). + /// Lazy (`_seqtools._FilterIter`) when the input is a VM-driven + /// iterable so filtering an unbounded source terminates; eager + /// native iterator otherwise (see [`Self::do_map_call`]). fn do_filter_call( &mut self, args: &[Object], globals: &Rc>, ) -> Result { + if object_needs_vm_iter(&args[1]) { + return match self.make_seqtools_iter("_FilterIter", args, globals)? { + Some(it) => Ok(it), + None => Err(runtime_error( + "internal: _seqtools._FilterIter unavailable", + )), + }; + } let func = args[0].clone(); let use_pred = !matches!(func, Object::None); let items = self.collect_iterable(&args[1], globals)?; @@ -5688,6 +6195,21 @@ impl Interpreter { ))); } } + // Lazy (CPython's `zip` object) when any input is VM-driven: no + // iterable is pre-materialised, so `zip(count(), count())` + // constructs instantly and pulls one tuple per `next()`. + // `_ZipIter` also carries the strict-mode mismatch diagnostics. + if args.iter().any(object_needs_vm_iter) { + let mut ctor_args = Vec::with_capacity(args.len() + 1); + ctor_args.push(Object::Bool(strict)); + ctor_args.extend_from_slice(args); + return match self.make_seqtools_iter("_ZipIter", &ctor_args, globals)? { + Some(it) => Ok(it), + None => Err(runtime_error("internal: _seqtools._ZipIter unavailable")), + }; + } + // Eager fast path for native finite containers; the result is a + // native iterator that plain builtins can consume directly. if args.is_empty() { return Ok(Object::new_list(Vec::new())); } @@ -5726,7 +6248,8 @@ impl Interpreter { } } } - return Ok(Object::new_list(out)); + let it = Object::new_list(out).make_iter()?; + return Ok(Object::Iter(Rc::new(RefCell::new(it)))); } } } @@ -6164,6 +6687,140 @@ impl Interpreter { /// reverse: a user `__reversed__`, or the legacy sequence protocol /// (`__len__` + `__getitem__`) when no `__reversed__` exists. Returns /// an iterator over the reversed items. + /// `dict.update([other], **kwargs)` with CPython's full protocol: + /// `other` may be a dict / mappingproxy (fast path), an arbitrary + /// mapping (defined by having a callable `keys`), or an iterable of + /// 2-element key/value sequences; keyword args are merged last. + fn do_dict_update_call( + &mut self, + args: &[Object], + kwargs: &[(String, Object)], + globals: &Rc>, + ) -> Result { + let receiver = args.first().cloned(); + let Some(receiver @ Object::Dict(_)) = receiver else { + return Err(type_error("update() requires a 'dict' receiver")); + }; + if args.len() > 2 { + return Err(type_error(format!( + "update expected at most 1 argument, got {}", + args.len() - 1 + ))); + } + if let Some(other) = args.get(1) { + self.dict_merge_from(&receiver, other, globals)?; + } + for (k, v) in kwargs { + self.store_subscr(&receiver, &Object::from_str(k.as_str()), v.clone(), globals)?; + } + Ok(Object::None) + } + + fn dict_merge_from( + &mut self, + dst: &Object, + other: &Object, + globals: &Rc>, + ) -> Result<(), RuntimeError> { + // Fast paths: snapshot first — source and destination may alias + // (`d.update(d)`), and GilCell forbids overlapping borrows. + let native_entries = match other { + Object::Dict(o) => Some( + o.borrow() + .iter() + .map(|(k, v)| (k.0.clone(), v.clone())) + .collect::>(), + ), + Object::MappingProxy(o) => Some( + o.borrow() + .iter() + .map(|(k, v)| (k.0.clone(), v.clone())) + .collect::>(), + ), + // dict subclass instance without custom `keys`/`__getitem__` + // still snapshots natively only when it *is* dict-backed; a + // subclass overriding `keys` must go through the protocol, so + // don't shortcut instances here. + _ => None, + }; + if let Some(entries) = native_entries { + for (k, v) in entries { + self.store_subscr(dst, &k, v, globals)?; + } + return Ok(()); + } + // Mapping protocol: `if hasattr(other, "keys")`. + let keys_fn = self.load_attr(other, "keys").ok(); + if let Some(keys_fn) = keys_fn { + let keys = self.call(&keys_fn, &[], &[], globals)?; + let it = self.make_iter(&keys, globals)?; + while let Some(k) = self.iter_next(&it, globals)? { + let v = self.subscr_via_protocol(other, &k, globals)?; + self.store_subscr(dst, &k, v, globals)?; + } + return Ok(()); + } + // Iterable of key/value pairs. + let it = self.make_iter(other, globals).map_err(|e| { + if is_type_error(&e) { + type_error(format!( + "'{}' object is not iterable", + other.type_name_owned() + )) + } else { + e + } + })?; + let mut idx: usize = 0; + while let Some(pair) = self.iter_next(&it, globals)? { + let inner = self.make_iter(&pair, globals).map_err(|e| { + if is_type_error(&e) { + type_error(format!( + "cannot convert dictionary update sequence element #{idx} to a sequence" + )) + } else { + e + } + })?; + // CPython materializes each element (PySequence_Fast) and + // reports its real length when it isn't exactly 2. + let mut kv: Vec = Vec::with_capacity(2); + let mut n: usize = 0; + while let Some(x) = self.iter_next(&inner, globals)? { + if kv.len() < 2 { + kv.push(x); + } + n += 1; + } + if n != 2 { + return Err(crate::error::value_error(format!( + "dictionary update sequence element #{idx} has length {n}; 2 is required" + ))); + } + let v = kv.pop().expect("len 2"); + let k = kv.pop().expect("len 2"); + self.store_subscr(dst, &k, v, globals)?; + idx += 1; + } + Ok(()) + } + + /// `other[key]` honouring a user `__getitem__` before the native + /// subscript (the merge protocol must call overridden lookups). + fn subscr_via_protocol( + &mut self, + container: &Object, + key: &Object, + globals: &Rc>, + ) -> Result { + if matches!(container, Object::Instance(_)) { + if let Some(m) = instance_method(container, "__getitem__") { + return self.call(&m, &[key.clone()], &[], globals); + } + } + self.binary_subscr(container, key) + } + fn do_reversed_call( &mut self, args: &[Object], @@ -6172,6 +6829,11 @@ impl Interpreter { let obj = args .first() .ok_or_else(|| type_error("reversed() missing required argument"))?; + // `reversed(SomeEnum)` — the metaclass's `__reversed__` bound to + // the class (CPython: `type(x).__reversed__(x)`). + if let Some(method) = metaclass_method(obj, "__reversed__") { + return self.call(&method, &[], &[], globals); + } if let Object::Instance(inst) = obj { if let Some(method) = instance_method(obj, "__reversed__") { return self.call(&method, &[], &[], globals); @@ -6288,6 +6950,14 @@ impl Interpreter { } return self.repr_of(v, globals); } + // `str(SomeClass)` consults the metaclass (`EnumType.__str__`). + if let Object::Type(_) = v { + if let Some(method) = metaclass_method(v, "__str__") { + let r = self.call(&method, &[], &[], globals)?; + return Ok(r.to_str()); + } + return self.repr_of(v, globals); + } if let Object::Long(b) = v { crate::builtins::long_str_limit_check(b)?; } @@ -6312,6 +6982,14 @@ impl Interpreter { return self.repr_of(&native, globals); } } + // `repr(SomeClass)` consults the metaclass (`EnumType.__repr__` + // renders `` instead of ``). + if let Object::Type(_) = v { + if let Some(method) = metaclass_method(v, "__repr__") { + let r = self.call(&method, &[], &[], globals)?; + return Ok(r.to_str()); + } + } if let Object::Long(b) = v { crate::builtins::long_str_limit_check(b)?; } @@ -6392,6 +7070,12 @@ impl Interpreter { ) -> Result { match v { Object::Generator(_) | Object::Iter(_) => Ok(v.clone()), + // `iter(SomeClass)` — `type(x).__iter__(x)` where `type(x)` is + // the metaclass (`iter(SomeEnum)` → `EnumType.__iter__`). + Object::Type(_) if metaclass_method(v, "__iter__").is_some() => { + let method = metaclass_method(v, "__iter__").expect("checked"); + self.call(&method, &[], &[], globals) + } Object::Instance(_) => { if let Some(method) = instance_method(v, "__iter__") { let result = self.call(&method, &[], &[], globals)?; @@ -6515,6 +7199,9 @@ impl Interpreter { // already drivable via SEND; treat it as its own // awaitable so the surrounding await-dance can run. Object::Coroutine(_) | Object::Generator(_) | Object::AsyncGenerator(_) => Ok(value), + // The deferred `asend`/`athrow`/`aclose` awaitable is already a + // drivable awaitable (SEND applies the op via `step_agen_await`). + Object::AsyncGenAwait(_) => Ok(value), Object::Instance(_) => { if let Some(method) = instance_method(&value, "__await__") { let it = self.call(&method, &[], &[], &fallback_globals())?; @@ -6695,6 +7382,13 @@ impl Interpreter { receiver: &Object, value: Object, ) -> Result { + // Driving the deferred `asend`/`athrow`/`aclose` awaitable directly + // (e.g. an event loop calling `.send(None)` on it): apply the op. + // `value` is forwarded so a re-drive (after the agen suspended on an + // inner `await`) resumes that await with the sent value. + if let Object::AsyncGenAwait(a) = receiver { + return self.step_agen_await(a, value); + } let (g, is_async_gen) = match receiver { Object::Generator(g) | Object::Coroutine(g) => (g.clone(), false), Object::AsyncGenerator(g) => (g.clone(), true), @@ -6715,6 +7409,183 @@ impl Interpreter { } } + /// Drive an [`AsyncGenAwait`] (the awaitable behind `agen.asend()` / + /// `.athrow()` / `.aclose()`) one step. In WeavePy's cooperative async + /// model the underlying op completes synchronously, so a drive always + /// *finishes* the await: success is reported the way the await/SEND + /// machinery expects a coroutine to finish — as `StopIteration(value)` + /// — while `StopAsyncIteration` (agen exhausted) and any real exception + /// from the agen body propagate verbatim. A second drive of an + /// already-consumed awaitable raises bare `StopIteration` (exhausted + /// iterator), matching CPython's single-shot `async_generator_asend`. + /// `asend_obj.throw(exc)` — CPython's `async_gen_asend_throw`: forward + /// the exception into the suspended agen, then report the outcome with + /// the same completion protocol as a normal drive (own-yield completes + /// as `StopIteration(value)`, inner-await passthrough stays pending). + fn agen_await_throw( + &mut self, + a: &Rc, + args: &[Object], + ) -> Result { + use crate::object::AgenAwaitKind; + // CPython has two awaitable types with distinct reuse messages: + // `async_generator_asend` vs `async_generator_athrow` (which + // backs both `athrow(...)` and `aclose()`). + let is_close = matches!(a.kind, AgenAwaitKind::Close); + if a.consumed.get() { + return Err(Self::agen_await_reuse_error(a.kind)); + } + if !a.started.get() { + if let Some(err) = Self::agen_await_running_guard(a) { + return Err(err); + } + } + a.started.set(true); + let agen = a.agen.clone(); + let outcome = self.gen_method_throw(&agen, args); + match outcome { + Ok(value) => { + if let Object::AsyncGenerator(g) = &a.agen { + if !Self::agen_yielded_a_value(g) { + return Ok(value); + } + } + a.consumed.set(true); + if is_close { + // aclose-mode (CPython `async_gen_athrow_throw` with + // no args): the agen answered the thrown exception + // with another yield — it ignored the close. + return Err(crate::error::runtime_error( + "async generator ignored GeneratorExit", + )); + } + Err(stop_iteration_with(value)) + } + Err(e) => { + a.consumed.set(true); + // aclose-mode: the agen finishing (StopAsyncIteration) or + // letting GeneratorExit out is a *successful* close — + // reported as bare StopIteration. + if is_close + && matches!(&e, RuntimeError::PyException(pe) if matches!(pe.type_name().as_str(), "GeneratorExit" | "StopAsyncIteration")) + { + return Err(stop_iteration()); + } + Err(e) + } + } + } + + /// CPython's `AWAITABLE_STATE_CLOSED` error: driving (or throwing into) + /// an awaitable that already completed. + fn agen_await_reuse_error(kind: crate::object::AgenAwaitKind) -> RuntimeError { + use crate::object::AgenAwaitKind; + crate::error::runtime_error(if matches!(kind, AgenAwaitKind::Send) { + "cannot reuse already awaited __anext__()/asend()" + } else { + "cannot reuse already awaited aclose()/athrow()" + }) + } + + /// CPython's `ag_running_async` guard: starting a *new* awaitable while + /// the agen is suspended inside an inner `await` (still being driven by + /// another awaitable) is a `RuntimeError`, and the rejected awaitable is + /// closed. Returns the error to raise, or `None` when starting is fine. + fn agen_await_running_guard(a: &Rc) -> Option { + use crate::object::AgenAwaitKind; + let Object::AsyncGenerator(g) = &a.agen else { + return None; + }; + let mid_await = matches!( + &*g.state.borrow(), + GeneratorState::Suspended(boxed) + if boxed.downcast_ref::().is_some_and(|f| !f.agen_yielded_value) + ); + if !mid_await { + return None; + } + a.consumed.set(true); + Some(crate::error::runtime_error(match a.kind { + AgenAwaitKind::Send => "anext(): asynchronous generator is already running", + AgenAwaitKind::Throw => "athrow(): asynchronous generator is already running", + AgenAwaitKind::Close => "aclose(): asynchronous generator is already running", + })) + } + + fn step_agen_await( + &mut self, + a: &Rc, + send_value: Object, + ) -> Result { + use crate::object::AgenAwaitKind; + if a.consumed.get() { + return Err(Self::agen_await_reuse_error(a.kind)); + } + // The first drive applies the operation's payload (`asend`'s value, + // `athrow`'s exception, `aclose`); a later drive is reached only when + // the agen suspended on an inner `await` and we passed its value + // through — then we forward the caller's sent value to resume it. + let first = !a.started.get(); + if first { + if let Some(err) = Self::agen_await_running_guard(a) { + return Err(err); + } + } + a.started.set(true); + let outcome = if first { + match a.kind { + AgenAwaitKind::Send => { + let value = a.args.first().cloned().unwrap_or(Object::None); + self.gen_method_send(&a.agen, value) + } + AgenAwaitKind::Throw => self.gen_method_throw(&a.agen, &a.args), + AgenAwaitKind::Close => self.gen_method_close(&a.agen), + } + } else { + self.gen_method_send(&a.agen, send_value) + }; + match outcome { + Ok(value) => { + // Did the agen suspend on an inner `await` (passing `value` + // through) rather than produce a value via its own `yield`? + // Then this awaitable is *not* done: yield `value` onward so + // the surrounding coroutine re-suspends, and keep ourselves + // drivable for the next resume. + if let Object::AsyncGenerator(g) = &a.agen { + if !Self::agen_yielded_a_value(g) { + return Ok(value); + } + } + // The agen yielded a consumer value / the op completed: + // express completion as `StopIteration(value)` so the SEND + // handler short-circuits. + a.consumed.set(true); + Err(stop_iteration_with(value)) + } + // `StopAsyncIteration` (agen finished) and genuine exceptions + // raised in the agen body propagate out of the await. + Err(e) => { + a.consumed.set(true); + Err(e) + } + } + } + + /// Did async generator `g`'s most recent suspension come from its own + /// `yield` (a value for the consumer), as opposed to an inner `await` + /// passing a suspension's value through? Read from the suspended frame's + /// [`Frame::agen_yielded_value`]. Defaults to `true` (the historical + /// "completes with value" behavior) when the frame can't be inspected + /// (e.g. the generator already finished). + fn agen_yielded_a_value(g: &Rc) -> bool { + match &*g.state.borrow() { + GeneratorState::Suspended(boxed) => boxed + .downcast_ref::() + .map_or(true, |f| f.agen_yielded_value), + _ => true, + } + } + /// `gen.throw(exc[, val[, tb]])` — inject an exception at the /// suspended yield-point. Minimal implementation: we don't try /// to faithfully resume the frame; we raise the exception out of @@ -6724,6 +7595,13 @@ impl Interpreter { receiver: &Object, args: &[Object], ) -> Result { + // `asend(...)/athrow(...)/__anext__()` awaitables expose `throw` + // too (CPython `async_gen_asend_throw`): forward the exception + // into the agen and report completion the way a drive does. + if let Object::AsyncGenAwait(a) = receiver { + let a = a.clone(); + return self.agen_await_throw(&a, args); + } let (g, is_async_gen) = match receiver { Object::Generator(g) | Object::Coroutine(g) => (g.clone(), false), Object::AsyncGenerator(g) => (g.clone(), true), @@ -6753,6 +7631,51 @@ impl Interpreter { } } + /// PEP 479. A `StopIteration` raised inside (and escaping) the body + /// of a generator or coroutine is silently confusing — it looks + /// just like the generator finishing — so Python 3.7+ replaces it + /// with a `RuntimeError` chained (`__cause__`) to the original. For + /// an async generator the same applies to both `StopIteration` and + /// `StopAsyncIteration` escaping the body. + /// + /// Only genuine body escapes reach here: the normal "I'm done" + /// signal a `return` produces (`stop_iteration_with`, from the + /// `Returned` arms) is left untouched so `send`/`next`/`yield from` + /// keep working. + fn pep479_escape(&self, gen: &Rc, err: RuntimeError) -> RuntimeError { + use crate::object::CoroutineKind; + let RuntimeError::PyException(exc) = err else { + return err; + }; + let Object::Instance(inst) = &exc.instance else { + return RuntimeError::PyException(exc); + }; + let bt = crate::builtin_types::builtin_types(); + let is_stop_iter = inst.class.is_subclass_of(&bt.stop_iteration); + let is_stop_async = inst.class.is_subclass_of(&bt.stop_async_iteration); + let msg = match gen.kind { + CoroutineKind::Generator if is_stop_iter => "generator raised StopIteration", + CoroutineKind::Coroutine if is_stop_iter => "coroutine raised StopIteration", + CoroutineKind::AsyncGenerator if is_stop_iter => { + "async generator raised StopIteration" + } + CoroutineKind::AsyncGenerator if is_stop_async => { + "async generator raised StopAsyncIteration" + } + _ => return RuntimeError::PyException(exc), + }; + let rt_inst = + crate::builtin_types::make_exception_with_class(bt.runtime_error.clone(), msg); + let mut new_exc = PyException::new(rt_inst); + // Keep the in-flight traceback so the RuntimeError points at the + // offending frame; chain the original as cause + context. + new_exc.traceback = exc.traceback.clone(); + new_exc.cause = Some(Box::new(exc.clone())); + new_exc.context = Some(Box::new(exc)); + Self::sync_exc_attrs(&new_exc); + RuntimeError::PyException(new_exc) + } + /// Inject `exc` into the suspended generator at its current /// resume point. The frame's exception table gets first crack; /// if no handler matches the exception bubbles out of `throw()`. @@ -6767,7 +7690,7 @@ impl Interpreter { fn generator_throw( &mut self, gen: &Rc, - exc: PyException, + mut exc: PyException, ) -> Result { let prev_state = std::mem::replace(&mut *gen.state.borrow_mut(), GeneratorState::Running); let mut frame = match prev_state { @@ -6782,6 +7705,11 @@ impl Interpreter { return Err(value_error("generator already executing")); } }; + // PEP 3134: an exception thrown into a generator suspended inside + // an `except`/`with` block chains to the exception that block was + // handling. Done before delegation/handling so the `__context__` + // travels with the exception however it propagates. + Self::chain_thrown_context(&mut exc, &frame); // PEP 380 delegation. We detect "frame paused in // yield-from" via the bytecode pattern: the most recently @@ -6829,7 +7757,7 @@ impl Interpreter { } Err(err) => { *gen.state.borrow_mut() = GeneratorState::Finished; - Err(err) + Err(self.pep479_escape(gen, err)) } }; } @@ -6869,13 +7797,13 @@ impl Interpreter { } Err(err) => { *gen.state.borrow_mut() = GeneratorState::Finished; - Err(err) + Err(self.pep479_escape(gen, err)) } }, Ok(None) => unreachable!(), Err(err) => { *gen.state.borrow_mut() = GeneratorState::Finished; - Err(err) + Err(self.pep479_escape(gen, err)) } } } @@ -6885,6 +7813,38 @@ impl Interpreter { /// `finally` blocks run; we mirror that by routing through /// `generator_throw` and absorbing the resulting StopIteration. fn gen_method_close(&mut self, receiver: &Object) -> Result { + // Closing a deferred `asend`/`athrow` awaitable (CPython + // `async_gen_asend_close`): a not-yet-started awaitable just flips + // to CLOSED; a started one must deliver `GeneratorExit` to the + // suspended agen, and the agen answering with another suspension + // means it ignored the exit. + if let Object::AsyncGenAwait(a) = receiver { + let a = a.clone(); + if a.consumed.get() { + return Ok(Object::None); + } + a.consumed.set(true); + if !a.started.get() { + return Ok(Object::None); + } + let bt = crate::builtin_types::builtin_types(); + let exc_inst = + crate::builtin_types::make_exception_with_class(bt.generator_exit.clone(), ""); + return match self.gen_method_throw(&a.agen, &[exc_inst]) { + Ok(_yielded) => Err(crate::error::runtime_error( + "coroutine ignored GeneratorExit", + )), + Err(RuntimeError::PyException(exc)) + if matches!( + exc.type_name().as_str(), + "GeneratorExit" | "StopIteration" | "StopAsyncIteration" + ) => + { + Ok(Object::None) + } + Err(err) => Err(err), + }; + } let g = match receiver { Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => g.clone(), _ => return Err(type_error("close() requires a generator/coroutine")), @@ -6912,6 +7872,21 @@ impl Interpreter { || exc.type_name() == "StopAsyncIteration" => { *g.state.borrow_mut() = GeneratorState::Finished; + // Python 3.13 (gh-104770): if the generator catches + // `GeneratorExit` and then *returns* a value, `close()` + // returns it. The return surfaces as `StopIteration(value)`; + // GeneratorExit propagating (exit not caught) yields None. + if exc.type_name() == "StopIteration" { + if let Object::Instance(inst) = &exc.instance { + if let Some(v) = inst + .dict + .borrow() + .get(&DictKey(Object::from_static("value"))) + { + return Ok(v.clone()); + } + } + } Ok(Object::None) } Err(err) => { @@ -6970,13 +7945,13 @@ impl Interpreter { } Err(err) => { *gen.state.borrow_mut() = GeneratorState::Finished; - Err(err) + Err(self.pep479_escape(gen, err)) } }, Ok(None) => unreachable!(), Err(err) => { *gen.state.borrow_mut() = GeneratorState::Finished; - Err(err) + Err(self.pep479_escape(gen, err)) } } } @@ -7045,7 +8020,7 @@ impl Interpreter { } Err(err) => { *gen.state.borrow_mut() = GeneratorState::Finished; - Err(err) + Err(self.pep479_escape(gen, err)) } } } @@ -8382,6 +9357,24 @@ impl Interpreter { ty.name ))); } + // A user metaclass `__setattr__` intercepts class-attribute + // writes (CPython: `type(cls).__setattr__(cls, …)` — + // `EnumType.__setattr__` raises on member reassignment). + if let Some(method) = metaclass_method(obj, "__setattr__") { + if !matches!( + method, + Object::BoundMethod(ref bm) if matches!(&bm.function, Object::Builtin(_)) + ) { + let g = self.builtins.clone(); + self.call( + &method, + &[Object::from_str(name), value], + &[], + &g, + )?; + return Ok(()); + } + } ty.dict .borrow_mut() .insert(DictKey(Object::from_str(name)), value); @@ -8461,6 +9454,32 @@ impl Interpreter { name ))), }, + // `gen.__name__` / `gen.__qualname__` are writable (str only) + // on all three generator flavours, mirroring CPython's + // `gi_name`/`gi_qualname` setters. + Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { + match name { + "__name__" | "__qualname__" => match value { + Object::Str(s) => { + let target = if name == "__name__" { + &g.name + } else { + &g.qualname + }; + *target.borrow_mut() = s.to_string(); + Ok(()) + } + _ => Err(type_error(format!( + "{name} must be set to a string object" + ))), + }, + _ => Err(attribute_error(format!( + "'{}' object has no attribute '{}'", + obj.type_name(), + name + ))), + } + } _ => Err(type_error(format!( "'{}' object has no attribute '{}'", obj.type_name(), @@ -8499,6 +9518,23 @@ impl Interpreter { return Ok(()); } } + // `obj.__dict__ = d` (CPython's `__dict__` getset descriptor): + // replace the instance dict's contents wholesale. Inline-values + // state is permanently cleared. Divergence: CPython aliases `d` + // itself as the instance dict; our `Rc` field can't be swapped, + // so we copy `d`'s contents instead. + if name == "__dict__" && !inst.class.forbids_dict { + let Object::Dict(src) = &value else { + return Err(type_error(format!( + "__dict__ must be set to a dictionary, not a '{}'", + value.type_name() + ))); + }; + let copied = src.borrow().clone(); + *inst.dict.borrow_mut() = copied; + inst.inline_values.set(false); + return Ok(()); + } if let Some(attr) = inst.class.lookup(name) { match &attr { Object::Property(prop) => { @@ -8563,6 +9599,15 @@ impl Interpreter { return Ok(()); } } + // `del obj.__dict__` (CPython's `__dict__` getset + // descriptor): detach the instance dict. The values are + // dropped and the instance reverts to an empty dict; the + // inline-values state is permanently cleared. + if name == "__dict__" && !inst.class.forbids_dict { + inst.dict.borrow_mut().clear(); + inst.inline_values.set(false); + return Ok(()); + } if let Some(attr) = inst.class.lookup(name) { match &attr { Object::Property(prop) => { @@ -8613,6 +9658,18 @@ impl Interpreter { Ok(()) } Object::Type(ty) => { + // A user metaclass `__delattr__` intercepts class-attribute + // deletion (`EnumType.__delattr__` raises for members). + if let Some(method) = metaclass_method(obj, "__delattr__") { + if !matches!( + method, + Object::BoundMethod(ref bm) if matches!(&bm.function, Object::Builtin(_)) + ) { + let g = self.builtins.clone(); + self.call(&method, &[Object::from_str(name)], &[], &g)?; + return Ok(()); + } + } // `del Cls.attr` (CPython `type.__delattr__`) removes the // name from the class's *own* dict only — inherited // attributes can't be deleted via a subclass. Mirrors @@ -8995,6 +10052,58 @@ impl Interpreter { let _ = outer_globals; match callable { Object::Builtin(b) => { + // Unbound generator-family methods reached via the type + // (`type(agen).__anext__`): the instance arrives as `args[0]`. + // Route to the same machinery the bound `.gen_*`/`.agen_*` + // sentinels use. See `unbound_gen_method_sentinel`. + if b.name.starts_with(".u.") { + let receiver = args.first().cloned().ok_or_else(|| { + type_error(format!( + "unbound method {}() needs an argument", + &b.name[3..] + )) + })?; + let rest: &[Object] = if args.is_empty() { &[] } else { &args[1..] }; + return match b.name { + ".u.gen_send" => self + .gen_method_send(&receiver, rest.first().cloned().unwrap_or(Object::None)), + ".u.gen_throw" => self.gen_method_throw(&receiver, rest), + ".u.gen_close" => self.gen_method_close(&receiver), + ".u.gen_next" => self.gen_method_send(&receiver, Object::None), + ".u.gen_iter" => Ok(receiver.clone()), + ".u.agen_aiter" => Ok(receiver.clone()), + ".u.agen_anext" => match &receiver { + Object::AsyncGenerator(_) => Ok(make_agen_await( + &receiver, + crate::object::AgenAwaitKind::Send, + vec![Object::None], + )), + other => Err(type_error(format!( + "__anext__ requires an async_generator, got '{}'", + other.type_name() + ))), + }, + ".u.agen_send" => Ok(make_agen_await( + &receiver, + crate::object::AgenAwaitKind::Send, + vec![rest.first().cloned().unwrap_or(Object::None)], + )), + ".u.agen_throw" => Ok(make_agen_await( + &receiver, + crate::object::AgenAwaitKind::Throw, + rest.to_vec(), + )), + ".u.agen_close" => Ok(make_agen_await( + &receiver, + crate::object::AgenAwaitKind::Close, + Vec::new(), + )), + _ => Err(RuntimeError::Internal(format!( + "unknown unbound gen sentinel {}", + b.name + ))), + }; + } if b.name == builtins::BUILD_CLASS_NAME { return self.build_class(args, kwargs); } @@ -9273,14 +10382,43 @@ impl Interpreter { self.run_pending_finalizers(); return Ok(result); } - // Pre-materialize generator/instance iterables for - // builtin methods that need to iterate them. The + // Pre-materialize VM-only iterables (generators, user + // `__iter__` instances, metaclass-iterable classes) for + // builtin methods that iterate their arguments. The // underlying static builtins call `Object::make_iter` - // directly, which can't drive a Python generator. - if matches!(b.name, "join" | "extend") && args.len() == 2 { - if matches!(&args[1], Object::Generator(_) | Object::Instance(_)) { - let collected = self.collect_iterable(&args[1], outer_globals)?; - let new_args = vec![args[0].clone(), Object::new_list(collected)]; + // directly, which can't drive a Python frame. `dict.update` + // has its own richer protocol below, so it's excluded here. + if matches!( + b.name, + "join" + | "extend" + | "update" + | "fromkeys" + | "union" + | "intersection" + | "difference" + | "symmetric_difference" + | "intersection_update" + | "difference_update" + | "symmetric_difference_update" + | "issubset" + | "issuperset" + | "isdisjoint" + | "writelines" + ) && !(b.name == "update" && !matches!(args.first(), Some(Object::Set(_)))) + { + // `fromkeys` may take its iterable in slot 0 (unbound + // classmethod form); the others iterate args[1..]. + let scan_from = usize::from(b.name != "fromkeys"); + if args.iter().skip(scan_from).any(object_needs_vm_iter) { + let mut new_args = args.to_vec(); + for a in new_args.iter_mut().skip(scan_from) { + if object_needs_vm_iter(a) { + *a = Object::new_list( + self.collect_iterable(a, outer_globals)?, + ); + } + } return (b.call)(&new_args); } } @@ -9292,15 +10430,7 @@ impl Interpreter { // `collect_iterable` when handed something only the // interpreter can iterate. Single-iterable builtins take it // in `args[0]`; `zip` takes one per argument. - fn needs_vm_iter(o: &Object) -> bool { - matches!( - o, - Object::Generator(_) - | Object::Coroutine(_) - | Object::AsyncGenerator(_) - | Object::Instance(_) - ) - } + let needs_vm_iter = object_needs_vm_iter; if matches!(b.name, "enumerate" | "sum" | "all" | "any") && args.first().is_some_and(needs_vm_iter) { @@ -9315,7 +10445,9 @@ impl Interpreter { // `zip` must NOT pre-materialise — it stops at the shortest // iterable, so a paired unbounded iterator (`itertools.count`) // would hang. Drive it lazily through the interpreter instead. - if b.name == "zip" && args.iter().any(needs_vm_iter) { + // Also route any `strict=` call here: the plain builtin + // can't take keywords. + if b.name == "zip" && (args.iter().any(needs_vm_iter) || !kwargs.is_empty()) { return self.do_zip_call(args, kwargs, outer_globals); } if b.name == "sorted" && !args.is_empty() { @@ -9330,6 +10462,36 @@ impl Interpreter { if b.name == "reversed" && args.first().is_some_and(needs_vm_iter) { return self.do_reversed_call(args, outer_globals); } + // `dict.update` accepts another dict, an arbitrary mapping + // (anything with `.keys()`), an iterable of key/value + // pairs, and keyword arguments — the latter three need the + // interpreter (user `keys`/`__getitem__`/`__iter__`), so + // route dict receivers here instead of the native builtin. + if b.name == "update" + && matches!(args.first(), Some(Object::Dict(_))) + { + return self.do_dict_update_call(args, kwargs, outer_globals); + } + // `dir(x)` honours a user `__dir__` — the instance's class + // method, or for a class the *metaclass* method + // (`dir(SomeEnum)` → `EnumType.__dir__`). CPython sorts + // the result. + if b.name == "dir" && args.len() == 1 { + let method = instance_method(&args[0], "__dir__") + .or_else(|| metaclass_method(&args[0], "__dir__")); + if let Some(method) = method { + let r = self.call(&method, &[], &[], outer_globals)?; + let it = self.make_iter(&r, outer_globals)?; + let mut names: Vec = Vec::new(); + while let Some(x) = self.iter_next(&it, outer_globals)? { + names.push(x.to_str()); + } + names.sort(); + return Ok(Object::new_list( + names.into_iter().map(Object::from_str).collect(), + )); + } + } // `setattr`/`delattr` must honour the descriptor protocol // (data descriptors / `property` setters), `__slots__` // enforcement, and a user `__setattr__`/`__delattr__` — the @@ -9473,13 +10635,25 @@ impl Interpreter { // --- async generator methods --------------------- // `__aiter__` returns the agen itself. ".agen_aiter" => return Ok(bm.receiver.clone()), - // `__anext__` returns the agen wrapped as a - // coroutine-shaped awaitable: when driven via - // SEND, it forwards to the underlying generator - // and translates StopIteration into - // StopAsyncIteration so async-for can terminate. + // `g.__anext__()` returns a deferred awaitable + // (CPython's `async_generator_asend`) that, when + // driven, advances the agen once. Returning a proper + // awaitable — rather than the agen itself — is what + // makes the manual-drive idiom `g.__anext__().__next__()` + // work (it raises `StopIteration(value)` per step and + // `StopAsyncIteration` at exhaustion); `await + // g.__anext__()` and `async for` keep working since the + // awaitable drives via SEND exactly like the agen did. + // (`anext()`/`async for` use `get_anext`, a separate + // path, so they are unaffected.) ".agen_anext" => match &bm.receiver { - Object::AsyncGenerator(_) => return Ok(bm.receiver.clone()), + Object::AsyncGenerator(_) => { + return Ok(make_agen_await( + &bm.receiver, + crate::object::AgenAwaitKind::Send, + vec![Object::None], + )) + } other => { return Err(type_error(format!( "__anext__ requires an async_generator, got '{}'", @@ -9487,15 +10661,34 @@ impl Interpreter { ))) } }, + // `asend`/`athrow`/`aclose` return a *deferred* + // awaitable (CPython `async_generator_asend` / + // `async_generator_athrow`); the operation runs only + // when the awaitable is driven, so `await + // agen.aclose()` is legal (the previous code executed + // eagerly and returned the result, yielding `await + // None`). See [`Self::step_agen_await`]. ".agen_send" => { let value = args.first().cloned().unwrap_or(Object::None); - return self.gen_method_send(&bm.receiver, value); + return Ok(make_agen_await( + &bm.receiver, + crate::object::AgenAwaitKind::Send, + vec![value], + )); } ".agen_throw" => { - return self.gen_method_throw(&bm.receiver, args); + return Ok(make_agen_await( + &bm.receiver, + crate::object::AgenAwaitKind::Throw, + args.to_vec(), + )); } ".agen_close" => { - return self.gen_method_close(&bm.receiver); + return Ok(make_agen_await( + &bm.receiver, + crate::object::AgenAwaitKind::Close, + Vec::new(), + )); } // `object.__reduce_ex__(self, protocol)` — the // default pickling/copy reduction. Needs VM access @@ -9601,15 +10794,12 @@ impl Interpreter { } } } - // `type(name, bases, ns)` builds a new class dynamically. - if Rc::ptr_eq(ty, &builtin_types().type_) && args.len() == 3 { - return self.dynamic_type_call_with_meta(ty.clone(), args, kwargs, true); - } - // `Meta(name, bases, ns)` for a user metaclass — - // route through the metaclass-aware class builder. + // `type(name, bases, ns)` / `Meta(name, bases, ns)` build a + // new class dynamically — through the winner-metaclass + // delegation that CPython's `type_new` performs. let bt = builtin_types(); - if ty.is_subclass_of(&bt.type_) && !Rc::ptr_eq(ty, &bt.type_) && args.len() == 3 { - return self.dynamic_type_call_with_meta(ty.clone(), args, kwargs, true); + if ty.is_subclass_of(&bt.type_) && args.len() == 3 { + return self.winner_aware_dynamic_type_call(ty.clone(), args, kwargs); } // If the class's *metaclass* overrides `__call__`, // dispatch through it so EnumMeta etc. can hook @@ -9648,6 +10838,116 @@ impl Interpreter { } } + /// PEP 3115: store `name = value` into a custom class namespace by + /// dispatching through its `__setitem__` (so the mapping observes the + /// binding — `enum._EnumDict` records members this way). Falls back to + /// the generic subscript-store for non-overriding mappings. + fn class_ns_store( + &mut self, + ns_obj: &Object, + name: &str, + value: Object, + globals: &Rc>, + ) -> Result<(), RuntimeError> { + let key = Object::from_str(name); + if let Some(method) = instance_method(ns_obj, "__setitem__") { + self.call(&method, &[key, value], &[], globals)?; + return Ok(()); + } + self.store_subscr(ns_obj, &key, value, globals) + } + + /// PEP 3115: read a name from a custom class namespace. For the + /// practical case — a `dict` subclass like `_EnumDict` — this reads the + /// wrapped native dict directly; `None` means "not bound here" so the + /// caller falls through to globals/builtins (matching CPython's + /// `LOAD_NAME`). + fn class_ns_load(&self, ns_obj: &Object, name: &str) -> Option { + let key = DictKey(Object::from_str(name)); + match ns_obj { + Object::Dict(d) => d.borrow().get(&key).cloned(), + Object::Instance(inst) => match &inst.native { + Some(Object::Dict(d)) => d.borrow().get(&key).cloned(), + _ => None, + }, + _ => None, + } + } + + /// PEP 3115: `del name` in a class body with a custom namespace. + fn class_ns_delete( + &mut self, + ns_obj: &Object, + name: &str, + globals: &Rc>, + ) -> Result<(), RuntimeError> { + let key = Object::from_str(name); + if let Some(method) = instance_method(ns_obj, "__delitem__") { + self.call(&method, std::slice::from_ref(&key), &[], globals)?; + return Ok(()); + } + self.delete_subscr(ns_obj, &key) + } + + /// Copy a class namespace mapping into a plain `DictData` for type + /// creation, like CPython's `type.__new__` folding the mapping into the + /// type dict. Handles a plain dict or a `dict` subclass instance (the + /// practical PEP 3115 case, e.g. `_EnumDict`, which stores into its + /// wrapped native dict). + fn materialize_class_mapping(&self, ns_obj: &Object) -> Result { + match ns_obj { + Object::Dict(d) => Ok(d.borrow().clone()), + Object::Instance(inst) => match &inst.native { + Some(Object::Dict(d)) => Ok(d.borrow().clone()), + _ => Err(type_error( + "class namespace must be a dict or dict subclass", + )), + }, + _ => Err(type_error( + "class namespace must be a dict or dict subclass", + )), + } + } + + /// PEP 3115: invoke `metaclass.__prepare__(name, bases, **kwds)` when the + /// metaclass overrides it, returning the namespace mapping the class body + /// should populate. `type` defines no `__prepare__`, so a `Some` result + /// here always reflects a genuine user override. + fn call_metaclass_prepare( + &mut self, + metaclass: &Rc, + name: &str, + bases: &[Rc], + kwds: &[(String, Object)], + globals: &Rc>, + ) -> Result, RuntimeError> { + let prep = match metaclass.lookup("__prepare__") { + Some(p) => p, + None => return Ok(None), + }; + // `__prepare__` is conventionally a classmethod; bind the metaclass + // as the implicit first argument when needed. + let (callable, prefix): (Object, Vec) = match &prep { + Object::ClassMethod(inner) => { + ((**inner).clone(), vec![Object::Type(metaclass.clone())]) + } + Object::StaticMethod(inner) => ((**inner).clone(), Vec::new()), + other => (other.clone(), vec![Object::Type(metaclass.clone())]), + }; + let bases_tuple = + Object::new_tuple(bases.iter().map(|b| Object::Type(b.clone())).collect()); + let mut call_args = prefix; + call_args.push(Object::from_str(name)); + call_args.push(bases_tuple); + let result = self.call(&callable, &call_args, kwds, globals)?; + // A plain dict means there's nothing to observe — keep the fast path. + if matches!(result, Object::Dict(_)) { + Ok(None) + } else { + Ok(Some(result)) + } + } + /// Run a `class` statement. /// /// `args[0]` is the class body function, `args[1]` is the class @@ -9723,6 +11023,13 @@ impl Interpreter { } } } + // CPython passes the *original* user bases (before defaulting to + // `object`) to the metaclass `__prepare__`/`__new__`; only the real + // `type.__new__` defaults an empty base list to `object`. `enum` + // depends on this: the base `Enum` class is created with no bases, + // and `EnumType._get_mixins_` rejects a stray `object` as the final + // base. Keep the injected list for the plain-`type` fast path. + let prepare_bases: Vec> = bases.clone(); if bases.is_empty() { bases.push(builtin_types().object_.clone()); } @@ -9748,8 +11055,50 @@ impl Interpreter { // metaclass of any base. let metaclass = resolve_metaclass(metaclass_arg, &bases)?; + // PEP 3115: a metaclass may supply a custom namespace mapping via + // `__prepare__`. When it does, the class body's name bindings flow + // through that mapping's `__setitem__` (see `class_ns_store`) so it + // observes every definition — `enum._EnumDict` records members and + // their order this way. + let ns_obj = self.call_metaclass_prepare( + &metaclass, + &name, + &prepare_bases, + &subclass_kwargs, + &body_fn.globals, + )?; + + // The dict the new type is ultimately built from. For the custom + // `__prepare__` path it is materialised from `ns_obj` after the body. let class_ns = Rc::new(RefCell::new(DictData::new())); - { + + // `__module__` copies whatever `globals['__name__']` is at definition + // time (`__main__` for top-level classes, else the module name) so + // `pickle`/introspection can find the qualified name. PEP 560 keeps + // the pre-resolution bases on `__orig_bases__`. + let module_name = body_fn + .globals + .borrow() + .get(&DictKey(Object::from_static("__name__"))) + .cloned(); + let orig_bases_obj = if bases_replaced { + Some(Object::new_tuple(orig_bases.clone())) + } else { + None + }; + if let Some(obj) = &ns_obj { + // Route the implicit names through the custom mapping so it sees + // them exactly as CPython does. + let g = body_fn.globals.clone(); + self.class_ns_store(obj, "__name__", Object::from_str(&name), &g)?; + self.class_ns_store(obj, "__qualname__", Object::from_str(&name), &g)?; + if let Some(m) = module_name { + self.class_ns_store(obj, "__module__", m, &g)?; + } + if let Some(ob) = orig_bases_obj { + self.class_ns_store(obj, "__orig_bases__", ob, &g)?; + } + } else { let mut ns = class_ns.borrow_mut(); ns.insert( DictKey(Object::from_static("__name__")), @@ -9759,27 +11108,11 @@ impl Interpreter { DictKey(Object::from_static("__qualname__")), Object::from_str(&name), ); - // Stamp `__module__` so `pickle` (and any user code that - // introspects classes) can find the qualified name. We - // copy whatever `globals['__name__']` is at definition - // time, which is `__main__` for top-level classes and the - // module name for everything else. - if let Some(module_name) = body_fn - .globals - .borrow() - .get(&DictKey(Object::from_static("__name__"))) - .cloned() - { - ns.insert(DictKey(Object::from_static("__module__")), module_name); - } - // PEP 560: preserve the pre-resolution bases so `typing` / - // `dataclasses` introspection (and `NamedTuple`/`Generic`) - // can read the original `(NamedTuple,)` / `(Generic[T],)`. - if bases_replaced { - ns.insert( - DictKey(Object::from_static("__orig_bases__")), - Object::new_tuple(orig_bases.clone()), - ); + if let Some(m) = module_name { + ns.insert(DictKey(Object::from_static("__module__")), m); + } + if let Some(ob) = orig_bases_obj { + ns.insert(DictKey(Object::from_static("__orig_bases__")), ob); } } // Build a frame for the class body. Locals are unused; names @@ -9793,9 +11126,45 @@ impl Interpreter { body_fn.globals.clone(), false, ); - frame.class_namespace = Some(class_ns.clone()); + if let Some(obj) = &ns_obj { + frame.class_namespace_obj = Some(obj.clone()); + } else { + frame.class_namespace = Some(class_ns.clone()); + } let _ = self.run_frame(&mut frame)?; + // PEP 3135: when the body created a `__class__` cell (a method + // references `super`/`__class__`), propagate it through the + // namespace as `__classcell__` — exactly like CPython's compiler. + // `type.__new__` (our construction cores) pops it and points it at + // the new class *before* `__set_name__`/`__init_subclass__` run, + // so zero-arg `super()` works inside hooks fired during class + // creation (e.g. enum member `__new__` via `_proto_member`). + let class_cell: Option>> = body_fn + .code + .cellvars + .iter() + .position(|c| c == "__class__") + .and_then(|i| frame.cells.get(i).cloned()); + if let Some(cell) = &class_cell { + let cell_obj = Object::Cell(cell.clone()); + if let Some(obj) = &ns_obj { + let g = body_fn.globals.clone(); + self.class_ns_store(obj, "__classcell__", cell_obj, &g)?; + } else { + class_ns.borrow_mut().insert( + DictKey(Object::from_static("__classcell__")), + cell_obj, + ); + } + } + + // PEP 3115: fold the custom mapping back into `class_ns` so doc + // defaulting and the plain-type fast path see the final namespace. + if let Some(obj) = &ns_obj { + *class_ns.borrow_mut() = self.materialize_class_mapping(obj)?; + } + // A class body with a leading docstring `STORE_NAME`s it as // `__doc__` (see the compiler); every other class needs an // explicit `__doc__ = None` so `Cls.__doc__` reads as `None` @@ -9816,9 +11185,15 @@ impl Interpreter { let bt = builtin_types(); let is_plain_type = Rc::ptr_eq(&metaclass, &bt.type_); let ty = if is_plain_type { - let dict = class_ns.borrow().clone(); + let mut dict = class_ns.borrow().clone(); + let classcell = + dict.shift_remove(&DictKey(Object::from_static("__classcell__"))); let ty = TypeObject::new_user(&name, bases.clone(), dict)?; ty.set_metaclass(metaclass.clone()); + // PEP 3135: fill the `__class__` cell before any hook runs. + if let Some(Object::Cell(cell)) = classcell { + *cell.borrow_mut() = Object::Type(ty.clone()); + } self.finalize_class_namespace(&ty)?; self.invoke_set_name_hooks(&ty)?; self.invoke_init_subclass(&ty, &subclass_kwargs)?; @@ -9828,9 +11203,20 @@ impl Interpreter { // The metaclass's `__new__` (if any) chains into // `type.__new__`, which we intercept via // `dynamic_type_call_with_meta` to actually build the type. - let bases_tuple = - Object::new_tuple(bases.iter().map(|b| Object::Type(b.clone())).collect()); - let ns_dict = Object::Dict(class_ns.clone()); + let bases_tuple = Object::new_tuple( + prepare_bases + .iter() + .map(|b| Object::Type(b.clone())) + .collect(), + ); + // PEP 3115: hand the metaclass the *same* mapping its + // `__prepare__` produced (e.g. the populated `_EnumDict`), so its + // `__new__` can read the metadata recorded during the body. Plain + // namespaces pass the materialised dict as before. + let ns_dict = match &ns_obj { + Some(obj) => obj.clone(), + None => Object::Dict(class_ns.clone()), + }; let call_args = vec![Object::from_str(&name), bases_tuple, ns_dict]; // Run the metaclass's __new__ first if it defines one; // otherwise fall through to the default class construction. @@ -9934,6 +11320,78 @@ impl Interpreter { Ok(Object::Type(ty)) } + /// Dynamic 3-arg class construction (`type(name, bases, ns)` or + /// `Meta(name, bases, ns)`) with CPython `type_new`'s winner rule: + /// the build is owned by the most-derived metaclass among the seed + /// and the bases' metaclasses. When that winner defines its own + /// (Python) `__new__`, delegate to it — that is how + /// `type(name, (member,), ns, boundary=…, _simple=True)` re-enters + /// `EnumType.__new__` so the metaclass keywords are consumed by the + /// code that understands them — then run the winner's `__init__` + /// exactly as `type.__call__` would. Otherwise fall through to the + /// default builder. + fn winner_aware_dynamic_type_call( + &mut self, + seed: Rc, + args: &[Object], + kwargs: &[(String, Object)], + ) -> Result { + let bases: Vec> = match args.get(1) { + Some(Object::Tuple(items)) => items + .iter() + .filter_map(|b| match b { + Object::Type(t) => Some(t.clone()), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + let winner = resolve_metaclass(Some(seed), &bases)?; + let new_method = winner.lookup("__new__"); + // The sentinel `type.__new__` builtin would recurse; the default + // builder below *is* that construction. + let is_type_new_sentinel = matches!( + new_method.as_ref(), + Some(Object::StaticMethod(inner)) if matches!( + inner.as_ref(), + Object::Builtin(b) if b.name == "__new__" + ) + ); + if let Some(new_method) = new_method { + if !is_type_new_sentinel { + let callable = match &new_method { + Object::StaticMethod(inner) | Object::ClassMethod(inner) => (**inner).clone(), + other => other.clone(), + }; + let mut new_args = Vec::with_capacity(args.len() + 1); + new_args.push(Object::Type(winner.clone())); + new_args.extend(args.iter().cloned()); + let result = self.call(&callable, &new_args, kwargs, &fallback_globals())?; + // `type.__call__` then runs `__init__` — only when + // `__new__` actually produced an instance of the winner. + if let Object::Type(created) = &result { + if created.metaclass_or_type().is_subclass_of(&winner) { + if let Some(init) = winner.lookup("__init__") { + // Only a Python metaclass `__init__` consumes + // class-creation keywords; the builtin + // `type.__init__` ignores them. + let init_consumes_kwargs = matches!(init, Object::Function(_)); + if init_consumes_kwargs { + let bound = Object::BoundMethod(Rc::new(BoundMethod { + receiver: result.clone(), + function: init, + })); + let _ = self.call(&bound, args, kwargs, &fallback_globals())?; + } + } + } + } + return Ok(result); + } + } + self.dynamic_type_call_with_meta(winner, args, kwargs, true) + } + /// `metaclass(name, bases, ns)` — the three-arg form that /// builds a new class. Used by `type(name, bases, ns)`, by /// custom metaclasses, and by the build_class path when the @@ -9972,6 +11430,14 @@ impl Interpreter { let ns_dict_obj = args[2].clone(); let mut ns = match &args[2] { Object::Dict(d) => d.borrow().clone(), + // PEP 3115: `type.__new__` accepts any mapping for the namespace + // and copies it. A metaclass chaining `super().__new__(...)` may + // pass the custom mapping from `__prepare__` directly (e.g. + // `enum.EnumMeta` forwarding its populated `_EnumDict`). + Object::Instance(inst) => match &inst.native { + Some(Object::Dict(d)) => d.borrow().clone(), + _ => return Err(type_error("type() arg 3 must be a dict")), + }, _ => return Err(type_error("type() arg 3 must be a dict")), }; // CPython's `type.__new__` defaults `__doc__` to `None` when the @@ -10001,8 +11467,14 @@ impl Interpreter { .cloned() .collect(); + // PEP 3135: `type.__new__` pops `__classcell__` from the namespace + // and points it at the new class before any creation hook runs. + let classcell = ns.shift_remove(&DictKey(Object::from_static("__classcell__"))); let ty = TypeObject::new_user(&name, effective_bases.clone(), ns)?; ty.set_metaclass(metaclass.clone()); + if let Some(Object::Cell(cell)) = classcell { + *cell.borrow_mut() = Object::Type(ty.clone()); + } self.finalize_class_namespace(&ty)?; // If we're under a user metaclass, run its `__init__` so it @@ -10390,10 +11862,11 @@ impl Interpreter { // route lazy iterables (generators, `zip`/`map`/`filter` // views, genexprs) through the VM-aware collector — the plain // builtins below can only drive eager containers (RFC 0033). - if matches!( + if (matches!( &args.first(), Some(Object::Generator(_) | Object::Iter(_) | Object::Instance(_)) - ) && args.len() == 1 + ) || args.first().is_some_and(object_needs_vm_iter)) + && args.len() == 1 && kwargs.is_empty() { if cls.name == "set" || cls.name == "frozenset" { @@ -10701,6 +12174,11 @@ impl Interpreter { /// canonical Python `.args` tuple. Used by both `raise` and /// explicit `ExceptionClass(...)` calls. fn build_exception_instance(&self, cls: Rc, args: &[Object]) -> Object { + let is_stop_iteration = cls + .mro + .borrow() + .iter() + .any(|t| t.name == "StopIteration"); let inst = PyInstance::new(cls); let args_tuple = Object::new_tuple(args.to_vec()); let mut dict = inst.dict.borrow_mut(); @@ -10708,6 +12186,16 @@ impl Interpreter { if let Some(first) = args.first() { dict.insert(DictKey(Object::from_static("message")), first.clone()); } + // PEP 380: `StopIteration.value` is the first constructor arg + // (or None). Generator `return` goes through + // `stop_iteration_with`, but user code constructs + // `StopIteration(x)` directly and reads `.value` too. + if is_stop_iteration { + dict.insert( + DictKey(Object::from_static("value")), + args.first().cloned().unwrap_or(Object::None), + ); + } // CPython's `BaseException` always exposes these slots (default // None/None/False/None), so attribute access and exception-context // chain walks (e.g. `contextlib._fix_exception_context`, which reads @@ -10973,7 +12461,32 @@ impl Interpreter { // frame in a PyGenerator and hand it back to the caller. match self.run_until_yield_or_return(&mut frame, None)? { FrameOutcome::StartGenerator => { - let gen = Rc::new(PyGenerator::new(f.name.clone(), Box::new(frame))); + let kind = if code.is_coroutine { + crate::object::CoroutineKind::Coroutine + } else if code.is_async_generator { + crate::object::CoroutineKind::AsyncGenerator + } else { + crate::object::CoroutineKind::Generator + }; + // CPython snapshots the *function's* current + // `__name__`/`__qualname__` (which user code may have + // reassigned; overrides live in `f.attrs`) into + // `gi_name`/`gi_qualname` at call time. + let attr_str = |attr: &'static str| -> Option { + match f.attrs.borrow().get(&DictKey(Object::from_static(attr))) { + Some(Object::Str(s)) => Some(s.to_string()), + _ => None, + } + }; + let gen_name = attr_str("__name__").unwrap_or_else(|| f.name.clone()); + let gen_qualname = + attr_str("__qualname__").unwrap_or_else(|| code.qualname.clone()); + let gen = Rc::new(PyGenerator::new( + gen_name, + gen_qualname, + kind, + Box::new(frame), + )); if code.is_coroutine { Ok(Object::Coroutine(gen)) } else if code.is_async_generator { @@ -11149,7 +12662,10 @@ impl Interpreter { stack: Vec::with_capacity(16), globals: f.globals.clone(), class_namespace: None, + class_namespace_obj: None, exc_handlers: Vec::new(), + saved_exc_info: Vec::new(), + agen_yielded_value: true, pc: 0, py_frame: None, }; @@ -12593,6 +14109,34 @@ fn resolve_metaclass( Ok(winner) } +/// Bind callable `m` to `receiver` (the dunder-dispatch convention used +/// throughout: the receiver flows in as the first argument). +pub(crate) fn bind_method(receiver: &Object, m: Object) -> Object { + Object::BoundMethod(Rc::new(BoundMethod { + receiver: receiver.clone(), + function: m, + })) +} + +/// Resolve `name` through the *metaclass* of class `v` and bind it to the +/// class — the dispatch CPython performs for protocol operations applied +/// to a class object itself (`len(SomeEnum)`, `x in SomeEnum`, +/// `reversed(SomeEnum)`, …): `type(cls).__dunder__(cls)`. +pub(crate) fn metaclass_method(v: &Object, name: &str) -> Option { + match v { + Object::Type(t) => { + let meta = t.metaclass_or_type(); + // `type` itself contributes no protocol dunders here; only a + // user metaclass (EnumType, ABCMeta, …) does. + if Rc::ptr_eq(&meta, &builtin_types().type_) { + return None; + } + meta.lookup(name).map(|m| bind_method(v, m)) + } + _ => None, + } +} + pub(crate) fn instance_method(obj: &Object, name: &str) -> Option { let inst = match obj { Object::Instance(i) => i.clone(), @@ -12605,6 +14149,52 @@ pub(crate) fn instance_method(obj: &Object, name: &str) -> Option { }))) } +thread_local! { + /// Cache of synthesized built-in slot wrappers, keyed by + /// `(type pointer, dunder name)`. Built-in types are per-thread + /// singletons with stable addresses, so caching by raw pointer keeps + /// type-level dunder access identity-stable: `int.__add__ is int.__add__` + /// and `getattr(object, '__repr__') is getattr(object, '__repr__')` — the + /// identity `enum`'s bootstrap (`found in (data_type_method, object_method)`) + /// depends on. Only built-in types are keyed, so the entries live as long + /// as the type singletons themselves. + static SLOT_WRAPPER_CACHE: std::cell::RefCell> = + std::cell::RefCell::new(std::collections::HashMap::new()); +} + +/// Resolve a built-in slot-wrapper dunder reached via *type-level* attribute +/// access (`int.__add__`, `object.__repr__`, `MyClass.__str__`). Walks `ty`'s +/// MRO and returns the wrapper contributed by the first *built-in* base that +/// defines `name`, synthesizing it once and caching it for stable identity. A +/// user subclass that doesn't override the slot therefore resolves to the +/// defining base's wrapper (so `MyClass.__repr__ is object.__repr__`, as in +/// CPython). Returns `None` when no built-in base defines the dunder, letting +/// the caller raise `AttributeError`. +/// +/// This is reached only from [`Interpreter::load_attr_type`] (the type-level +/// path); instance attribute access keeps using `repr_of` / `stringify` / +/// `instance_method`, so the hot per-object dispatch is unchanged. +fn builtin_slot_wrapper(ty: &Rc, name: &str) -> Option { + let mro: Vec> = ty.mro.borrow().iter().cloned().collect(); + for base in mro { + if !base.flags.is_builtin { + continue; + } + let ptr = Rc::as_ptr(&base) as usize; + if let Some(o) = SLOT_WRAPPER_CACHE.with(|c| c.borrow().get(&(ptr, name.to_owned())).cloned()) + { + return Some(o); + } + if let Some(o) = crate::builtins::builtin_type_dunder(&base.name, name) { + SLOT_WRAPPER_CACHE.with(|c| { + c.borrow_mut().insert((ptr, name.to_owned()), o.clone()); + }); + return Some(o); + } + } + None +} + /// Return a fresh empty globals dict — used by the awaitable /// dispatch paths that don't have a frame's globals handy. The /// dispatched method itself carries its own `__globals__`. @@ -12654,6 +14244,23 @@ fn init_is_from_object(cls: &Rc) -> bool { false } +/// Build the deferred awaitable returned by `agen.asend(v)` / +/// `.athrow(e)` / `.aclose()` (PEP 525). The operation is applied only +/// when the awaitable is driven — see [`Interpreter::step_agen_await`]. +fn make_agen_await( + receiver: &Object, + kind: crate::object::AgenAwaitKind, + args: Vec, +) -> Object { + Object::AsyncGenAwait(Rc::new(crate::object::AsyncGenAwait { + agen: receiver.clone(), + kind, + args, + consumed: crate::sync::Cell::new(false), + started: crate::sync::Cell::new(false), + })) +} + /// Build the `Object::BoundMethod` returned by /// `.send` / `.throw` / `.close` / `.__next__` / `.__iter__`. /// The actual dispatch is handled by [`Interpreter::call`] via the @@ -12688,6 +14295,50 @@ fn make_gen_method(name: &str, receiver: &Object) -> Object { })) } +/// Map a `type().` access to the sentinel name for +/// its *unbound* form — the function that takes the instance as `args[0]`, +/// e.g. `type(agen).__anext__(agen)`. CPython exposes generator/coroutine/ +/// async-generator methods as `method_descriptor`s on the type; pure-Python +/// code such as test_asyncgen's `py_anext` reads `type(it).__anext__` and +/// then calls it with the instance. The `.u.` namespace is dispatched in +/// [`Interpreter::call`]'s plain-`Builtin` arm with `args[0]` as receiver. +/// +/// Gated on `Rc::ptr_eq` against the canonical built-in types so a user +/// class merely *named* "generator" can't accidentally borrow these. +fn unbound_gen_method_sentinel(ty: &Rc, name: &str) -> Option<&'static str> { + let bt = builtin_types(); + if Rc::ptr_eq(ty, &bt.generator_) { + return match name { + "send" => Some(".u.gen_send"), + "throw" => Some(".u.gen_throw"), + "close" => Some(".u.gen_close"), + "__next__" => Some(".u.gen_next"), + "__iter__" => Some(".u.gen_iter"), + _ => None, + }; + } + if Rc::ptr_eq(ty, &bt.coroutine_) { + return match name { + "send" => Some(".u.gen_send"), + "throw" => Some(".u.gen_throw"), + "close" => Some(".u.gen_close"), + "__await__" => Some(".u.gen_iter"), + _ => None, + }; + } + if Rc::ptr_eq(ty, &bt.async_generator_) { + return match name { + "__aiter__" => Some(".u.agen_aiter"), + "__anext__" => Some(".u.agen_anext"), + "asend" => Some(".u.agen_send"), + "athrow" => Some(".u.agen_throw"), + "aclose" => Some(".u.agen_close"), + _ => None, + }; + } + None +} + /// Look up the `value` attribute on a `StopIteration` instance. Falls /// back to `None` if absent. fn exception_value(instance: &Object) -> Object { @@ -14643,6 +16294,26 @@ fn group_decimal(mag: u64, sep: char) -> String { out } +/// Does iterating `o` require driving the interpreter (a generator +/// resume or an instance `__next__`/`__iter__` call)? Such sources are +/// potentially unbounded and side-effecting, so `map`/`filter`/`zip` +/// build *lazy* iterators over them; plain native containers take the +/// eager fast path. +fn object_needs_vm_iter(o: &Object) -> bool { + // A class whose *metaclass* defines `__iter__` (e.g. `list(SomeEnum)` + // → `EnumType.__iter__`) iterates through the interpreter too. + if let Object::Type(_) = o { + return metaclass_method(o, "__iter__").is_some(); + } + matches!( + o, + Object::Generator(_) + | Object::Coroutine(_) + | Object::AsyncGenerator(_) + | Object::Instance(_) + ) +} + /// Is `e` an `IndexError` (or subclass)? Used by the legacy /// `__getitem__` iteration protocol to detect the end of a sequence. fn is_index_error(e: &RuntimeError) -> bool { @@ -16090,6 +17761,69 @@ mod tests { assert_eq!(run(src), "B-A\n"); } + #[test] + fn metaclass_prepare_custom_namespace() { + // PEP 3115: a metaclass `__prepare__` returning a custom mapping must + // observe every class-body binding through its `__setitem__`, and the + // metaclass `__new__` must receive that same mapping. + let src = concat!( + "class NS(dict):\n", + " log = None\n", + " def __init__(self):\n", + " super().__init__()\n", + " type(self).log = []\n", + " def __setitem__(self, k, v):\n", + " if not k.startswith('__'):\n", + " NS.log.append(k)\n", + " super().__setitem__(k, v)\n", + "class Meta(type):\n", + " @classmethod\n", + " def __prepare__(mcs, name, bases, **kw):\n", + " return NS()\n", + " def __new__(mcs, name, bases, ns, **kw):\n", + " return super().__new__(mcs, name, bases, dict(ns))\n", + "class C(metaclass=Meta):\n", + " A = 1\n", + " B = 2\n", + " def m(self):\n", + " return self.A\n", + "print(NS.log)\n", + "print(type(C).__name__)\n", + "print(C().m())\n", + ); + assert_eq!(run(src), "['A', 'B', 'm']\nMeta\n1\n"); + } + + #[test] + fn builtin_slot_wrappers_identity_and_calls() { + // Type-level object-protocol dunders are gettable, identity-stable, and + // resolve through the MRO to the defining built-in base — the contract + // CPython's `enum` bootstrap relies on + // (`found_method in (data_type_method, object_method)`). + let src = concat!( + // gettable on object / int / str + "print(callable(object.__repr__), callable(int.__str__), callable(str.__format__))\n", + // identity stability + "print(object.__repr__ is object.__repr__)\n", + "print(int.__add__ is int.__add__)\n", + // int inherits object.__str__ (CPython: int has no own __str__), + // str overrides it + "print(int.__str__ is object.__str__)\n", + "print(str.__str__ is object.__str__)\n", + // a user class inherits object's wrappers + "class C: pass\n", + "print(C.__repr__ is object.__repr__)\n", + // calls unwrap native payloads correctly + "print(int.__repr__(5), int.__format__(255, 'x'), str.__repr__('hi'))\n", + // the exact enum-bootstrap membership shape + "print(int.__repr__ in (int.__repr__, object.__repr__))\n", + ); + assert_eq!( + run(src), + "True True True\nTrue\nTrue\nTrue\nFalse\nTrue\n5 ff 'hi'\nTrue\n" + ); + } + #[test] fn nested_try_except() { let src = concat!( diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 3672a09..ff13016 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -82,6 +82,10 @@ pub enum Object { /// returned from calling an `async def` that contains `yield`. /// Consumable via `async for`. AsyncGenerator(Rc), + /// Deferred awaitable produced by `agen.asend()` / `.athrow()` / + /// `.aclose()` (PEP 525). Awaiting it applies the operation to the + /// underlying async generator. See [`AsyncGenAwait`]. + AsyncGenAwait(Rc), /// Immutable byte string `b"..."`. Bytes(Rc<[u8]>), /// Mutable byte string `bytearray(...)`. @@ -170,9 +174,12 @@ impl fmt::Debug for Object { Object::Type(t) => write!(f, "", t.name), Object::Instance(i) => write!(f, "<{} object>", i.class.name), Object::Module(m) => write!(f, "", m.name), - Object::Generator(g) => write!(f, "", g.name), - Object::Coroutine(g) => write!(f, "", g.name), - Object::AsyncGenerator(g) => write!(f, "", g.name), + Object::Generator(g) => write!(f, "", g.name.borrow()), + Object::Coroutine(g) => write!(f, "", g.name.borrow()), + Object::AsyncGenerator(g) => { + write!(f, "", g.name.borrow()) + } + Object::AsyncGenAwait(a) => write!(f, "<{} object>", a.kind.type_name()), Object::Bytes(b) => write!(f, "Bytes({})", b.len()), Object::ByteArray(b) => write!(f, "ByteArray({})", b.borrow().len()), Object::Set(s) => f.debug_set().entries(s.borrow().iter()).finish(), @@ -719,14 +726,31 @@ pub struct PySlice { /// itself is opaque to outside code — it's owned by the VM module via /// `state` and only legal to inspect via interpreter methods. pub struct PyGenerator { - pub name: String, + /// `gi_name`. Seeded from the function's `__name__` at call time; + /// user code may reassign it (`gen.__name__ = ...`). + pub name: RefCell, + /// `gi_qualname` (PEP 3155). Seeded from the function's + /// `__qualname__` at call time; reassignable like `name`. + pub qualname: RefCell, + /// Whether this is a plain generator, a coroutine, or an async + /// generator. Needed so the shared send/throw machinery can apply + /// PEP 479 (a `StopIteration` escaping the *body* becomes a + /// `RuntimeError`) with the right wording per flavour. + pub kind: CoroutineKind, pub state: RefCell, } impl PyGenerator { - pub fn new(name: impl Into, frame: Box) -> Self { + pub fn new( + name: impl Into, + qualname: impl Into, + kind: CoroutineKind, + frame: Box, + ) -> Self { Self { - name: name.into(), + name: RefCell::new(name.into()), + qualname: RefCell::new(qualname.into()), + kind, state: RefCell::new(GeneratorState::Created(frame)), } } @@ -738,7 +762,7 @@ impl PyGenerator { impl fmt::Debug for PyGenerator { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "", self.name) + write!(f, "", self.name.borrow()) } } @@ -779,6 +803,63 @@ impl fmt::Debug for GeneratorState { } } +/// The deferred operation carried by an [`AsyncGenAwait`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AgenAwaitKind { + /// `agen.asend(value)` — resume the agen, sending `value` in. + Send, + /// `agen.athrow(exc[, val[, tb]])` — throw into the agen. + Throw, + /// `agen.aclose()` — throw `GeneratorExit` into the agen. + Close, +} + +impl AgenAwaitKind { + /// CPython type name of the awaitable this op produces: `asend` + /// yields `async_generator_asend`; `athrow`/`aclose` both yield + /// `async_generator_athrow`. + pub fn type_name(self) -> &'static str { + match self { + AgenAwaitKind::Send => "async_generator_asend", + AgenAwaitKind::Throw | AgenAwaitKind::Close => "async_generator_athrow", + } + } +} + +/// Deferred awaitable returned by `agen.asend(v)` / `agen.athrow(e)` / +/// `agen.aclose()` (PEP 525). Mirrors CPython's `async_generator_asend` +/// and `async_generator_athrow`: the operation on the underlying async +/// generator is *deferred* until the awaitable is driven (`await`ed), +/// rather than running eagerly at call time. WeavePy's cooperative async +/// model has no real suspension inside the agen body, so a single drive +/// applies the op and completes the await — but routing through an +/// awaitable (instead of executing inside `asend`/`athrow`/`aclose`) is +/// exactly what makes `await agen.aclose()` legal rather than the bug it +/// replaces (`await None`). +pub struct AsyncGenAwait { + /// The `Object::AsyncGenerator` this operation targets. + pub agen: Object, + pub kind: AgenAwaitKind, + /// Operation payload: `asend` -> `[value]`, `athrow` -> the throw + /// args (`[exc, val?, tb?]`), `aclose` -> empty. + pub args: Vec, + /// Set once the awaitable has been driven, so a second pull behaves + /// like an exhausted iterator (`StopIteration`) instead of replaying + /// the operation. + pub consumed: Cell, + /// Set on the first drive. The first drive applies the operation payload + /// (`args`); later drives — reached only when the agen suspended on an + /// inner `await` and we passed its value through — forward the caller's + /// sent value to resume that inner await. + pub started: Cell, +} + +impl fmt::Debug for AsyncGenAwait { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "<{} object>", self.kind.type_name()) + } +} + /// File-like object exposed to Python. Wraps a [`FileBackend`] so /// the same wrapper can talk to a real file, an in-memory buffer /// (`io.StringIO`/`io.BytesIO`), or the interpreter's stdout/stderr @@ -1412,6 +1493,7 @@ impl Object { | Object::Generator(_) | Object::Coroutine(_) | Object::AsyncGenerator(_) + | Object::AsyncGenAwait(_) | Object::File(_) | Object::Property(_) | Object::StaticMethod(_) @@ -1491,6 +1573,7 @@ impl Object { (Object::Generator(a), Object::Generator(b)) => Rc::ptr_eq(a, b), (Object::Coroutine(a), Object::Coroutine(b)) => Rc::ptr_eq(a, b), (Object::AsyncGenerator(a), Object::AsyncGenerator(b)) => Rc::ptr_eq(a, b), + (Object::AsyncGenAwait(a), Object::AsyncGenAwait(b)) => Rc::ptr_eq(a, b), (Object::Bytes(a), Object::Bytes(b)) => Rc::ptr_eq(a, b), (Object::ByteArray(a), Object::ByteArray(b)) => Rc::ptr_eq(a, b), (Object::Set(a), Object::Set(b)) => Rc::ptr_eq(a, b), @@ -1918,6 +2001,7 @@ impl Object { Object::Generator(_) => "generator", Object::Coroutine(_) => "coroutine", Object::AsyncGenerator(_) => "async_generator", + Object::AsyncGenAwait(a) => a.kind.type_name(), Object::Bytes(_) => "bytes", Object::ByteArray(_) => "bytearray", Object::Set(_) => "set", @@ -2061,21 +2145,27 @@ impl Object { Some(path) => format!("", m.name, path), None => format!("", m.name), }, + // CPython's repr shows the qualified name (PEP 3155). Object::Generator(g) => format!( "", - g.name, + g.qualname.borrow(), Rc::as_ptr(g) as usize ), Object::Coroutine(g) => format!( "", - g.name, + g.qualname.borrow(), Rc::as_ptr(g) as usize ), Object::AsyncGenerator(g) => format!( "", - g.name, + g.qualname.borrow(), Rc::as_ptr(g) as usize ), + Object::AsyncGenAwait(a) => format!( + "<{} object at 0x{:x}>", + a.kind.type_name(), + Rc::as_ptr(a) as usize + ), Object::Bytes(b) => bytes_repr(b), Object::ByteArray(b) => format!("bytearray({})", bytes_repr(&b.borrow())), Object::Set(s) => set_repr(&s.borrow(), "set"), @@ -2091,10 +2181,13 @@ impl Object { file.mode ), Object::Instance(inst) => { - // Defer to __repr__ on the class if present; otherwise - // synthesize a default. The caller is expected to run - // __repr__ through the interpreter for user methods — - // here we only handle the default case. + // Defer to __repr__ on the class when present. This path + // is reached from *native* rendering (container reprs, + // error messages, the Debug impl), so the user `__repr__` + // must be run by re-entering the live interpreter — the + // same reentry the dunder coercions use. Without it, + // `repr([Color.RED])` would render the elements as + // `` instead of ``. let key = DictKey(Object::from_static("__repr__")); let has_user_repr = inst .class @@ -2103,6 +2196,19 @@ impl Object { .iter() .any(|t| t.dict.borrow().contains_key(&key)); if has_user_repr { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still + // live on this thread; the GIL keeps it exclusive. + let interp = unsafe { &mut *ptr }; + if let Some(method) = crate::instance_method(self, "__repr__") { + let globals = interp.builtins_dict(); + if let Ok(r) = + interp.call_object_with_globals(&method, &[], &[], &globals) + { + return r.to_str(); + } + } + } format!("<{} object>", inst.class.name) } else { format!( @@ -2512,6 +2618,7 @@ pub(crate) fn identity_hash(obj: &Object) -> i64 { Object::Generator(r) | Object::Coroutine(r) | Object::AsyncGenerator(r) => { rot(Rc::as_ptr(r).cast()) } + Object::AsyncGenAwait(r) => rot(Rc::as_ptr(r).cast()), Object::File(r) => rot(Rc::as_ptr(r).cast()), Object::Property(r) => rot(Rc::as_ptr(r).cast()), Object::StaticMethod(r) => rot(Rc::as_ptr(r).cast()), diff --git a/crates/weavepy-vm/src/stdlib/mod.rs b/crates/weavepy-vm/src/stdlib/mod.rs index 65894dd..179f413 100644 --- a/crates/weavepy-vm/src/stdlib/mod.rs +++ b/crates/weavepy-vm/src/stdlib/mod.rs @@ -94,6 +94,7 @@ pub fn register_all(cache: &ModuleCache) { cache.register_builtin("time", time::build); cache.register_builtin("_thread", thread_real::build); cache.register_builtin("errno", errno_mod::build); + cache.register_builtin("_testinternalcapi", testinternalcapi_mod::build); cache.register_builtin("signal", signal_mod::build); cache.register_builtin("select", select_mod::build); cache.register_builtin("_socket", socket_mod::build); @@ -878,11 +879,54 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/importlib_resources.py"), is_package: false, }, + // CPython's frozen import-core modules; stdlib code (pydoc, + // pkgutil-adjacent paths) imports these by name. + FrozenSource { + name: "importlib._bootstrap", + source: include_str!("python/importlib_bootstrap.py"), + is_package: false, + }, + FrozenSource { + name: "importlib._bootstrap_external", + source: include_str!("python/importlib_bootstrap_external.py"), + is_package: false, + }, FrozenSource { name: "pkgutil", source: include_str!("python/pkgutil.py"), is_package: false, }, + // RFC 0037 WS8 — pydoc and its dependency closure. + FrozenSource { + name: "pydoc", + source: include_str!("python/pydoc.py"), + is_package: false, + }, + FrozenSource { + name: "token", + source: include_str!("python/token.py"), + is_package: false, + }, + FrozenSource { + name: "tokenize", + source: include_str!("python/tokenize.py"), + is_package: false, + }, + FrozenSource { + name: "sysconfig", + source: include_str!("python/sysconfig.py"), + is_package: false, + }, + FrozenSource { + name: "_pyrepl", + source: include_str!("python/_pyrepl_init.py"), + is_package: true, + }, + FrozenSource { + name: "_pyrepl.pager", + source: include_str!("python/_pyrepl_pager.py"), + is_package: false, + }, FrozenSource { name: "venv", source: include_str!("python/venv_mod.py"), @@ -1232,5 +1276,15 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/sre_compile.py"), is_package: false, }, + // Pure-Python stand-in for CPython's `_testlimitedcapi` C test + // helper. The conformance suite (e.g. `test_bytes`) imports it at + // class-body scope; without it the whole module aborts. We supply + // faithful Python equivalents of the abstract `PySequence_*` + // wrappers it exercises. + FrozenSource { + name: "_testlimitedcapi", + source: include_str!("python/_testlimitedcapi.py"), + is_package: false, + }, ] } diff --git a/crates/weavepy-vm/src/stdlib/python/_pyrepl_init.py b/crates/weavepy-vm/src/stdlib/python/_pyrepl_init.py new file mode 100644 index 0000000..f8b5404 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_pyrepl_init.py @@ -0,0 +1,6 @@ +"""``_pyrepl`` — package shell. + +CPython's new REPL implementation. WeavePy carries only the +``_pyrepl.pager`` submodule (verbatim), which ``pydoc`` imports for its +paging helpers; the interactive REPL itself is Rust-native. +""" diff --git a/crates/weavepy-vm/src/stdlib/python/_pyrepl_pager.py b/crates/weavepy-vm/src/stdlib/python/_pyrepl_pager.py new file mode 100644 index 0000000..1fddc63 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_pyrepl_pager.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import io +import os +import re +import sys + + +# types +if False: + from typing import Protocol + class Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: + ... + + +def get_pager() -> Pager: + """Decide what method to use for paging through text.""" + if not hasattr(sys.stdin, "isatty"): + return plain_pager + if not hasattr(sys.stdout, "isatty"): + return plain_pager + if not sys.stdin.isatty() or not sys.stdout.isatty(): + return plain_pager + if sys.platform == "emscripten": + return plain_pager + use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') + if use_pager: + if sys.platform == 'win32': # pipes completely broken in Windows + return lambda text, title='': tempfile_pager(plain(text), use_pager) + elif os.environ.get('TERM') in ('dumb', 'emacs'): + return lambda text, title='': pipe_pager(plain(text), use_pager, title) + else: + return lambda text, title='': pipe_pager(text, use_pager, title) + if os.environ.get('TERM') in ('dumb', 'emacs'): + return plain_pager + if sys.platform == 'win32': + return lambda text, title='': tempfile_pager(plain(text), 'more <') + if hasattr(os, 'system') and os.system('(pager) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'pager', title) + if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'less', title) + + import tempfile + (fd, filename) = tempfile.mkstemp() + os.close(fd) + try: + if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: + return lambda text, title='': pipe_pager(text, 'more', title) + else: + return tty_pager + finally: + os.unlink(filename) + + +def escape_stdout(text: str) -> str: + # Escape non-encodable characters to avoid encoding errors later + encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' + return text.encode(encoding, 'backslashreplace').decode(encoding) + + +def escape_less(s: str) -> str: + return re.sub(r'([?:.%\\])', r'\\\1', s) + + +def plain(text: str) -> str: + """Remove boldface formatting from text.""" + return re.sub('.\b', '', text) + + +def tty_pager(text: str, title: str = '') -> None: + """Page through text on a text terminal.""" + lines = plain(escape_stdout(text)).split('\n') + has_tty = False + try: + import tty + import termios + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + tty.setcbreak(fd) + has_tty = True + + def getchar() -> str: + return sys.stdin.read(1) + + except (ImportError, AttributeError, io.UnsupportedOperation): + def getchar() -> str: + return sys.stdin.readline()[:-1][:1] + + try: + try: + h = int(os.environ.get('LINES', 0)) + except ValueError: + h = 0 + if h <= 1: + h = 25 + r = inc = h - 1 + sys.stdout.write('\n'.join(lines[:inc]) + '\n') + while lines[r:]: + sys.stdout.write('-- more --') + sys.stdout.flush() + c = getchar() + + if c in ('q', 'Q'): + sys.stdout.write('\r \r') + break + elif c in ('\r', '\n'): + sys.stdout.write('\r \r' + lines[r] + '\n') + r = r + 1 + continue + if c in ('b', 'B', '\x1b'): + r = r - inc - inc + if r < 0: r = 0 + sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') + r = r + inc + + finally: + if has_tty: + termios.tcsetattr(fd, termios.TCSAFLUSH, old) + + +def plain_pager(text: str, title: str = '') -> None: + """Simply print unformatted text. This is the ultimate fallback.""" + sys.stdout.write(plain(escape_stdout(text))) + + +def pipe_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by feeding it to another program.""" + import subprocess + env = os.environ.copy() + if title: + title += ' ' + esc_title = escape_less(title) + prompt_string = ( + f' {esc_title}' + + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + errors='backslashreplace', env=env) + assert proc.stdin is not None + try: + with proc.stdin as pipe: + try: + pipe.write(text) + except KeyboardInterrupt: + # We've hereby abandoned whatever text hasn't been written, + # but the pager is still in control of the terminal. + pass + except OSError: + pass # Ignore broken pipes caused by quitting the pager program. + while True: + try: + proc.wait() + break + except KeyboardInterrupt: + # Ignore ctl-c like the pager itself does. Otherwise the pager is + # left running and the terminal is in raw mode and unusable. + pass + + +def tempfile_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by invoking a program on a temporary file.""" + import tempfile + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, 'pydoc.out') + with open(filename, 'w', errors='backslashreplace', + encoding=os.device_encoding(0) if + sys.platform == 'win32' else None + ) as file: + file.write(text) + os.system(cmd + ' "' + filename + '"') diff --git a/crates/weavepy-vm/src/stdlib/python/_seqtools.py b/crates/weavepy-vm/src/stdlib/python/_seqtools.py index ca2399e..2ff4ca5 100644 --- a/crates/weavepy-vm/src/stdlib/python/_seqtools.py +++ b/crates/weavepy-vm/src/stdlib/python/_seqtools.py @@ -140,3 +140,121 @@ def __reduce__(self): if self._callable is None: return (_iter, ((),)) return (_iter, (self._callable, self._sentinel)) + + +class _FilterIter: + """Lazy ``filter(func, iterable)`` — CPython's ``filterobject``. + + Items are pulled (and the predicate run) one ``next()`` at a time, so + filtering an unbounded source (``filter(p, itertools.count())``) + terminates and predicate side effects interleave with consumption + exactly as in CPython. + """ + + __slots__ = ("_func", "_it") + + def __init__(self, func, iterable): + self._func = func + self._it = iter(iterable) + + def __iter__(self): + return self + + def __next__(self): + func = self._func + it = self._it + while True: + item = next(it) + if func is None or func is bool: + if item: + return item + elif func(item): + return item + + def __reduce__(self): + return (filter, (self._func, self._it)) + + +class _MapIter: + """Lazy ``map(func, *iterables)`` — CPython's ``mapobject``. + + ``func`` is applied on demand; iteration stops at the shortest + iterable. Lazy evaluation means mapping over unbounded sources works + and exceptions from ``func`` surface mid-stream, as in CPython. + """ + + __slots__ = ("_func", "_iters") + + def __init__(self, func, *iterables): + self._func = func + self._iters = tuple(iter(it) for it in iterables) + + def __iter__(self): + return self + + def __next__(self): + args = [] + for it in self._iters: + args.append(next(it)) + return self._func(*args) + + def __reduce__(self): + return (map, (self._func,) + self._iters) + + +def _zip_arg_range(count): + """`argument 1` / `arguments 1-N` phrasing of zip-strict errors.""" + return "argument 1" if count == 1 else f"arguments 1-{count}" + + +class _ZipIter: + """Lazy ``zip(*iterables, strict=...)`` — CPython's ``zipobject``. + + Stops at the shortest iterable without pre-materialising any of + them, so zipping unbounded iterators works. With ``strict=True``, + raises ``ValueError`` on length mismatch with CPython's wording. + """ + + __slots__ = ("_iters", "_strict") + + def __init__(self, strict, *iterables): + self._iters = tuple(iter(it) for it in iterables) + self._strict = strict + + def __iter__(self): + return self + + def __next__(self): + iters = self._iters + if iters is None or not iters: + raise StopIteration + result = [] + for i, it in enumerate(iters): + try: + result.append(next(it)) + except StopIteration: + if not self._strict: + self._iters = None + raise + if i > 0: + self._iters = None + raise ValueError( + f"zip() argument {i+1} is shorter than {_zip_arg_range(i)}" + ) from None + # First iterator exhausted: with strict the rest must be + # exhausted too. + for j, jt in enumerate(iters[1:], 1): + try: + next(jt) + except StopIteration: + continue + self._iters = None + raise ValueError( + f"zip() argument {j+1} is longer than {_zip_arg_range(j)}" + ) from None + self._iters = None + raise + return tuple(result) + + def __reduce__(self): + return (zip, self._iters if self._iters is not None else ((),)) diff --git a/crates/weavepy-vm/src/stdlib/python/_testlimitedcapi.py b/crates/weavepy-vm/src/stdlib/python/_testlimitedcapi.py new file mode 100644 index 0000000..9bad119 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_testlimitedcapi.py @@ -0,0 +1,24 @@ +"""Pure-Python stand-in for CPython's ``_testlimitedcapi`` test helper. + +CPython's test suite reaches for this C extension to exercise the public +*abstract* object protocol from C. WeavePy has no C extensions, so we +provide faithful Python equivalents of the handful of wrappers the +conformance targets actually use. Each mirrors the corresponding +``PySequence_*`` C-API call, which for the built-in sequence types under +test is plain subscripting. +""" + + +def sequence_getitem(obj, i): + # PySequence_GetItem(obj, i) + return obj[i] + + +def sequence_setitem(obj, i, value): + # PySequence_SetItem(obj, i, value) + obj[i] = value + + +def sequence_delitem(obj, i): + # PySequence_DelItem(obj, i) + del obj[i] diff --git a/crates/weavepy-vm/src/stdlib/python/enum.py b/crates/weavepy-vm/src/stdlib/python/enum.py index 49de907..5e7b006 100644 --- a/crates/weavepy-vm/src/stdlib/python/enum.py +++ b/crates/weavepy-vm/src/stdlib/python/enum.py @@ -1,426 +1,2182 @@ -"""Python enumerations — small WeavePy-compatible subset. +import sys +import builtins as bltns +from functools import partial +from types import MappingProxyType, DynamicClassAttribute -Models the most-used surface of CPython's :mod:`enum`: -- :class:`Enum`, :class:`IntEnum`, :class:`Flag`, :class:`IntFlag` -- the :data:`auto` helper -- the :func:`unique` decorator -- ``Color.RED``, ``Color['RED']`` look-up -- iteration over members in declaration order -- ``Color(1)`` value-based lookup -- ``__str__`` / ``__repr__`` matching CPython's defaults -- Bitwise operations on :class:`Flag` / :class:`IntFlag` +__all__ = [ + 'EnumType', 'EnumMeta', 'EnumDict', + 'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag', 'ReprEnum', + 'auto', 'unique', 'property', 'verify', 'member', 'nonmember', + 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', + 'global_flag_repr', 'global_enum_repr', 'global_str', 'global_enum', + 'EnumCheck', 'CONTINUOUS', 'NAMED_FLAGS', 'UNIQUE', + 'pickle_by_global_name', 'pickle_by_enum_name', + ] + + +# Dummy value for Enum and Flag as there are explicit checks for them +# before they have been created. +# This is also why there are checks in EnumType like `if Enum is not None` +Enum = Flag = EJECT = _stdlib_enums = ReprEnum = None + +class nonmember(object): + """ + Protects item from becoming an Enum member during class creation. + """ + def __init__(self, value): + self.value = value -Not implemented: +class member(object): + """ + Forces item to become an Enum member during class creation. + """ + def __init__(self, value): + self.value = value -- ``_missing_`` hook for custom value coercion -- ``__init_subclass__`` integration with ``StrEnum`` (Python 3.11+) -- The ``_generate_next_value_`` customisation point -- Bare-class introspection helpers like ``Enum.__members__`` ordered - dict — we expose a plain dict in declaration order, which is - effectively the same on modern CPython. -""" +def _is_descriptor(obj): + """ + Returns True if obj is a descriptor, False otherwise. + """ + return not isinstance(obj, partial) and ( + hasattr(obj, '__get__') or + hasattr(obj, '__set__') or + hasattr(obj, '__delete__') + ) +def _is_dunder(name): + """ + Returns True if a __dunder__ name, False otherwise. + """ + return ( + len(name) > 4 and + name[:2] == name[-2:] == '__' and + name[2] != '_' and + name[-3] != '_' + ) + +def _is_sunder(name): + """ + Returns True if a _sunder_ name, False otherwise. + """ + return ( + len(name) > 2 and + name[0] == name[-1] == '_' and + name[1] != '_' and + name[-2] != '_' + ) + +def _is_internal_class(cls_name, obj): + # do not use `re` as `re` imports `enum` + if not isinstance(obj, type): + return False + qualname = getattr(obj, '__qualname__', '') + s_pattern = cls_name + '.' + getattr(obj, '__name__', '') + e_pattern = '.' + s_pattern + return qualname == s_pattern or qualname.endswith(e_pattern) + +def _is_private(cls_name, name): + # do not use `re` as `re` imports `enum` + pattern = '_%s__' % (cls_name, ) + pat_len = len(pattern) + if ( + len(name) > pat_len + and name.startswith(pattern) + and (name[-1] != '_' or name[-2] != '_') + ): + return True + else: + return False + +def _is_single_bit(num): + """ + True if only one bit set in num (should be an int) + """ + if num == 0: + return False + num &= num - 1 + return num == 0 + +def _make_class_unpicklable(obj): + """ + Make the given obj un-picklable. + + obj should be either a dictionary, or an Enum + """ + def _break_on_call_reduce(self, proto): + raise TypeError('%r cannot be pickled' % self) + if isinstance(obj, dict): + obj['__reduce_ex__'] = _break_on_call_reduce + obj['__module__'] = '' + else: + setattr(obj, '__reduce_ex__', _break_on_call_reduce) + setattr(obj, '__module__', '') + +def _iter_bits_lsb(num): + # num must be a positive integer + original = num + if isinstance(num, Enum): + num = num.value + if num < 0: + raise ValueError('%r is not a positive integer' % original) + while num: + b = num & (~num + 1) + yield b + num ^= b + +def show_flag_values(value): + return list(_iter_bits_lsb(value)) + +def bin(num, max_bits=None): + """ + Like built-in bin(), except negative values are represented in + twos-complement, and the leading bit always indicates sign + (0=positive, 1=negative). + + >>> bin(10) + '0b0 1010' + >>> bin(~10) # ~10 is -11 + '0b1 0101' + """ + + num = num.__index__() + ceiling = 2 ** (num).bit_length() + if num >= 0: + s = bltns.bin(num + ceiling).replace('1', '0', 1) + else: + s = bltns.bin(~num ^ (ceiling - 1) + ceiling) + sign = s[:3] + digits = s[3:] + if max_bits is not None: + if len(digits) < max_bits: + digits = (sign[-1] * max_bits + digits)[-max_bits:] + return "%s %s" % (sign, digits) + +def _dedent(text): + """ + Like textwrap.dedent. Rewritten because we cannot import textwrap. + """ + lines = text.split('\n') + for i, ch in enumerate(lines[0]): + if ch != ' ': + break + for j, l in enumerate(lines): + lines[j] = l[i:] + return '\n'.join(lines) + +class _not_given: + def __repr__(self): + return('') +_not_given = _not_given() + +class _auto_null: + def __repr__(self): + return '_auto_null' +_auto_null = _auto_null() class auto: - """Sentinel used inside an Enum class body to request - auto-numbered values. The replacement happens in - :class:`EnumMeta.__init__`.""" - - __slots__ = ("value",) - - def __init__(self): - self.value = None - - -def _next_power_of_two(n): - """Smallest power of two strictly greater than ``n - 1``. - - Used by :class:`Flag` to keep auto-generated values pure bit-flags - even when the user mixes explicit numeric values with ``auto()``. - """ - - if n <= 1: - return 1 - p = 1 - while p < n: - p *= 2 - return p - - -class EnumMeta(type): - """The metaclass for all :class:`Enum` subclasses.""" - - def __new__(mcs, name, bases, namespace, **kwargs): - # The bare Enum/Flag/IntEnum/IntFlag base classes are - # constructed before they themselves exist, so skip member - # collection when there is no concrete Enum base in `bases`. - # We use the presence of `_member_map_` *as a class attribute* - # (set during a previous EnumMeta.__new__) to detect that a - # base is an Enum-shaped class; whether the map is empty or - # populated doesn't matter — what matters is that the - # attribute exists. - enum_base = None - for b in bases: - if isinstance(b, EnumMeta) and "_member_map_" in b.__dict__: - enum_base = b - break - - members = {} - if enum_base is not None: - # Flag/IntFlag use power-of-2 auto values; plain Enum uses - # sequential integers. Detect FlagMeta lazily — the very - # first time we run, FlagMeta itself hasn't been defined - # yet, but no Flag bases exist either. - flag_meta = globals().get("FlagMeta") - is_flag_like = flag_meta is not None and issubclass(mcs, flag_meta) - next_value = 1 - for key, value in list(namespace.items()): - if key.startswith("_") and key.endswith("_"): - continue - if callable(value): - continue - if isinstance(value, (property, staticmethod, classmethod)): - continue - if isinstance(value, auto): - value = next_value - next_value = next_value * 2 if is_flag_like else next_value + 1 - else: - if isinstance(value, int): - next_value = ( - _next_power_of_two(value + 1) if is_flag_like else value + 1 + """ + Instances are replaced with an appropriate value in Enum class suites. + """ + def __init__(self, value=_auto_null): + self.value = value + + def __repr__(self): + return "auto(%r)" % self.value + +class property(DynamicClassAttribute): + """ + This is a descriptor, used to define attributes that act differently + when accessed through an enum member and through an enum class. + Instance access is the same as property(), but access to an attribute + through the enum class will instead look in the class' _member_map_ for + a corresponding enum member. + """ + + member = None + _attr_type = None + _cls_type = None + + def __get__(self, instance, ownerclass=None): + if instance is None: + if self.member is not None: + return self.member + else: + raise AttributeError( + '%r has no attribute %r' % (ownerclass, self.name) ) - members[key] = value - for key in members: - namespace.pop(key, None) - - cls = super().__new__(mcs, name, bases, namespace, **kwargs) - - if enum_base is None: - cls._member_map_ = None - cls._value2member_map_ = None - return cls - - cls._member_map_ = {} - cls._value2member_map_ = {} - # `_member_names_` is the canonical list (no aliases). CPython - # exposes it as the iteration order for `for x in Enum:` and - # for `list(Enum)` — aliases never appear there. - cls._member_names_ = [] - for member_name, member_value in members.items(): + if self.fget is not None: + # use previous enum.property + return self.fget(instance) + elif self._attr_type == 'attr': + # look up previous attibute + return getattr(self._cls_type, self.name) + elif self._attr_type == 'desc': + # use previous descriptor + return getattr(instance._value_, self.name) + # look for a member by this name. + try: + return ownerclass._member_map_[self.name] + except KeyError: + raise AttributeError( + '%r has no attribute %r' % (ownerclass, self.name) + ) from None + + def __set__(self, instance, value): + if self.fset is not None: + return self.fset(instance, value) + raise AttributeError( + " cannot set attribute %r" % (self.clsname, self.name) + ) + + def __delete__(self, instance): + if self.fdel is not None: + return self.fdel(instance) + raise AttributeError( + " cannot delete attribute %r" % (self.clsname, self.name) + ) + + def __set_name__(self, ownerclass, name): + self.name = name + self.clsname = ownerclass.__name__ + + +class _proto_member: + """ + intermediate step for enum members between class execution and final creation + """ + + def __init__(self, value): + self.value = value + + def __set_name__(self, enum_class, member_name): + """ + convert each quasi-member into an instance of the new enum class + """ + # first step: remove ourself from enum_class + delattr(enum_class, member_name) + # second step: create member based on enum_class + value = self.value + if not isinstance(value, tuple): + args = (value, ) + else: + args = value + if enum_class._member_type_ is tuple: # special case for tuple enums + args = (args, ) # wrap it one more time + if not enum_class._use_args_: + enum_member = enum_class._new_member_(enum_class) + else: + enum_member = enum_class._new_member_(enum_class, *args) + if not hasattr(enum_member, '_value_'): + if enum_class._member_type_ is object: + enum_member._value_ = value + else: + try: + enum_member._value_ = enum_class._member_type_(*args) + except Exception as exc: + new_exc = TypeError( + '_value_ not set in __new__, unable to create it' + ) + new_exc.__cause__ = exc + raise new_exc + value = enum_member._value_ + enum_member._name_ = member_name + enum_member.__objclass__ = enum_class + enum_member.__init__(*args) + enum_member._sort_order_ = len(enum_class._member_names_) + + if Flag is not None and issubclass(enum_class, Flag): + if isinstance(value, int): + enum_class._flag_mask_ |= value + if _is_single_bit(value): + enum_class._singles_mask_ |= value + enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1 + + # If another member with the same value was already defined, the + # new member becomes an alias to the existing one. + try: try: - existing = member_value in cls._value2member_map_ + # try to do a fast lookup to avoid the quadratic loop + enum_member = enum_class._value2member_map_[value] except TypeError: - existing = False - if existing: - # Aliases: bind the same member at the class level so - # ``MyEnum.ALIAS is MyEnum.ORIGINAL`` works, but - # don't add to `_member_names_` so iteration skips it. - cls._member_map_[member_name] = cls._value2member_map_[member_value] - setattr(cls, member_name, cls._value2member_map_[member_value]) - continue - member = cls._create_member_(member_name, member_value) - cls._member_map_[member_name] = member - cls._value2member_map_[member_value] = member - cls._member_names_.append(member_name) - setattr(cls, member_name, member) - return cls + for name, canonical_member in enum_class._member_map_.items(): + if canonical_member._value_ == value: + enum_member = canonical_member + break + else: + raise KeyError + except KeyError: + # this could still be an alias if the value is multi-bit and the + # class is a flag class + if ( + Flag is None + or not issubclass(enum_class, Flag) + ): + # no other instances found, record this member in _member_names_ + enum_class._member_names_.append(member_name) + elif ( + Flag is not None + and issubclass(enum_class, Flag) + and isinstance(value, int) + and _is_single_bit(value) + ): + # no other instances found, record this member in _member_names_ + enum_class._member_names_.append(member_name) + + enum_class._add_member_(member_name, enum_member) + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_.setdefault(value, enum_member) + if value not in enum_class._hashable_values_: + enum_class._hashable_values_.append(value) + except TypeError: + # keep track of the value in a list so containment checks are quick + enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(member_name, []).append(value) + + +class EnumDict(dict): + """ + Track enum member order and ensure member names are not reused. - def __call__(cls, value=None, *args, **kwargs): - # Member lookup form: `Color(1)`. - if cls._member_map_ is not None and not args and not kwargs: - if value in cls._value2member_map_: - return cls._value2member_map_[value] - if isinstance(cls, FlagMeta): - return cls._decompose_flag(value) - raise ValueError(f"{value!r} is not a valid {cls.__name__}") - # Functional API: `Color = Enum('Color', 'RED GREEN BLUE')` - if args or kwargs: - return cls._create_(value, *args, **kwargs) - return super().__call__(value) + EnumType will use the names found in self._member_names as the + enumeration member names. + """ + def __init__(self, cls_name=None): + super().__init__() + self._member_names = {} # use a dict -- faster look-up than a list, and keeps insertion order since 3.7 + self._last_values = [] + self._ignore = [] + self._auto_called = False + self._cls_name = cls_name + + def __setitem__(self, key, value): + """ + Changes anything not dundered or not a descriptor. + + If an enum member name is used twice, an error is raised; duplicate + values are not checked for. + + Single underscore (sunder) names are reserved. + """ + if self._cls_name is not None and _is_private(self._cls_name, key): + # do nothing, name will be a normal attribute + pass + elif _is_sunder(key): + if key not in ( + '_order_', + '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', + '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', + '_add_alias_', '_add_value_alias_', + # While not in use internally, those are common for pretty + # printing and thus excluded from Enum's reservation of + # _sunder_ names + ) and not key.startswith('_repr_'): + raise ValueError( + '_sunder_ names, such as %r, are reserved for future Enum use' + % (key, ) + ) + if key == '_generate_next_value_': + # check if members already defined as auto() + if self._auto_called: + raise TypeError("_generate_next_value_ must be defined before members") + _gnv = value.__func__ if isinstance(value, staticmethod) else value + setattr(self, '_generate_next_value', _gnv) + elif key == '_ignore_': + if isinstance(value, str): + value = value.replace(',',' ').split() + else: + value = list(value) + self._ignore = value + already = set(value) & set(self._member_names) + if already: + raise ValueError( + '_ignore_ cannot specify already set names: %r' + % (already, ) + ) + elif _is_dunder(key): + if key == '__order__': + key = '_order_' + elif key in self._member_names: + # descriptor overwriting an enum? + raise TypeError('%r already defined as %r' % (key, self[key])) + elif key in self._ignore: + pass + elif isinstance(value, nonmember): + # unwrap value here; it won't be processed by the below `else` + value = value.value + elif isinstance(value, partial): + import warnings + warnings.warn('functools.partial will be a method descriptor ' + 'in future Python versions; wrap it in ' + 'enum.member() if you want to preserve the ' + 'old behavior', FutureWarning, stacklevel=2) + elif _is_descriptor(value): + pass + elif self._cls_name is not None and _is_internal_class(self._cls_name, value): + # do nothing, name will be a normal attribute + pass + else: + if key in self: + # enum overwriting a descriptor? + raise TypeError('%r already defined as %r' % (key, self[key])) + elif isinstance(value, member): + # unwrap value here -- it will become a member + value = value.value + non_auto_store = True + single = False + if isinstance(value, auto): + single = True + value = (value, ) + if isinstance(value, tuple) and any(isinstance(v, auto) for v in value): + # insist on an actual tuple, no subclasses, in keeping with only supporting + # top-level auto() usage (not contained in any other data structure) + auto_valued = [] + t = type(value) + for v in value: + if isinstance(v, auto): + non_auto_store = False + if v.value == _auto_null: + v.value = self._generate_next_value( + key, 1, len(self._member_names), self._last_values[:], + ) + self._auto_called = True + v = v.value + self._last_values.append(v) + auto_valued.append(v) + if single: + value = auto_valued[0] + else: + try: + # accepts iterable as multiple arguments? + value = t(auto_valued) + except TypeError: + # then pass them in singly + value = t(*auto_valued) + self._member_names[key] = None + if non_auto_store: + self._last_values.append(value) + super().__setitem__(key, value) + + @property + def member_names(self): + return list(self._member_names) + + def update(self, members, **more_members): + try: + for name in members.keys(): + self[name] = members[name] + except AttributeError: + for name, value in members: + self[name] = value + for name, value in more_members.items(): + self[name] = value + +_EnumDict = EnumDict # keep private name for backwards compatibility + + +class EnumType(type): + """ + Metaclass for Enum + """ + + @classmethod + def __prepare__(metacls, cls, bases, **kwds): + # check that previous enum members do not exist + metacls._check_for_existing_members_(cls, bases) + # create the namespace dict + enum_dict = EnumDict(cls) + # inherit previous flags and _generate_next_value_ function + member_type, first_enum = metacls._get_mixins_(cls, bases) + if first_enum is not None: + enum_dict['_generate_next_value_'] = getattr( + first_enum, '_generate_next_value_', None, + ) + return enum_dict + + def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **kwds): + # an Enum class is final once enumeration items have been defined; it + # cannot be mixed with other types (int, float, etc.) if it has an + # inherited __new__ unless a new __new__ is defined (or the resulting + # class will fail). + # + if _simple: + return super().__new__(metacls, cls, bases, classdict, **kwds) + # + # remove any keys listed in _ignore_ + classdict.setdefault('_ignore_', []).append('_ignore_') + ignore = classdict['_ignore_'] + for key in ignore: + classdict.pop(key, None) + # + # grab member names + member_names = classdict._member_names + # + # check for illegal enum names (any others?) + invalid_names = set(member_names) & {'mro', ''} + if invalid_names: + raise ValueError('invalid enum member name(s) %s' % ( + ','.join(repr(n) for n in invalid_names) + )) + # + # adjust the sunders + _order_ = classdict.pop('_order_', None) + _gnv = classdict.get('_generate_next_value_') + if _gnv is not None and type(_gnv) is not staticmethod: + _gnv = staticmethod(_gnv) + # convert to normal dict + classdict = dict(classdict.items()) + if _gnv is not None: + classdict['_generate_next_value_'] = _gnv + # + # data type of member and the controlling Enum class + member_type, first_enum = metacls._get_mixins_(cls, bases) + __new__, save_new, use_args = metacls._find_new_( + classdict, member_type, first_enum, + ) + classdict['_new_member_'] = __new__ + classdict['_use_args_'] = use_args + # + # convert future enum members into temporary _proto_members + for name in member_names: + value = classdict[name] + classdict[name] = _proto_member(value) + # + # house-keeping structures + classdict['_member_names_'] = [] + classdict['_member_map_'] = {} + classdict['_value2member_map_'] = {} + classdict['_hashable_values_'] = [] # for comparing with non-hashable types + classdict['_unhashable_values_'] = [] # e.g. frozenset() with set() + classdict['_unhashable_values_map_'] = {} + classdict['_member_type_'] = member_type + # now set the __repr__ for the value + classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases) + # + # Flag structures (will be removed if final class is not a Flag + classdict['_boundary_'] = ( + boundary + or getattr(first_enum, '_boundary_', None) + ) + classdict['_flag_mask_'] = 0 + classdict['_singles_mask_'] = 0 + classdict['_all_bits_'] = 0 + classdict['_inverted_'] = None + try: + classdict['_%s__in_progress' % cls] = True + enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) + classdict['_%s__in_progress' % cls] = False + delattr(enum_class, '_%s__in_progress' % cls) + except Exception as e: + # since 3.12 the note "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError, so discard it + if hasattr(e, '__notes__'): + del e.__notes__ + raise + # update classdict with any changes made by __init_subclass__ + classdict.update(enum_class.__dict__) + # + # double check that repr and friends are not the mixin's or various + # things break (such as pickle) + # however, if the method is defined in the Enum itself, don't replace + # it + # + # Also, special handling for ReprEnum + if ReprEnum is not None and ReprEnum in bases: + if member_type is object: + raise TypeError( + 'ReprEnum subclasses must be mixed with a data type (i.e.' + ' int, str, float, etc.)' + ) + if '__format__' not in classdict: + enum_class.__format__ = member_type.__format__ + classdict['__format__'] = enum_class.__format__ + if '__str__' not in classdict: + method = member_type.__str__ + if method is object.__str__: + # if member_type does not define __str__, object.__str__ will use + # its __repr__ instead, so we'll also use its __repr__ + method = member_type.__repr__ + enum_class.__str__ = method + classdict['__str__'] = enum_class.__str__ + for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'): + if name not in classdict: + # check for mixin overrides before replacing + enum_method = getattr(first_enum, name) + found_method = getattr(enum_class, name) + object_method = getattr(object, name) + data_type_method = getattr(member_type, name) + if found_method in (data_type_method, object_method): + setattr(enum_class, name, enum_method) + # + # for Flag, add __or__, __and__, __xor__, and __invert__ + if Flag is not None and issubclass(enum_class, Flag): + for name in ( + '__or__', '__and__', '__xor__', + '__ror__', '__rand__', '__rxor__', + '__invert__' + ): + if name not in classdict: + enum_method = getattr(Flag, name) + setattr(enum_class, name, enum_method) + classdict[name] = enum_method + # + # replace any other __new__ with our own (as long as Enum is not None, + # anyway) -- again, this is to support pickle + if Enum is not None: + # if the user defined their own __new__, save it before it gets + # clobbered in case they subclass later + if save_new: + enum_class.__new_member__ = __new__ + enum_class.__new__ = Enum.__new__ + # + # py3 support for definition order (helps keep py2/py3 code in sync) + # + # _order_ checking is spread out into three/four steps + # - if enum_class is a Flag: + # - remove any non-single-bit flags from _order_ + # - remove any aliases from _order_ + # - check that _order_ and _member_names_ match + # + # step 1: ensure we have a list + if _order_ is not None: + if isinstance(_order_, str): + _order_ = _order_.replace(',', ' ').split() + # + # remove Flag structures if final class is not a Flag + if ( + Flag is None and cls != 'Flag' + or Flag is not None and not issubclass(enum_class, Flag) + ): + delattr(enum_class, '_boundary_') + delattr(enum_class, '_flag_mask_') + delattr(enum_class, '_singles_mask_') + delattr(enum_class, '_all_bits_') + delattr(enum_class, '_inverted_') + elif Flag is not None and issubclass(enum_class, Flag): + # set correct __iter__ + member_list = [m._value_ for m in enum_class] + if member_list != sorted(member_list): + enum_class._iter_member_ = enum_class._iter_member_by_def_ + if _order_: + # _order_ step 2: remove any items from _order_ that are not single-bit + _order_ = [ + o + for o in _order_ + if o not in enum_class._member_map_ or _is_single_bit(enum_class[o]._value_) + ] + # + if _order_: + # _order_ step 3: remove aliases from _order_ + _order_ = [ + o + for o in _order_ + if ( + o not in enum_class._member_map_ + or + (o in enum_class._member_map_ and o in enum_class._member_names_) + )] + # _order_ step 4: verify that _order_ and _member_names_ match + if _order_ != enum_class._member_names_: + raise TypeError( + 'member order does not match _order_:\n %r\n %r' + % (enum_class._member_names_, _order_) + ) + # + return enum_class + + def __bool__(cls): + """ + classes/types should always be True. + """ + return True + + def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, type=None, start=1, boundary=None): + """ + Either returns an existing member, or creates a new enum class. + + This method is used both when an enum class is given a value to match + to an enumeration member (i.e. Color(3)) and for the functional API + (i.e. Color = Enum('Color', names='RED GREEN BLUE')). + + The value lookup branch is chosen if the enum is final. + + When used for the functional API: + + `value` will be the name of the new class. + + `names` should be either a string of white-space/comma delimited names + (values will start at `start`), or an iterator/mapping of name, value pairs. + + `module` should be set to the module this class is being created in; + if it is not set, an attempt to find that module will be made, but if + it fails the class will not be picklable. + + `qualname` should be set to the actual location this class can be found + at in its module; by default it is set to the global scope. If this is + not correct, unpickling will fail in some circumstances. + + `type`, if set, will be mixed in as the first base class. + """ + if cls._member_map_: + # simple value lookup if members exist + if names is not _not_given: + value = (value, names) + values + return cls.__new__(cls, value) + # otherwise, functional API: we're creating a new Enum type + if names is _not_given and type is None: + # no body? no data-type? possibly wrong usage + raise TypeError( + f"{cls} has no members; specify `names=()` if you meant to create a new, empty, enum" + ) + return cls._create_( + class_name=value, + names=None if names is _not_given else names, + module=module, + qualname=qualname, + type=type, + start=start, + boundary=boundary, + ) + + def __contains__(cls, value): + """Return True if `value` is in `cls`. + + `value` is in `cls` if: + 1) `value` is a member of `cls`, or + 2) `value` is the value of one of the `cls`'s members. + 3) `value` is a pseudo-member (flags) + """ + if isinstance(value, cls): + return True + if issubclass(cls, Flag): + try: + result = cls._missing_(value) + return isinstance(result, cls) + except ValueError: + pass + return ( + value in cls._unhashable_values_ # both structures are lists + or value in cls._hashable_values_ + ) + + def __delattr__(cls, attr): + # nicer error message when someone tries to delete an attribute + # (see issue19025). + if attr in cls._member_map_: + raise AttributeError("%r cannot delete member %r." % (cls.__name__, attr)) + super().__delattr__(attr) + + def __dir__(cls): + interesting = set([ + '__class__', '__contains__', '__doc__', '__getitem__', + '__iter__', '__len__', '__members__', '__module__', + '__name__', '__qualname__', + ] + + cls._member_names_ + ) + if cls._new_member_ is not object.__new__: + interesting.add('__new__') + if cls.__init_subclass__ is not object.__init_subclass__: + interesting.add('__init_subclass__') + if cls._member_type_ is object: + return sorted(interesting) + else: + # return whatever mixed-in data type has + return sorted(set(dir(cls._member_type_)) | interesting) def __getitem__(cls, name): - if cls._member_map_ is None: - raise KeyError(name) + """ + Return the member matching `name`. + """ return cls._member_map_[name] def __iter__(cls): - if cls._member_map_ is None: - return iter(()) - # Iterate canonical names only — aliases are intentionally - # skipped to match CPython's `for member in Enum:` semantics. - names = getattr(cls, "_member_names_", None) or list(cls._member_map_.keys()) - return iter(cls._member_map_[n] for n in names) + """ + Return members in definition order. + """ + return (cls._member_map_[name] for name in cls._member_names_) def __len__(cls): - if cls._member_map_ is None: - return 0 - names = getattr(cls, "_member_names_", None) - return len(names) if names is not None else len(cls._member_map_) + """ + Return the number of members (no aliases) + """ + return len(cls._member_names_) - def __contains__(cls, member): - return cls._member_map_ is not None and member in cls._member_map_.values() - - @property + @bltns.property def __members__(cls): - return dict(cls._member_map_) if cls._member_map_ is not None else {} - - def _create_member_(cls, name, value): - # For int-backed enums (IntEnum/IntFlag) build a real int - # instance so members *are* ints — `IntEnum.X + 1`, `flags & - # member`, `int(member)` and dict/set interchange with the - # bare value all work exactly as in CPython. - if isinstance(value, int) and issubclass(cls, int): - member = int.__new__(cls, value) - else: - member = object.__new__(cls) - member._name_ = name - member._value_ = value - return member + """ + Returns a mapping of member name->value. + + This mapping lists all enum members, including aliases. Note that this + is a read-only view of the internal mapping. + """ + return MappingProxyType(cls._member_map_) - def _create_(cls, name, names, module=None, qualname=None, type_=None, start=1): - # Minimal Functional API: accepts a string of space/comma- - # separated names, or an iterable. + def __repr__(cls): + if Flag is not None and issubclass(cls, Flag): + return "" % cls.__name__ + else: + return "" % cls.__name__ + + def __reversed__(cls): + """ + Return members in reverse definition order. + """ + return (cls._member_map_[name] for name in reversed(cls._member_names_)) + + def __setattr__(cls, name, value): + """ + Block attempts to reassign Enum members. + + A simple assignment to the class namespace only changes one of the + several possible ways to get an Enum member from the Enum class, + resulting in an inconsistent Enumeration. + """ + member_map = cls.__dict__.get('_member_map_', {}) + if name in member_map: + raise AttributeError('cannot reassign member %r' % (name, )) + super().__setattr__(name, value) + + def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1, boundary=None): + """ + Convenience method to create a new Enum class. + + `names` can be: + + * A string containing member names, separated either with spaces or + commas. Values are incremented by 1 from `start`. + * An iterable of member names. Values are incremented by 1 from `start`. + * An iterable of (member name, value) pairs. + * A mapping of member name -> value pairs. + """ + metacls = cls.__class__ + bases = (cls, ) if type is None else (type, cls) + _, first_enum = cls._get_mixins_(class_name, bases) + classdict = metacls.__prepare__(class_name, bases) + + # special processing needed for names? if isinstance(names, str): - names = names.replace(",", " ").split() - bases = (cls,) if type_ is None else (type_, cls) - body = {} - next_val = start - for entry in names: - if isinstance(entry, tuple): - key, val = entry + names = names.replace(',', ' ').split() + if isinstance(names, (tuple, list)) and names and isinstance(names[0], str): + original_names, names = names, [] + last_values = [] + for count, name in enumerate(original_names): + value = first_enum._generate_next_value_(name, start, count, last_values[:]) + last_values.append(value) + names.append((name, value)) + if names is None: + names = () + + # Here, names is either an iterable of (name, value) or a mapping. + for item in names: + if isinstance(item, str): + member_name, member_value = item, names[item] else: - key, val = entry, next_val - next_val += 1 - body[key] = val - new_cls = EnumMeta(name, bases, body) - return new_cls + member_name, member_value = item + classdict[member_name] = member_value + if module is None: + try: + module = sys._getframemodulename(2) + except AttributeError: + # Fall back on _getframe if _getframemodulename is missing + try: + module = sys._getframe(2).f_globals['__name__'] + except (AttributeError, ValueError, KeyError): + pass + if module is None: + _make_class_unpicklable(classdict) + else: + classdict['__module__'] = module + if qualname is not None: + classdict['__qualname__'] = qualname + + return metacls.__new__(metacls, class_name, bases, classdict, boundary=boundary) + + def _convert_(cls, name, module, filter, source=None, *, boundary=None, as_global=False): + """ + Create a new Enum subclass that replaces a collection of global constants + """ + # convert all constants from source (or module) that pass filter() to + # a new Enum called name, and export the enum and its members back to + # module; + # also, replace the __reduce_ex__ method so unpickling works in + # previous Python versions + module_globals = sys.modules[module].__dict__ + if source: + source = source.__dict__ + else: + source = module_globals + # _value2member_map_ is populated in the same order every time + # for a consistent reverse mapping of number to name when there + # are multiple names for the same number. + members = [ + (name, value) + for name, value in source.items() + if filter(name)] + try: + # sort by value + members.sort(key=lambda t: (t[1], t[0])) + except TypeError: + # unless some values aren't comparable, in which case sort by name + members.sort(key=lambda t: t[0]) + body = {t[0]: t[1] for t in members} + body['__module__'] = module + tmp_cls = type(name, (object, ), body) + cls = _simple_enum(etype=cls, boundary=boundary or KEEP)(tmp_cls) + if as_global: + global_enum(cls) + else: + sys.modules[cls.__module__].__dict__.update(cls.__members__) + module_globals[name] = cls + return cls -class Enum(metaclass=EnumMeta): - @property - def name(self): - return self._name_ + @classmethod + def _check_for_existing_members_(mcls, class_name, bases): + for chain in bases: + for base in chain.__mro__: + if isinstance(base, EnumType) and base._member_names_: + raise TypeError( + " cannot extend %r" + % (class_name, base) + ) + + @classmethod + def _get_mixins_(mcls, class_name, bases): + """ + Returns the type for creating enum members, and the first inherited + enum class. + + bases: the tuple of bases that was given to __new__ + """ + if not bases: + return object, Enum + # ensure final parent class is an Enum derivative, find any concrete + # data type, and check that Enum has no members + first_enum = bases[-1] + if not isinstance(first_enum, EnumType): + raise TypeError("new enumerations should be created as " + "`EnumName([mixin_type, ...] [data_type,] enum_type)`") + member_type = mcls._find_data_type_(class_name, bases) or object + return member_type, first_enum + + @classmethod + def _find_data_repr_(mcls, class_name, bases): + for chain in bases: + for base in chain.__mro__: + if base is object: + continue + elif isinstance(base, EnumType): + # if we hit an Enum, use it's _value_repr_ + return base._value_repr_ + elif '__repr__' in base.__dict__: + # this is our data repr + # double-check if a dataclass with a default __repr__ + if ( + '__dataclass_fields__' in base.__dict__ + and '__dataclass_params__' in base.__dict__ + and base.__dict__['__dataclass_params__'].repr + ): + return _dataclass_repr + else: + return base.__dict__['__repr__'] + return None + + @classmethod + def _find_data_type_(mcls, class_name, bases): + # a datatype has a __new__ method, or a __dataclass_fields__ attribute + data_types = set() + base_chain = set() + for chain in bases: + candidate = None + for base in chain.__mro__: + base_chain.add(base) + if base is object: + continue + elif isinstance(base, EnumType): + if base._member_type_ is not object: + data_types.add(base._member_type_) + break + elif '__new__' in base.__dict__ or '__dataclass_fields__' in base.__dict__: + data_types.add(candidate or base) + break + else: + candidate = candidate or base + if len(data_types) > 1: + raise TypeError('too many data types for %r: %r' % (class_name, data_types)) + elif data_types: + return data_types.pop() + else: + return None + + @classmethod + def _find_new_(mcls, classdict, member_type, first_enum): + """ + Returns the __new__ to be used for creating the enum members. + + classdict: the class dictionary given to __new__ + member_type: the data type whose __new__ will be used by default + first_enum: enumeration to check for an overriding __new__ + """ + # now find the correct __new__, checking to see of one was defined + # by the user; also check earlier enum classes in case a __new__ was + # saved as __new_member__ + __new__ = classdict.get('__new__', None) + + # should __new__ be saved as __new_member__ later? + save_new = first_enum is not None and __new__ is not None + + if __new__ is None: + # check all possibles for __new_member__ before falling back to + # __new__ + for method in ('__new_member__', '__new__'): + for possible in (member_type, first_enum): + target = getattr(possible, method, None) + if target not in { + None, + None.__new__, + object.__new__, + Enum.__new__, + }: + __new__ = target + break + if __new__ is not None: + break + else: + __new__ = object.__new__ - @property - def value(self): - return self._value_ + # if a non-object.__new__ is used then whatever value/tuple was + # assigned to the enum member name will be passed to __new__ and to the + # new enum member's __init__ + if first_enum is None or __new__ in (Enum.__new__, object.__new__): + use_args = False + else: + use_args = True + return __new__, save_new, use_args + + def _add_member_(cls, name, member): + # _value_ structures are not updated + if name in cls._member_map_: + if cls._member_map_[name] is not member: + raise NameError('%r is already bound: %r' % (name, cls._member_map_[name])) + return + # + # if necessary, get redirect in place and then add it to _member_map_ + found_descriptor = None + descriptor_type = None + class_type = None + for base in cls.__mro__[1:]: + attr = base.__dict__.get(name) + if attr is not None: + if isinstance(attr, (property, DynamicClassAttribute)): + found_descriptor = attr + class_type = base + descriptor_type = 'enum' + break + elif _is_descriptor(attr): + found_descriptor = attr + descriptor_type = descriptor_type or 'desc' + class_type = class_type or base + continue + else: + descriptor_type = 'attr' + class_type = base + if found_descriptor: + redirect = property() + redirect.member = member + redirect.__set_name__(cls, name) + if descriptor_type in ('enum', 'desc'): + # earlier descriptor found; copy fget, fset, fdel to this one. + redirect.fget = getattr(found_descriptor, 'fget', None) + redirect._get = getattr(found_descriptor, '__get__', None) + redirect.fset = getattr(found_descriptor, 'fset', None) + redirect._set = getattr(found_descriptor, '__set__', None) + redirect.fdel = getattr(found_descriptor, 'fdel', None) + redirect._del = getattr(found_descriptor, '__delete__', None) + redirect._attr_type = descriptor_type + redirect._cls_type = class_type + setattr(cls, name, redirect) + else: + setattr(cls, name, member) + # now add to _member_map_ (even aliases) + cls._member_map_[name] = member - def __repr__(self): - return f"<{type(self).__name__}.{self._name_}: {self._value_!r}>" +EnumMeta = EnumType # keep EnumMeta name for backwards compatibility - def __str__(self): - return f"{type(self).__name__}.{self._name_}" - def __eq__(self, other): - if isinstance(other, Enum): - return self is other - return NotImplemented +class Enum(metaclass=EnumType): + """ + Create a collection of name/value pairs. + + Example enumeration: + + >>> class Color(Enum): + ... RED = 1 + ... BLUE = 2 + ... GREEN = 3 - def __ne__(self, other): - return not self.__eq__(other) + Access them by: + + - attribute access: + + >>> Color.RED + + + - value lookup: + + >>> Color(1) + + + - name lookup: + + >>> Color['RED'] + + + Enumerations can be iterated over, and know how many members they have: + + >>> len(Color) + 3 + + >>> list(Color) + [, , ] + + Methods can be added to enumerations, and members can have their own + attributes -- see the documentation for details. + """ + + @classmethod + def __signature__(cls): + if cls._member_names_: + return '(*values)' + else: + return '(new_class_name, /, names, *, module=None, qualname=None, type=None, start=1, boundary=None)' + + def __new__(cls, value): + # all enum instances are actually created during class construction + # without calling this method; this method is called by the metaclass' + # __call__ (i.e. Color(3) ), and by pickle + if type(value) is cls: + # For lookups like Color(Color.RED) + return value + # by-value search for a matching enum member + # see if it's in the reverse mapping (for hashable values) + try: + return cls._value2member_map_[value] + except KeyError: + # Not found, no need to do long O(n) search + pass + except TypeError: + # not there, now do long search -- O(n) behavior + for name, unhashable_values in cls._unhashable_values_map_.items(): + if value in unhashable_values: + return cls[name] + for name, member in cls._member_map_.items(): + if value == member._value_: + return cls[name] + # still not found -- verify that members exist, in-case somebody got here mistakenly + # (such as via super when trying to override __new__) + if not cls._member_map_: + if getattr(cls, '_%s__in_progress' % cls.__name__, False): + raise TypeError('do not use `super().__new__; call the appropriate __new__ directly') from None + raise TypeError("%r has no members defined" % cls) + # + # still not found -- try _missing_ hook + try: + exc = None + result = cls._missing_(value) + except Exception as e: + exc = e + result = None + try: + if isinstance(result, cls): + return result + elif ( + Flag is not None and issubclass(cls, Flag) + and cls._boundary_ is EJECT and isinstance(result, int) + ): + return result + else: + ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__)) + if result is None and exc is None: + raise ve_exc + elif exc is None: + exc = TypeError( + 'error in %s._missing_: returned %r instead of None or a valid member' + % (cls.__name__, result) + ) + if not isinstance(exc, ValueError): + exc.__context__ = ve_exc + raise exc + finally: + # ensure all variables that could hold an exception are destroyed + exc = None + ve_exc = None + + def __init__(self, *args, **kwds): + pass + + def _add_alias_(self, name): + self.__class__._add_member_(name, self) + + def _add_value_alias_(self, value): + cls = self.__class__ + try: + if value in cls._value2member_map_: + if cls._value2member_map_[value] is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + except TypeError: + # unhashable value, do long search + for m in cls._member_map_.values(): + if m._value_ == value: + if m is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + cls._value2member_map_.setdefault(value, self) + cls._hashable_values_.append(value) + except TypeError: + # keep track of the value in a list so containment checks are quick + cls._unhashable_values_.append(value) + cls._unhashable_values_map_.setdefault(self.name, []).append(value) + + @staticmethod + def _generate_next_value_(name, start, count, last_values): + """ + Generate the next value when not given. + + name: the name of the member + start: the initial start value or None + count: the number of existing members + last_values: the list of values assigned + """ + if not last_values: + return start + try: + last_value = sorted(last_values).pop() + except TypeError: + raise TypeError('unable to sort non-numeric values') from None + try: + return last_value + 1 + except TypeError: + raise TypeError('unable to increment %r' % (last_value, )) from None + + @classmethod + def _missing_(cls, value): + return None + + def __repr__(self): + v_repr = self.__class__._value_repr_ or repr + return "<%s.%s: %s>" % (self.__class__.__name__, self._name_, v_repr(self._value_)) + + def __str__(self): + return "%s.%s" % (self.__class__.__name__, self._name_, ) + + def __dir__(self): + """ + Returns public methods and other interesting attributes. + """ + interesting = set() + if self.__class__._member_type_ is not object: + interesting = set(object.__dir__(self)) + for name in getattr(self, '__dict__', []): + if name[0] != '_' and name not in self._member_map_: + interesting.add(name) + for cls in self.__class__.mro(): + for name, obj in cls.__dict__.items(): + if name[0] == '_': + continue + if isinstance(obj, property): + # that's an enum.property + if obj.fget is not None or name not in self._member_map_: + interesting.add(name) + else: + # in case it was added by `dir(self)` + interesting.discard(name) + elif name not in self._member_map_: + interesting.add(name) + names = sorted( + set(['__class__', '__doc__', '__eq__', '__hash__', '__module__']) + | interesting + ) + return names + + def __format__(self, format_spec): + return str.__format__(str(self), format_spec) def __hash__(self): return hash(self._name_) + def __reduce_ex__(self, proto): + return self.__class__, (self._value_, ) + + def __deepcopy__(self,memo): + return self + + def __copy__(self): + return self + + # enum.property is used to provide access to the `name` and + # `value` attributes of enum members while keeping some measure of + # protection from modification, while still allowing for an enumeration + # to have members named `name` and `value`. This works because each + # instance of enum.property saves its companion member, which it returns + # on class lookup; on instance lookup it either executes a provided function + # or raises an AttributeError. -class IntEnum(int, Enum): - """Mirror of :class:`Enum` whose members are also genuine ints, so - they compare and operate exactly like their integer value - (CPython's ``class IntEnum(int, Enum)``).""" + @property + def name(self): + """The name of the Enum member.""" + return self._name_ - def __int__(self): + @property + def value(self): + """The value of the Enum member.""" return self._value_ - def __eq__(self, other): - if isinstance(other, IntEnum): - return self._value_ == other._value_ - if isinstance(other, int): - return self._value_ == other - return NotImplemented - def __ne__(self, other): - eq = self.__eq__(other) - if eq is NotImplemented: - return NotImplemented - return not eq +class ReprEnum(Enum): + """ + Only changes the repr(), leaving str() and format() to the mixed-in type. + """ - def __hash__(self): - return hash(self._value_) - def __add__(self, other): - return self._value_ + (other._value_ if isinstance(other, IntEnum) else other) +class IntEnum(int, ReprEnum): + """ + Enum where members are also (and must be) ints + """ - def __radd__(self, other): - return other + self._value_ - def __sub__(self, other): - return self._value_ - (other._value_ if isinstance(other, IntEnum) else other) +class StrEnum(str, ReprEnum): + """ + Enum where members are also (and must be) strings + """ - def __rsub__(self, other): - return other - self._value_ + def __new__(cls, *values): + "values must already be of type `str`" + if len(values) > 3: + raise TypeError('too many arguments for str(): %r' % (values, )) + if len(values) == 1: + # it must be a string + if not isinstance(values[0], str): + raise TypeError('%r is not a string' % (values[0], )) + if len(values) >= 2: + # check that encoding argument is a string + if not isinstance(values[1], str): + raise TypeError('encoding must be a string, not %r' % (values[1], )) + if len(values) == 3: + # check that errors argument is a string + if not isinstance(values[2], str): + raise TypeError('errors must be a string, not %r' % (values[2])) + value = str(*values) + member = str.__new__(cls, value) + member._value_ = value + return member - def __mul__(self, other): - return self._value_ * (other._value_ if isinstance(other, IntEnum) else other) + @staticmethod + def _generate_next_value_(name, start, count, last_values): + """ + Return the lower-cased version of the member name. + """ + return name.lower() - def __rmul__(self, other): - return other * self._value_ - def __index__(self): - return self._value_ +def pickle_by_global_name(self, proto): + # should not be used with Flag-type enums + return self.name +_reduce_ex_by_global_name = pickle_by_global_name - def __lt__(self, other): - if isinstance(other, IntEnum): - return self._value_ < other._value_ - if isinstance(other, int): - return self._value_ < other - return NotImplemented +def pickle_by_enum_name(self, proto): + # should not be used with Flag-type enums + return getattr, (self.__class__, self._name_) - def __le__(self, other): - if isinstance(other, IntEnum): - return self._value_ <= other._value_ - if isinstance(other, int): - return self._value_ <= other - return NotImplemented +class FlagBoundary(StrEnum): + """ + control how out of range values are handled + "strict" -> error is raised [default for Flag] + "conform" -> extra bits are discarded + "eject" -> lose flag status + "keep" -> keep flag status and all bits [default for IntFlag] + """ + STRICT = auto() + CONFORM = auto() + EJECT = auto() + KEEP = auto() +STRICT, CONFORM, EJECT, KEEP = FlagBoundary - def __gt__(self, other): - if isinstance(other, IntEnum): - return self._value_ > other._value_ - if isinstance(other, int): - return self._value_ > other - return NotImplemented - def __ge__(self, other): - if isinstance(other, IntEnum): - return self._value_ >= other._value_ - if isinstance(other, int): - return self._value_ >= other - return NotImplemented +class Flag(Enum, boundary=STRICT): + """ + Support for flags + """ + + _numeric_repr_ = repr + + @staticmethod + def _generate_next_value_(name, start, count, last_values): + """ + Generate the next value when not given. + + name: the name of the member + start: the initial start value or None + count: the number of existing members + last_values: the last value assigned or None + """ + if not count: + return start if start is not None else 1 + last_value = max(last_values) + try: + high_bit = _high_bit(last_value) + except Exception: + raise TypeError('invalid flag value %r' % last_value) from None + return 2 ** (high_bit+1) + + @classmethod + def _iter_member_by_value_(cls, value): + """ + Extract all members from the value in definition (i.e. increasing value) order. + """ + for val in _iter_bits_lsb(value & cls._flag_mask_): + yield cls._value2member_map_.get(val) + + _iter_member_ = _iter_member_by_value_ + + @classmethod + def _iter_member_by_def_(cls, value): + """ + Extract all members from the value in definition order. + """ + yield from sorted( + cls._iter_member_by_value_(value), + key=lambda m: m._sort_order_, + ) + + @classmethod + def _missing_(cls, value): + """ + Create a composite member containing all canonical members present in `value`. + + If non-member values are present, result depends on `_boundary_` setting. + """ + if not isinstance(value, int): + raise ValueError( + "%r is not a valid %s" % (value, cls.__qualname__) + ) + # check boundaries + # - value must be in range (e.g. -16 <-> +15, i.e. ~15 <-> 15) + # - value must not include any skipped flags (e.g. if bit 2 is not + # defined, then 0d10 is invalid) + flag_mask = cls._flag_mask_ + singles_mask = cls._singles_mask_ + all_bits = cls._all_bits_ + neg_value = None + if ( + not ~all_bits <= value <= all_bits + or value & (all_bits ^ flag_mask) + ): + if cls._boundary_ is STRICT: + max_bits = max(value.bit_length(), flag_mask.bit_length()) + raise ValueError( + "%r invalid value %r\n given %s\n allowed %s" % ( + cls, value, bin(value, max_bits), bin(flag_mask, max_bits), + )) + elif cls._boundary_ is CONFORM: + value = value & flag_mask + elif cls._boundary_ is EJECT: + return value + elif cls._boundary_ is KEEP: + if value < 0: + value = ( + max(all_bits+1, 2**(value.bit_length())) + + value + ) + else: + raise ValueError( + '%r unknown flag boundary %r' % (cls, cls._boundary_, ) + ) + if value < 0: + neg_value = value + value = all_bits + 1 + value + # get members and unknown + unknown = value & ~flag_mask + aliases = value & ~singles_mask + member_value = value & singles_mask + if unknown and cls._boundary_ is not KEEP: + raise ValueError( + '%s(%r) --> unknown values %r [%s]' + % (cls.__name__, value, unknown, bin(unknown)) + ) + # normal Flag? + if cls._member_type_ is object: + # construct a singleton enum pseudo-member + pseudo_member = object.__new__(cls) + else: + pseudo_member = cls._member_type_.__new__(cls, value) + if not hasattr(pseudo_member, '_value_'): + pseudo_member._value_ = value + if member_value or aliases: + members = [] + combined_value = 0 + for m in cls._iter_member_(member_value): + members.append(m) + combined_value |= m._value_ + if aliases: + value = member_value | aliases + for n, pm in cls._member_map_.items(): + if pm not in members and pm._value_ and pm._value_ & value == pm._value_: + members.append(pm) + combined_value |= pm._value_ + unknown = value ^ combined_value + pseudo_member._name_ = '|'.join([m._name_ for m in members]) + if not combined_value: + pseudo_member._name_ = None + elif unknown and cls._boundary_ is STRICT: + raise ValueError('%r: no members with value %r' % (cls, unknown)) + elif unknown: + pseudo_member._name_ += '|%s' % cls._numeric_repr_(unknown) + else: + pseudo_member._name_ = None + # use setdefault in case another thread already created a composite + # with this value + # note: zero is a special case -- always add it + pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) + if neg_value is not None: + cls._value2member_map_[neg_value] = pseudo_member + return pseudo_member + def __contains__(self, other): + """ + Returns True if self has at least the same flags set as other. + """ + if not isinstance(other, self.__class__): + raise TypeError( + "unsupported operand type(s) for 'in': %r and %r" % ( + type(other).__qualname__, self.__class__.__qualname__)) + return other._value_ & self._value_ == other._value_ + + def __iter__(self): + """ + Returns flags in definition order. + """ + yield from self._iter_member_(self._value_) + + def __len__(self): + return self._value_.bit_count() -class FlagMeta(EnumMeta): - """Metaclass for :class:`Flag` — adds bitwise-decomposed lookups.""" - - def _decompose_flag(cls, value): - # Combine individual single-bit members covered by `value`. - if cls._member_map_ is None: - raise ValueError(f"{value!r} is not a valid {cls.__name__}") - combined_name = [] - combined_value = 0 - for name, member in cls._member_map_.items(): - if member._value_ & value == member._value_ and member._value_: - combined_name.append(name) - combined_value |= member._value_ - if combined_value != value: - raise ValueError(f"{value!r} is not a valid {cls.__name__}") - if issubclass(cls, int): - new_member = int.__new__(cls, value) + def __repr__(self): + cls_name = self.__class__.__name__ + v_repr = self.__class__._value_repr_ or repr + if self._name_ is None: + return "<%s: %s>" % (cls_name, v_repr(self._value_)) else: - new_member = object.__new__(cls) - new_member._name_ = "|".join(combined_name) - new_member._value_ = value - return new_member + return "<%s.%s: %s>" % (cls_name, self._name_, v_repr(self._value_)) + def __str__(self): + cls_name = self.__class__.__name__ + if self._name_ is None: + return '%s(%r)' % (cls_name, self._value_) + else: + return "%s.%s" % (cls_name, self._name_) -class Flag(Enum, metaclass=FlagMeta): - def __or__(self, other): - if isinstance(other, type(self)): - return type(self)._decompose_flag(self._value_ | other._value_) + def __bool__(self): + return bool(self._value_) + + def _get_value(self, flag): + if isinstance(flag, self.__class__): + return flag._value_ + elif self._member_type_ is not object and isinstance(flag, self._member_type_): + return flag return NotImplemented + def __or__(self, other): + other_value = self._get_value(other) + if other_value is NotImplemented: + return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with |") + value = self._value_ + return self.__class__(value | other_value) + def __and__(self, other): - if isinstance(other, type(self)): - return type(self)._decompose_flag(self._value_ & other._value_) - return NotImplemented + other_value = self._get_value(other) + if other_value is NotImplemented: + return NotImplemented - def __xor__(self, other): - if isinstance(other, type(self)): - return type(self)._decompose_flag(self._value_ ^ other._value_) - return NotImplemented + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with &") + value = self._value_ + return self.__class__(value & other_value) - def __invert__(self): - all_bits = 0 - for member in type(self)._member_map_.values(): - all_bits |= member._value_ - return type(self)._decompose_flag(all_bits & ~self._value_) + def __xor__(self, other): + other_value = self._get_value(other) + if other_value is NotImplemented: + return NotImplemented - def __contains__(self, other): - if isinstance(other, type(self)): - return (self._value_ & other._value_) == other._value_ - return False + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with ^") + value = self._value_ + return self.__class__(value ^ other_value) - def __bool__(self): - return bool(self._value_) + def __invert__(self): + if self._get_value(self) is None: + raise TypeError(f"'{self}' cannot be inverted") + if self._inverted_ is None: + if self._boundary_ in (EJECT, KEEP): + self._inverted_ = self.__class__(~self._value_) + else: + self._inverted_ = self.__class__(self._singles_mask_ & ~self._value_) + return self._inverted_ -class IntFlag(int, Flag): - """Like :class:`IntEnum` but for bitfield-style values; members are - genuine ints (CPython's ``class IntFlag(int, Flag)``).""" + __rand__ = __and__ + __ror__ = __or__ + __rxor__ = __xor__ - def __int__(self): - return self._value_ - def __eq__(self, other): - if isinstance(other, IntFlag): - return self._value_ == other._value_ - if isinstance(other, int): - return self._value_ == other - return NotImplemented +class IntFlag(int, ReprEnum, Flag, boundary=KEEP): + """ + Support for integer-based Flags + """ - def __hash__(self): - return hash(self._value_) +def _high_bit(value): + """ + returns index of highest bit, or -1 if value is zero or negative + """ + return value.bit_length() - 1 def unique(enumeration): - """Class decorator that ensures only one name maps to any value.""" - seen = {} + """ + Class decorator for enumerations ensuring unique member values. + """ duplicates = [] - for name, member in (enumeration._member_map_ or {}).items(): - if member._value_ in seen: - duplicates.append((name, seen[member._value_])) - else: - seen[member._value_] = name + for name, member in enumeration.__members__.items(): + if name != member.name: + duplicates.append((name, member.name)) if duplicates: - joined = ", ".join(f"{n} -> {a}" for n, a in duplicates) - raise ValueError(f"duplicate values found in {enumeration!r}: {joined}") + alias_details = ', '.join( + ["%s -> %s" % (alias, name) for (alias, name) in duplicates]) + raise ValueError('duplicate values found in %r: %s' % + (enumeration, alias_details)) return enumeration +def _dataclass_repr(self): + dcf = self.__dataclass_fields__ + return ', '.join( + '%s=%r' % (k, getattr(self, k)) + for k in dcf.keys() + if dcf[k].repr + ) def global_enum_repr(self): - """`repr` that references the member's *module* rather than its class — - used for enums hoisted into a module namespace via :func:`global_enum` - (e.g. ``calendar.JANUARY``).""" - return f"{self.__class__.__module__}.{self._name_}" + """ + use module.enum_name instead of class.enum_name + the module is the last module in case of a multi-module name + """ + module = self.__class__.__module__.split('.')[-1] + return '%s.%s' % (module, self._name_) -def global_str(self): +def global_flag_repr(self): + """ + use module.flag_name instead of class.flag_name + + the module is the last module in case of a multi-module name + """ + module = self.__class__.__module__.split('.')[-1] cls_name = self.__class__.__name__ - return f"{cls_name}.{self._name_}" + if self._name_ is None: + return "%s.%s(%r)" % (module, cls_name, self._value_) + if _is_single_bit(self._value_): + return '%s.%s' % (module, self._name_) + if self._boundary_ is not FlagBoundary.KEEP: + return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')]) + else: + name = [] + for n in self._name_.split('|'): + if n[0].isdigit(): + name.append(n) + else: + name.append('%s.%s' % (module, n)) + return '|'.join(name) +def global_str(self): + """ + use enum_name instead of class.enum_name + """ + if self._name_ is None: + cls_name = self.__class__.__name__ + return "%s(%r)" % (cls_name, self._value_) + else: + return self._name_ def global_enum(cls, update_str=False): - """Class decorator that exports an enum's members into its defining - module's global namespace and switches member ``repr`` to the - module-qualified form (CPython's ``enum.global_enum``). ``IntEnum`` - keeps ``int``'s ``__str__`` unless ``update_str`` is set.""" - import sys - cls.__repr__ = global_enum_repr - if update_str: + """ + decorator that makes the repr() of an enum member reference its module + instead of its class; also exports all members to the enum's module's + global namespace + """ + if issubclass(cls, Flag): + cls.__repr__ = global_flag_repr + else: + cls.__repr__ = global_enum_repr + if not issubclass(cls, ReprEnum) or update_str: cls.__str__ = global_str - module = sys.modules.get(cls.__module__) - if module is not None: - module.__dict__.update(cls.__members__) + sys.modules[cls.__module__].__dict__.update(cls.__members__) return cls +def _simple_enum(etype=Enum, *, boundary=None, use_args=None): + """ + Class decorator that converts a normal class into an :class:`Enum`. No + safety checks are done, and some advanced behavior (such as + :func:`__init_subclass__`) is not available. Enum creation can be faster + using :func:`_simple_enum`. + + >>> from enum import Enum, _simple_enum + >>> @_simple_enum(Enum) + ... class Color: + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> Color + + """ + def convert_class(cls): + nonlocal use_args + cls_name = cls.__name__ + if use_args is None: + use_args = etype._use_args_ + __new__ = cls.__dict__.get('__new__') + if __new__ is not None: + new_member = __new__.__func__ + else: + new_member = etype._member_type_.__new__ + attrs = {} + body = {} + if __new__ is not None: + body['__new_member__'] = new_member + body['_new_member_'] = new_member + body['_use_args_'] = use_args + body['_generate_next_value_'] = gnv = etype._generate_next_value_ + body['_member_names_'] = member_names = [] + body['_member_map_'] = member_map = {} + body['_value2member_map_'] = value2member_map = {} + body['_hashable_values_'] = hashable_values = [] + body['_unhashable_values_'] = unhashable_values = [] + body['_unhashable_values_map_'] = {} + body['_member_type_'] = member_type = etype._member_type_ + body['_value_repr_'] = etype._value_repr_ + if issubclass(etype, Flag): + body['_boundary_'] = boundary or etype._boundary_ + body['_flag_mask_'] = None + body['_all_bits_'] = None + body['_singles_mask_'] = None + body['_inverted_'] = None + body['__or__'] = Flag.__or__ + body['__xor__'] = Flag.__xor__ + body['__and__'] = Flag.__and__ + body['__ror__'] = Flag.__ror__ + body['__rxor__'] = Flag.__rxor__ + body['__rand__'] = Flag.__rand__ + body['__invert__'] = Flag.__invert__ + for name, obj in cls.__dict__.items(): + if name in ('__dict__', '__weakref__'): + continue + if _is_dunder(name) or _is_private(cls_name, name) or _is_sunder(name) or _is_descriptor(obj): + body[name] = obj + else: + attrs[name] = obj + if cls.__dict__.get('__doc__') is None: + body['__doc__'] = 'An enumeration.' + # + # double check that repr and friends are not the mixin's or various + # things break (such as pickle) + # however, if the method is defined in the Enum itself, don't replace + # it + enum_class = type(cls_name, (etype, ), body, boundary=boundary, _simple=True) + for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'): + if name not in body: + # check for mixin overrides before replacing + enum_method = getattr(etype, name) + found_method = getattr(enum_class, name) + object_method = getattr(object, name) + data_type_method = getattr(member_type, name) + if found_method in (data_type_method, object_method): + setattr(enum_class, name, enum_method) + gnv_last_values = [] + if issubclass(enum_class, Flag): + # Flag / IntFlag + single_bits = multi_bits = 0 + for name, value in attrs.items(): + if isinstance(value, auto) and auto.value is _auto_null: + value = gnv(name, 1, len(member_names), gnv_last_values) + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values or member.value in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: + # an alias to an existing member + contained._add_alias_(name) + else: + # finish creating member + member._name_ = name + member.__objclass__ = enum_class + member.__init__(value) + member._sort_order_ = len(member_names) + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) + value2member_map[value] = member + hashable_values.append(value) + if _is_single_bit(value): + # not a multi-bit alias, record in _member_names_ and _flag_mask_ + member_names.append(name) + single_bits |= value + else: + multi_bits |= value + gnv_last_values.append(value) + enum_class._flag_mask_ = single_bits | multi_bits + enum_class._singles_mask_ = single_bits + enum_class._all_bits_ = 2 ** ((single_bits|multi_bits).bit_length()) - 1 + # set correct __iter__ + member_list = [m._value_ for m in enum_class] + if member_list != sorted(member_list): + enum_class._iter_member_ = enum_class._iter_member_by_def_ + else: + # Enum / IntEnum / StrEnum + for name, value in attrs.items(): + if isinstance(value, auto): + if value.value is _auto_null: + value.value = gnv(name, 1, len(member_names), gnv_last_values) + value = value.value + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values or member._value_ in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: + # an alias to an existing member + contained._add_alias_(name) + else: + # finish creating member + member._name_ = name + member.__objclass__ = enum_class + member.__init__(value) + member._sort_order_ = len(member_names) + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) + member_names.append(name) + gnv_last_values.append(value) + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_.setdefault(value, member) + if value not in hashable_values: + hashable_values.append(value) + except TypeError: + # keep track of the value in a list so containment checks are quick + enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(name, []).append(value) + if '__new__' in body: + enum_class.__new_member__ = enum_class.__new__ + enum_class.__new__ = Enum.__new__ + return enum_class + return convert_class + +@_simple_enum(StrEnum) +class EnumCheck: + """ + various conditions to check an enumeration for + """ + CONTINUOUS = "no skipped integer values" + NAMED_FLAGS = "multi-flag aliases may not contain unnamed flags" + UNIQUE = "one name per value" +CONTINUOUS, NAMED_FLAGS, UNIQUE = EnumCheck + -__all__ = [ - "auto", - "EnumMeta", - "Enum", - "IntEnum", - "FlagMeta", - "Flag", - "IntFlag", - "unique", - "global_enum", -] +class verify: + """ + Check an enumeration for various constraints. (see EnumCheck) + """ + def __init__(self, *checks): + self.checks = checks + def __call__(self, enumeration): + checks = self.checks + cls_name = enumeration.__name__ + if Flag is not None and issubclass(enumeration, Flag): + enum_type = 'flag' + elif issubclass(enumeration, Enum): + enum_type = 'enum' + else: + raise TypeError("the 'verify' decorator only works with Enum and Flag") + for check in checks: + if check is UNIQUE: + # check for duplicate names + duplicates = [] + for name, member in enumeration.__members__.items(): + if name != member.name: + duplicates.append((name, member.name)) + if duplicates: + alias_details = ', '.join( + ["%s -> %s" % (alias, name) for (alias, name) in duplicates]) + raise ValueError('aliases found in %r: %s' % + (enumeration, alias_details)) + elif check is CONTINUOUS: + values = set(e.value for e in enumeration) + if len(values) < 2: + continue + low, high = min(values), max(values) + missing = [] + if enum_type == 'flag': + # check for powers of two + for i in range(_high_bit(low)+1, _high_bit(high)): + if 2**i not in values: + missing.append(2**i) + elif enum_type == 'enum': + # check for missing consecutive integers + for i in range(low+1, high): + if i not in values: + missing.append(i) + else: + raise Exception('verify: unknown type %r' % enum_type) + if missing: + raise ValueError(('invalid %s %r: missing values %s' % ( + enum_type, cls_name, ', '.join((str(m) for m in missing))) + )[:256]) + # limit max length to protect against DOS attacks + elif check is NAMED_FLAGS: + # examine each alias and check for unnamed flags + member_names = enumeration._member_names_ + member_values = [m.value for m in enumeration] + missing_names = [] + missing_value = 0 + for name, alias in enumeration._member_map_.items(): + if name in member_names: + # not an alias + continue + if alias.value < 0: + # negative numbers are not checked + continue + values = list(_iter_bits_lsb(alias.value)) + missed = [v for v in values if v not in member_values] + if missed: + missing_names.append(name) + for val in missed: + missing_value |= val + if missing_names: + if len(missing_names) == 1: + alias = 'alias %s is missing' % missing_names[0] + else: + alias = 'aliases %s and %s are missing' % ( + ', '.join(missing_names[:-1]), missing_names[-1] + ) + if _is_single_bit(missing_value): + value = 'value 0x%x' % missing_value + else: + value = 'combined values of 0x%x' % missing_value + raise ValueError( + 'invalid Flag %r: %s %s [use enum.show_flag_values(value) for details]' + % (cls_name, alias, value) + ) + return enumeration + +def _test_simple_enum(checked_enum, simple_enum): + """ + A function that can be used to test an enum created with :func:`_simple_enum` + against the version created by subclassing :class:`Enum`:: + + >>> from enum import Enum, _simple_enum, _test_simple_enum + >>> @_simple_enum(Enum) + ... class Color: + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> class CheckedColor(Enum): + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> _test_simple_enum(CheckedColor, Color) + + If differences are found, a :exc:`TypeError` is raised. + """ + failed = [] + if checked_enum.__dict__ != simple_enum.__dict__: + checked_dict = checked_enum.__dict__ + checked_keys = list(checked_dict.keys()) + simple_dict = simple_enum.__dict__ + simple_keys = list(simple_dict.keys()) + member_names = set( + list(checked_enum._member_map_.keys()) + + list(simple_enum._member_map_.keys()) + ) + for key in set(checked_keys + simple_keys): + if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', + '__static_attributes__', '__firstlineno__'): + # keys known to be different, or very long + continue + elif key in member_names: + # members are checked below + continue + elif key not in simple_keys: + failed.append("missing key: %r" % (key, )) + elif key not in checked_keys: + failed.append("extra key: %r" % (key, )) + else: + checked_value = checked_dict[key] + simple_value = simple_dict[key] + if callable(checked_value) or isinstance(checked_value, bltns.property): + continue + if key == '__doc__': + # remove all spaces/tabs + compressed_checked_value = checked_value.replace(' ','').replace('\t','') + compressed_simple_value = simple_value.replace(' ','').replace('\t','') + if compressed_checked_value != compressed_simple_value: + failed.append("%r:\n %s\n %s" % ( + key, + "checked -> %r" % (checked_value, ), + "simple -> %r" % (simple_value, ), + )) + elif checked_value != simple_value: + failed.append("%r:\n %s\n %s" % ( + key, + "checked -> %r" % (checked_value, ), + "simple -> %r" % (simple_value, ), + )) + failed.sort() + for name in member_names: + failed_member = [] + if name not in simple_keys: + failed.append('missing member from simple enum: %r' % name) + elif name not in checked_keys: + failed.append('extra member in simple enum: %r' % name) + else: + checked_member_dict = checked_enum[name].__dict__ + checked_member_keys = list(checked_member_dict.keys()) + simple_member_dict = simple_enum[name].__dict__ + simple_member_keys = list(simple_member_dict.keys()) + for key in set(checked_member_keys + simple_member_keys): + if key in ('__module__', '__objclass__', '_inverted_'): + # keys known to be different or absent + continue + elif key not in simple_member_keys: + failed_member.append("missing key %r not in the simple enum member %r" % (key, name)) + elif key not in checked_member_keys: + failed_member.append("extra key %r in simple enum member %r" % (key, name)) + else: + checked_value = checked_member_dict[key] + simple_value = simple_member_dict[key] + if checked_value != simple_value: + failed_member.append("%r:\n %s\n %s" % ( + key, + "checked member -> %r" % (checked_value, ), + "simple member -> %r" % (simple_value, ), + )) + if failed_member: + failed.append('%r member mismatch:\n %s' % ( + name, '\n '.join(failed_member), + )) + for method in ( + '__str__', '__repr__', '__reduce_ex__', '__format__', + '__getnewargs_ex__', '__getnewargs__', '__reduce_ex__', '__reduce__' + ): + if method in simple_keys and method in checked_keys: + # cannot compare functions, and it exists in both, so we're good + continue + elif method not in simple_keys and method not in checked_keys: + # method is inherited -- check it out + checked_method = getattr(checked_enum, method, None) + simple_method = getattr(simple_enum, method, None) + if hasattr(checked_method, '__func__'): + checked_method = checked_method.__func__ + simple_method = simple_method.__func__ + if checked_method != simple_method: + failed.append("%r: %-30s %s" % ( + method, + "checked -> %r" % (checked_method, ), + "simple -> %r" % (simple_method, ), + )) + else: + # if the method existed in only one of the enums, it will have been caught + # in the first checks above + pass + if failed: + raise TypeError('enum mismatch:\n %s' % '\n '.join(failed)) + +def _old_convert_(etype, name, module, filter, source=None, *, boundary=None): + """ + Create a new Enum subclass that replaces a collection of global constants + """ + # convert all constants from source (or module) that pass filter() to + # a new Enum called name, and export the enum and its members back to + # module; + # also, replace the __reduce_ex__ method so unpickling works in + # previous Python versions + module_globals = sys.modules[module].__dict__ + if source: + source = source.__dict__ + else: + source = module_globals + # _value2member_map_ is populated in the same order every time + # for a consistent reverse mapping of number to name when there + # are multiple names for the same number. + members = [ + (name, value) + for name, value in source.items() + if filter(name)] + try: + # sort by value + members.sort(key=lambda t: (t[1], t[0])) + except TypeError: + # unless some values aren't comparable, in which case sort by name + members.sort(key=lambda t: t[0]) + cls = etype(name, members, module=module, boundary=boundary or KEEP) + return cls + +_stdlib_enums = IntEnum, StrEnum, IntFlag diff --git a/crates/weavepy-vm/src/stdlib/python/importlib_bootstrap.py b/crates/weavepy-vm/src/stdlib/python/importlib_bootstrap.py new file mode 100644 index 0000000..75301ba --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/importlib_bootstrap.py @@ -0,0 +1,41 @@ +"""``importlib._bootstrap`` — WeavePy façade. + +In CPython this is the frozen core of the import system; ``importlib`` +itself aliases it (``importlib._bootstrap = _bootstrap``). WeavePy's +import core lives in Rust, so this module exposes the handful of +bootstrap entry points stdlib code reaches for directly (notably +``pydoc.importfile`` calling ``_bootstrap._load(spec)``), implemented +over the same spec/loader machinery as ``importlib.util``. +""" + +import sys + +from importlib.util import module_from_spec as _module_from_spec + +__all__ = ['_load', 'spec_from_loader', 'ModuleSpec'] + +from importlib.util import spec_from_loader +from importlib.machinery import ModuleSpec + + +def _load(spec): + """Create, register, and execute the module described by *spec*. + + Mirrors CPython's `_bootstrap._load`: the module is inserted into + ``sys.modules`` *before* execution (so circular imports during exec + see the partial module) and removed again if execution fails. + """ + module = _module_from_spec(spec) + sys.modules[spec.name] = module + try: + if spec.loader is not None: + spec.loader.exec_module(module) + except BaseException: + try: + del sys.modules[spec.name] + except KeyError: + pass + raise + # An import hook may have replaced the entry; honour what's there, + # like CPython does. + return sys.modules.get(spec.name, module) diff --git a/crates/weavepy-vm/src/stdlib/python/importlib_bootstrap_external.py b/crates/weavepy-vm/src/stdlib/python/importlib_bootstrap_external.py new file mode 100644 index 0000000..c987a57 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/importlib_bootstrap_external.py @@ -0,0 +1,26 @@ +"""``importlib._bootstrap_external`` — WeavePy façade. + +In CPython this frozen module defines the filesystem loaders, which +``importlib.machinery`` then re-exports. WeavePy defines the loaders in +``importlib.machinery`` directly, so this module is the alias in the +other direction — stdlib code (e.g. ``pydoc.locate``-adjacent paths) +imports the names from here. +""" + +from importlib.machinery import ( + SOURCE_SUFFIXES, + BYTECODE_SUFFIXES, + EXTENSION_SUFFIXES, + SourceFileLoader, + SourcelessFileLoader, + ExtensionFileLoader, +) + +__all__ = [ + 'SOURCE_SUFFIXES', + 'BYTECODE_SUFFIXES', + 'EXTENSION_SUFFIXES', + 'SourceFileLoader', + 'SourcelessFileLoader', + 'ExtensionFileLoader', +] diff --git a/crates/weavepy-vm/src/stdlib/python/inspect.py b/crates/weavepy-vm/src/stdlib/python/inspect.py index b3c2c6e..cb431cb 100644 --- a/crates/weavepy-vm/src/stdlib/python/inspect.py +++ b/crates/weavepy-vm/src/stdlib/python/inspect.py @@ -35,6 +35,16 @@ "isgetsetdescriptor", "isdatadescriptor", "ismethoddescriptor", + "ismethodwrapper", + "classify_class_attrs", + "Attribute", + "getclasstree", + "walktree", + "getcomments", + "getabsfile", + "getattr_static", + "indentsize", + "findsource", "currentframe", "stack", "trace", @@ -317,11 +327,30 @@ def isgetsetdescriptor(obj): def isdatadescriptor(obj): - return hasattr(obj, "__set__") and hasattr(obj, "__get__") + """CPython semantics: data descriptors define `__set__` or + `__delete__` *on their type* (properties, slots, C getsets).""" + if isclass(obj) or ismethod(obj) or isfunction(obj): + # mutual exclusion, as in CPython + return False + tp = type(obj) + return hasattr(tp, "__set__") or hasattr(tp, "__delete__") def ismethoddescriptor(obj): - return hasattr(obj, "__get__") and not hasattr(obj, "__set__") and not hasattr(obj, "__delete__") + """CPython semantics: non-data descriptors with a `__get__` whose + type carries neither `__set__` nor `__delete__`.""" + if isclass(obj) or ismethod(obj) or isfunction(obj): + # mutual exclusion, as in CPython + return False + tp = type(obj) + return (hasattr(tp, "__get__") + and not hasattr(tp, "__set__") + and not hasattr(tp, "__delete__")) + + +def ismethodwrapper(obj): + """Return true if the object is a method wrapper (bound slot wrapper).""" + return isinstance(obj, types.MethodWrapperType) # ---------------- frames / stack ---------------- # @@ -465,6 +494,110 @@ def getsource(obj): return "".join(lines) +def getabsfile(obj, _filename=None): + """Return an absolute path to the source or compiled file for an object. + + The idea is for each object to have a unique origin, so this routine + normalizes the result as much as possible. (CPython `inspect.getabsfile`.) + """ + import os + if _filename is None: + _filename = getsourcefile(obj) or getfile(obj) + return os.path.normcase(os.path.abspath(_filename)) + + +def indentsize(line): + """Return the indent size, in spaces, at the start of a line of text.""" + expline = line.expandtabs() + return len(expline) - len(expline.lstrip()) + + +def findsource(obj): + """Return the entire source file and starting line number for an object. + + The argument may be a module, class, method, function, traceback, frame, + or code object. The source code is returned as a list of all the lines + in the file and the line number indexes a line in that list. An OSError + is raised if the source code cannot be retrieved. + """ + filename = getsourcefile(obj) + if filename is None: + raise OSError("source code not available") + lines = linecache.getlines(filename) + if not lines: + raise OSError("could not get source code") + if ismodule(obj): + return lines, 0 + if isclass(obj): + block, lnum = _class_block(lines, obj.__name__) + if not block: + raise OSError("could not find class definition") + return lines, lnum - 1 + if ismethod(obj): + obj = obj.__func__ + if isfunction(obj): + obj = getattr(obj, "__code__", None) + if istraceback(obj): + obj = obj.tb_frame + if isframe(obj): + obj = obj.f_code + if iscode(obj): + lnum = obj.co_firstlineno - 1 + if lnum < 0 or lnum >= len(lines): + raise OSError("lineno is out of bounds") + return lines, lnum + raise OSError("could not find code object") + + +def getcomments(obj): + """Get lines of comments immediately preceding an object's source code. + + Returns None when source can't be found. (CPython `inspect.getcomments`.) + """ + try: + lines, lnum = findsource(obj) + except (OSError, TypeError): + return None + + if ismodule(obj): + # Look for a comment block at the top of the file. + start = 0 + if lines and lines[0][:2] == '#!': + start = 1 + while start < len(lines) and lines[start].strip() in ('', '#'): + start = start + 1 + if start < len(lines) and lines[start][:1] == '#': + comments = [] + end = start + while end < len(lines) and lines[end][:1] == '#': + comments.append(lines[end].expandtabs()) + end = end + 1 + return ''.join(comments) + + # Look for a comment block preceding the object. + elif lnum > 0: + indent = indentsize(lines[lnum]) + end = lnum - 1 + if end >= 0 and lines[end].lstrip()[:1] == '#' and \ + indentsize(lines[end]) == indent: + comments = [lines[end].expandtabs().lstrip()] + if end > 0: + end = end - 1 + comment = lines[end].expandtabs().lstrip() + while comment[:1] == '#' and indentsize(lines[end]) == indent: + comments[:0] = [comment] + end = end - 1 + if end < 0: + break + comment = lines[end].expandtabs().lstrip() + while comments and comments[0].strip() == '#': + comments[:1] = [] + while comments and comments[-1].strip() == '#': + comments[-1:] = [] + return ''.join(comments) + return None + + def _block_around(lines, start): if start < 0 or start >= len(lines): return [], 1 @@ -546,6 +679,197 @@ def getmembers(obj, predicate=None): return out +try: + from collections import namedtuple as _namedtuple + + Attribute = _namedtuple('Attribute', 'name kind defining_class object') +except Exception: # pragma: no cover - collections is always frozen + Attribute = None + + +def classify_class_attrs(cls): + """Return list of attribute-descriptor tuples. + + CPython `inspect.classify_class_attrs`: for each name in `dir(cls)` + (plus DynamicClassAttributes found on the MRO), a 4-tuple of + (name, kind, defining class, object). Kind is one of 'class method', + 'static method', 'property', 'method', 'data'. + """ + mro = getmro(cls) + metamro = getmro(type(cls)) # for attributes stored in the metaclass + metamro = tuple(c for c in metamro if c not in (type, object)) + class_bases = (cls,) + tuple(mro) + all_bases = class_bases + metamro + names = dir(cls) + # Add any DynamicClassAttributes to the list of names; + # this may result in duplicate entries if, for example, a virtual + # attribute with the same name as a DynamicClassAttribute exists. + for base in mro: + for k, v in base.__dict__.items(): + if isinstance(v, types.DynamicClassAttribute) and v.fget is not None: + names.append(k) + result = [] + processed = set() + + for name in names: + # Get the object associated with the name, and where it was defined. + homecls = None + get_obj = None + dict_obj = None + if name not in processed: + try: + if name == '__dict__': + raise Exception("__dict__ is special, don't want the proxy") + get_obj = getattr(cls, name) + except Exception: + pass + else: + homecls = getattr(get_obj, "__objclass__", homecls) + if homecls not in class_bases: + # if the resulting object does not live somewhere in the + # mro, drop it and search the mro manually + homecls = None + last_cls = None + # first look in the classes + for srch_cls in class_bases: + srch_obj = getattr(srch_cls, name, None) + if srch_obj is get_obj: + last_cls = srch_cls + # then check the metaclasses + for srch_cls in metamro: + try: + srch_obj = srch_cls.__getattr__(cls, name) + except AttributeError: + continue + if srch_obj is get_obj: + last_cls = srch_cls + if last_cls is not None: + homecls = last_cls + for base in all_bases: + if name in base.__dict__: + dict_obj = base.__dict__[name] + if homecls not in metamro: + homecls = base + break + if homecls is None: + # unable to locate the attribute anywhere, most likely due to + # buggy custom __dir__; discard and move on + continue + obj = get_obj if get_obj is not None else dict_obj + # Classify the object or its descriptor. + if isinstance(dict_obj, (staticmethod, types.BuiltinMethodType)): + kind = "static method" + obj = dict_obj + elif isinstance(dict_obj, (classmethod, types.ClassMethodDescriptorType)): + kind = "class method" + obj = dict_obj + elif isinstance(dict_obj, property): + kind = "property" + obj = dict_obj + elif isroutine(obj): + kind = "method" + else: + kind = "data" + result.append(Attribute(name, kind, homecls, obj)) + processed.add(name) + return result + + +def walktree(classes, children, parent): + """Recursive helper function for getclasstree().""" + results = [] + classes.sort(key=lambda c: (c.__module__, c.__name__)) + for c in classes: + results.append((c, c.__bases__)) + if c in children: + results.append(walktree(children[c], children, c)) + return results + + +def getclasstree(classes, unique=False): + """Arrange the given list of classes into a hierarchy of nested lists. + + Where a nested list appears, it contains classes derived from the class + whose entry immediately precedes the list. (CPython `inspect.getclasstree`.) + """ + children = {} + roots = [] + for c in classes: + if c.__bases__: + for parent in c.__bases__: + if parent not in children: + children[parent] = [] + if c not in children[parent]: + children[parent].append(c) + if unique and parent in classes: + break + elif c not in roots: + roots.append(c) + for parent in children: + if parent not in classes: + roots.append(parent) + return walktree(roots, children, None) + + +_static_sentinel = object() + + +def _static_lookup_in_dict(obj_dict, attr): + try: + return obj_dict[attr], True + except (KeyError, TypeError): + return None, False + + +def getattr_static(obj, attr, default=_static_sentinel): + """Retrieve attributes without triggering dynamic lookup via the + descriptor protocol, __getattr__ or __getattribute__. + + Behavioural port of CPython `inspect.getattr_static`: walk the + instance `__dict__` and the type's MRO dictionaries directly. Data + descriptors found on the type take precedence over instance + attributes, mirroring `object.__getattribute__`'s static order. + """ + instance_result = _static_sentinel + klass = type(obj) + if not isclass(obj): + dict_attr, found = _static_lookup_in_dict( + getattr(obj, "__dict__", {}) or {}, attr) + if found: + instance_result = dict_attr + else: + klass = obj + + klass_result = _static_sentinel + for entry in getmro(klass): + d = entry.__dict__ + if attr in d: + klass_result = d[attr] + break + + if instance_result is not _static_sentinel and \ + klass_result is not _static_sentinel: + # A data descriptor on the class shadows the instance dict. + if hasattr(type(klass_result), "__set__") or \ + hasattr(type(klass_result), "__delete__"): + return klass_result + return instance_result + if instance_result is not _static_sentinel: + return instance_result + if klass_result is not _static_sentinel: + return klass_result + + if isclass(obj): + # Search the metaclass MRO as well. + for entry in getmro(type(obj)): + d = entry.__dict__ + if attr in d: + return d[attr] + if default is not _static_sentinel: + return default + raise AttributeError(attr) + + # ---------------- argspec / signature ---------------- # class FullArgSpec: @@ -818,7 +1142,13 @@ def _bind(self, args, kwargs, partial): raise TypeError(f"missing a required argument: {p.name!r}") return BoundArguments(self, arguments) - def __str__(self): + def format(self, *, max_width=None): + """Create a string representation of the Signature object. + + If *max_width* is passed and the one-line rendering is longer, + every parameter goes on its own line (CPython 3.13 + `Signature.format`). + """ result = [] render_pos_only_separator = False render_kw_only_separator = True @@ -842,10 +1172,15 @@ def __str__(self): # There were only positional-only parameters, hence the flag was # not reset to 'False'. result.append("/") - ret = "" + rendered = "(" + ", ".join(result) + ")" + if max_width is not None and len(rendered) > max_width: + rendered = "(\n " + ",\n ".join(result) + "\n)" if self._return_annotation is not _empty: - ret = f" -> {self._return_annotation!r}" - return "(" + ", ".join(result) + ")" + ret + rendered += f" -> {self._return_annotation!r}" + return rendered + + def __str__(self): + return self.format() @classmethod def from_callable(cls, func): diff --git a/crates/weavepy-vm/src/stdlib/python/itertools.py b/crates/weavepy-vm/src/stdlib/python/itertools.py index b01058c..d43e956 100644 --- a/crates/weavepy-vm/src/stdlib/python/itertools.py +++ b/crates/weavepy-vm/src/stdlib/python/itertools.py @@ -177,9 +177,68 @@ def takewhile(predicate, iterable): yield item +class _TeeState: + """Source iterator shared by the branches of one ``tee()`` call. + + ``busy`` guards the source pull: CPython's C ``tee`` raises + ``RuntimeError`` when one branch tries to advance the shared source + while another is already blocked inside it (test_tee_concurrent). + """ + + __slots__ = ("it", "busy") + + def __init__(self, it): + self.it = it + self.busy = False + + +class _TeeIter: + """One branch of a lazy ``tee()``. + + Branches share a singly-linked buffer of ``[value, next_link]`` + cells; ``next_link is None`` marks the frontier where the source + iterator must be advanced. The source is consumed on demand, so + ``tee`` works on infinite and partially-consumed iterators. + """ + + __slots__ = ("_state", "_link") + + def __init__(self, state, link): + self._state = state + self._link = link + + def __iter__(self): + return self + + def __next__(self): + link = self._link + if link is None: + raise StopIteration + if link[1] is None: + state = self._state + if state.busy: + raise RuntimeError("cannot re-enter the tee iterator") + state.busy = True + try: + value = next(state.it) + except StopIteration: + self._link = None + raise + finally: + state.busy = False + link[0] = value + link[1] = [None, None] + value, self._link = link + return value + + def tee(iterable, n=2): - items = list(iterable) - return tuple(iter(items) for _ in range(n)) + if n < 0: + raise ValueError("n must be >= 0") + it = iter(iterable) + state = _TeeState(it) + link = [None, None] + return tuple(_TeeIter(state, link) for _ in range(n)) def zip_longest(*iterables, fillvalue=None): diff --git a/crates/weavepy-vm/src/stdlib/python/pydoc.py b/crates/weavepy-vm/src/stdlib/python/pydoc.py new file mode 100755 index 0000000..d5b56f7 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/pydoc.py @@ -0,0 +1,2859 @@ +#!/usr/bin/env python3 +"""Generate Python documentation in HTML or text for interactive use. + +At the Python interactive prompt, calling help(thing) on a Python object +documents the object, and calling help() starts up an interactive +help session. + +Or, at the shell command line outside of Python: + +Run "pydoc " to show documentation on something. may be +the name of a function, module, package, or a dotted reference to a +class or function within a module or module in a package. If the +argument contains a path segment delimiter (e.g. slash on Unix, +backslash on Windows) it is treated as the path to a Python source file. + +Run "pydoc -k " to search for a keyword in the synopsis lines +of all available modules. + +Run "pydoc -n " to start an HTTP server with the given +hostname (default: localhost) on the local machine. + +Run "pydoc -p " to start an HTTP server on the given port on the +local machine. Port number 0 can be used to get an arbitrary unused port. + +Run "pydoc -b" to start an HTTP server on an arbitrary unused port and +open a web browser to interactively browse documentation. Combine with +the -n and -p options to control the hostname and port used. + +Run "pydoc -w " to write out the HTML documentation for a module +to a file named ".html". + +Module docs for core modules are assumed to be in + + https://docs.python.org/X.Y/library/ + +This can be overridden by setting the PYTHONDOCS environment variable +to a different URL or to a local directory containing the Library +Reference Manual pages. +""" +__all__ = ['help'] +__author__ = "Ka-Ping Yee " +__date__ = "26 February 2001" + +__credits__ = """Guido van Rossum, for an excellent programming language. +Tommy Burnette, the original creator of manpy. +Paul Prescod, for all his work on onlinehelp. +Richard Chamberlain, for the first implementation of textdoc. +""" + +# Known bugs that can't be fixed here: +# - synopsis() cannot be prevented from clobbering existing +# loaded modules. +# - If the __file__ attribute on a module is a relative path and +# the current directory is changed with os.chdir(), an incorrect +# path will be displayed. + +import ast +import __future__ +import builtins +import importlib._bootstrap +import importlib._bootstrap_external +import importlib.machinery +import importlib.util +import inspect +import io +import os +import pkgutil +import platform +import re +import sys +import sysconfig +import time +import tokenize +import urllib.parse +import warnings +from collections import deque +from reprlib import Repr +from traceback import format_exception_only + +from _pyrepl.pager import (get_pager, plain, pipe_pager, + plain_pager, tempfile_pager, tty_pager) + + +# --------------------------------------------------------- old names + +getpager = get_pager +pipepager = pipe_pager +plainpager = plain_pager +tempfilepager = tempfile_pager +ttypager = tty_pager + + +# --------------------------------------------------------- common routines + +def pathdirs(): + """Convert sys.path into a list of absolute, existing, unique paths.""" + dirs = [] + normdirs = [] + for dir in sys.path: + dir = os.path.abspath(dir or '.') + normdir = os.path.normcase(dir) + if normdir not in normdirs and os.path.isdir(dir): + dirs.append(dir) + normdirs.append(normdir) + return dirs + +def _findclass(func): + cls = sys.modules.get(func.__module__) + if cls is None: + return None + for name in func.__qualname__.split('.')[:-1]: + cls = getattr(cls, name) + if not inspect.isclass(cls): + return None + return cls + +def _finddoc(obj): + if inspect.ismethod(obj): + name = obj.__func__.__name__ + self = obj.__self__ + if (inspect.isclass(self) and + getattr(getattr(self, name, None), '__func__') is obj.__func__): + # classmethod + cls = self + else: + cls = self.__class__ + elif inspect.isfunction(obj): + name = obj.__name__ + cls = _findclass(obj) + if cls is None or getattr(cls, name) is not obj: + return None + elif inspect.isbuiltin(obj): + name = obj.__name__ + self = obj.__self__ + if (inspect.isclass(self) and + self.__qualname__ + '.' + name == obj.__qualname__): + # classmethod + cls = self + else: + cls = self.__class__ + # Should be tested before isdatadescriptor(). + elif isinstance(obj, property): + name = obj.__name__ + cls = _findclass(obj.fget) + if cls is None or getattr(cls, name) is not obj: + return None + elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): + name = obj.__name__ + cls = obj.__objclass__ + if getattr(cls, name) is not obj: + return None + if inspect.ismemberdescriptor(obj): + slots = getattr(cls, '__slots__', None) + if isinstance(slots, dict) and name in slots: + return slots[name] + else: + return None + for base in cls.__mro__: + try: + doc = _getowndoc(getattr(base, name)) + except AttributeError: + continue + if doc is not None: + return doc + return None + +def _getowndoc(obj): + """Get the documentation string for an object if it is not + inherited from its class.""" + try: + doc = object.__getattribute__(obj, '__doc__') + if doc is None: + return None + if obj is not type: + typedoc = type(obj).__doc__ + if isinstance(typedoc, str) and typedoc == doc: + return None + return doc + except AttributeError: + return None + +def _getdoc(object): + """Get the documentation string for an object. + + All tabs are expanded to spaces. To clean up docstrings that are + indented to line up with blocks of code, any whitespace than can be + uniformly removed from the second line onwards is removed.""" + doc = _getowndoc(object) + if doc is None: + try: + doc = _finddoc(object) + except (AttributeError, TypeError): + return None + if not isinstance(doc, str): + return None + return inspect.cleandoc(doc) + +def getdoc(object): + """Get the doc string or comments for an object.""" + result = _getdoc(object) or inspect.getcomments(object) + return result and re.sub('^ *\n', '', result.rstrip()) or '' + +def splitdoc(doc): + """Split a doc string into a synopsis line (if any) and the rest.""" + lines = doc.strip().split('\n') + if len(lines) == 1: + return lines[0], '' + elif len(lines) >= 2 and not lines[1].rstrip(): + return lines[0], '\n'.join(lines[2:]) + return '', '\n'.join(lines) + +def _getargspec(object): + try: + signature = inspect.signature(object) + if signature: + name = getattr(object, '__name__', '') + # function are always single-line and should not be formatted + max_width = (80 - len(name)) if name != '' else None + return signature.format(max_width=max_width) + except (ValueError, TypeError): + argspec = getattr(object, '__text_signature__', None) + if argspec: + if argspec[:2] == '($': + argspec = '(' + argspec[2:] + if getattr(object, '__self__', None) is not None: + # Strip the bound argument. + m = re.match(r'\(\w+(?:(?=\))|,\s*(?:/(?:(?=\))|,\s*))?)', argspec) + if m: + argspec = '(' + argspec[m.end():] + return argspec + return None + +def classname(object, modname): + """Get a class name and qualify it with a module name if necessary.""" + name = object.__name__ + if object.__module__ != modname: + name = object.__module__ + '.' + name + return name + +def parentname(object, modname): + """Get a name of the enclosing class (qualified it with a module name + if necessary) or module.""" + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname and object.__module__ is not None: + return object.__module__ + '.' + name + else: + return name + else: + if object.__module__ != modname: + return object.__module__ + +def isdata(object): + """Check if an object is of a type that probably means it's data.""" + return not (inspect.ismodule(object) or inspect.isclass(object) or + inspect.isroutine(object) or inspect.isframe(object) or + inspect.istraceback(object) or inspect.iscode(object)) + +def replace(text, *pairs): + """Do a series of global replacements on a string.""" + while pairs: + text = pairs[1].join(text.split(pairs[0])) + pairs = pairs[2:] + return text + +def cram(text, maxlen): + """Omit part of a string if needed to make it fit in a maximum length.""" + if len(text) > maxlen: + pre = max(0, (maxlen-3)//2) + post = max(0, maxlen-3-pre) + return text[:pre] + '...' + text[len(text)-post:] + return text + +_re_stripid = re.compile(r' at 0x[0-9a-f]{6,16}(>+)$', re.IGNORECASE) +def stripid(text): + """Remove the hexadecimal id from a Python object representation.""" + # The behaviour of %p is implementation-dependent in terms of case. + return _re_stripid.sub(r'\1', text) + +def _is_bound_method(fn): + """ + Returns True if fn is a bound method, regardless of whether + fn was implemented in Python or in C. + """ + if inspect.ismethod(fn): + return True + if inspect.isbuiltin(fn): + self = getattr(fn, '__self__', None) + return not (inspect.ismodule(self) or (self is None)) + return False + + +def allmethods(cl): + methods = {} + for key, value in inspect.getmembers(cl, inspect.isroutine): + methods[key] = 1 + for base in cl.__bases__: + methods.update(allmethods(base)) # all your base are belong to us + for key in methods.keys(): + methods[key] = getattr(cl, key) + return methods + +def _split_list(s, predicate): + """Split sequence s via predicate, and return pair ([true], [false]). + + The return value is a 2-tuple of lists, + ([x for x in s if predicate(x)], + [x for x in s if not predicate(x)]) + """ + + yes = [] + no = [] + for x in s: + if predicate(x): + yes.append(x) + else: + no.append(x) + return yes, no + +_future_feature_names = set(__future__.all_feature_names) + +def visiblename(name, all=None, obj=None): + """Decide whether to show documentation on a variable.""" + # Certain special names are redundant or internal. + # XXX Remove __initializing__? + if name in {'__author__', '__builtins__', '__cached__', '__credits__', + '__date__', '__doc__', '__file__', '__spec__', + '__loader__', '__module__', '__name__', '__package__', + '__path__', '__qualname__', '__slots__', '__version__', + '__static_attributes__', '__firstlineno__'}: + return 0 + # Private names are hidden, but special names are displayed. + if name.startswith('__') and name.endswith('__'): return 1 + # Namedtuples have public fields and methods with a single leading underscore + if name.startswith('_') and hasattr(obj, '_fields'): + return True + # Ignore __future__ imports. + if obj is not __future__ and name in _future_feature_names: + if isinstance(getattr(obj, name, None), __future__._Feature): + return False + if all is not None: + # only document that which the programmer exported in __all__ + return name in all + else: + return not name.startswith('_') + +def classify_class_attrs(object): + """Wrap inspect.classify_class_attrs, with fixup for data descriptors and bound methods.""" + results = [] + for (name, kind, cls, value) in inspect.classify_class_attrs(object): + if inspect.isdatadescriptor(value): + kind = 'data descriptor' + if isinstance(value, property) and value.fset is None: + kind = 'readonly property' + elif kind == 'method' and _is_bound_method(value): + kind = 'static method' + results.append((name, kind, cls, value)) + return results + +def sort_attributes(attrs, object): + 'Sort the attrs list in-place by _fields and then alphabetically by name' + # This allows data descriptors to be ordered according + # to a _fields attribute if present. + fields = getattr(object, '_fields', []) + try: + field_order = {name : i-len(fields) for (i, name) in enumerate(fields)} + except TypeError: + field_order = {} + keyfunc = lambda attr: (field_order.get(attr[0], 0), attr[0]) + attrs.sort(key=keyfunc) + +# ----------------------------------------------------- module manipulation + +def ispackage(path): + """Guess whether a path refers to a package directory.""" + warnings.warn('The pydoc.ispackage() function is deprecated', + DeprecationWarning, stacklevel=2) + if os.path.isdir(path): + for ext in ('.py', '.pyc'): + if os.path.isfile(os.path.join(path, '__init__' + ext)): + return True + return False + +def source_synopsis(file): + """Return the one-line summary of a file object, if present""" + + string = '' + try: + tokens = tokenize.generate_tokens(file.readline) + for tok_type, tok_string, _, _, _ in tokens: + if tok_type == tokenize.STRING: + string += tok_string + elif tok_type == tokenize.NEWLINE: + with warnings.catch_warnings(): + # Ignore the "invalid escape sequence" warning. + warnings.simplefilter("ignore", SyntaxWarning) + docstring = ast.literal_eval(string) + if not isinstance(docstring, str): + return None + return docstring.strip().split('\n')[0].strip() + elif tok_type == tokenize.OP and tok_string in ('(', ')'): + string += tok_string + elif tok_type not in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING): + return None + except (tokenize.TokenError, UnicodeDecodeError, SyntaxError): + return None + return None + +def synopsis(filename, cache={}): + """Get the one-line summary out of a module file.""" + mtime = os.stat(filename).st_mtime + lastupdate, result = cache.get(filename, (None, None)) + if lastupdate is None or lastupdate < mtime: + # Look for binary suffixes first, falling back to source. + if filename.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): + loader_cls = importlib.machinery.SourcelessFileLoader + elif filename.endswith(tuple(importlib.machinery.EXTENSION_SUFFIXES)): + loader_cls = importlib.machinery.ExtensionFileLoader + else: + loader_cls = None + # Now handle the choice. + if loader_cls is None: + # Must be a source file. + try: + file = tokenize.open(filename) + except OSError: + # module can't be opened, so skip it + return None + # text modules can be directly examined + with file: + result = source_synopsis(file) + else: + # Must be a binary module, which has to be imported. + loader = loader_cls('__temp__', filename) + # XXX We probably don't need to pass in the loader here. + spec = importlib.util.spec_from_file_location('__temp__', filename, + loader=loader) + try: + module = importlib._bootstrap._load(spec) + except: + return None + del sys.modules['__temp__'] + result = module.__doc__.splitlines()[0] if module.__doc__ else None + # Cache the result. + cache[filename] = (mtime, result) + return result + +class ErrorDuringImport(Exception): + """Errors that occurred while trying to import something to document it.""" + def __init__(self, filename, exc_info): + if not isinstance(exc_info, tuple): + assert isinstance(exc_info, BaseException) + self.exc = type(exc_info) + self.value = exc_info + self.tb = exc_info.__traceback__ + else: + warnings.warn("A tuple value for exc_info is deprecated, use an exception instance", + DeprecationWarning) + + self.exc, self.value, self.tb = exc_info + self.filename = filename + + def __str__(self): + exc = self.exc.__name__ + return 'problem in %s - %s: %s' % (self.filename, exc, self.value) + +def importfile(path): + """Import a Python source file or compiled file given its path.""" + magic = importlib.util.MAGIC_NUMBER + with open(path, 'rb') as file: + is_bytecode = magic == file.read(len(magic)) + filename = os.path.basename(path) + name, ext = os.path.splitext(filename) + if is_bytecode: + loader = importlib._bootstrap_external.SourcelessFileLoader(name, path) + else: + loader = importlib._bootstrap_external.SourceFileLoader(name, path) + # XXX We probably don't need to pass in the loader here. + spec = importlib.util.spec_from_file_location(name, path, loader=loader) + try: + return importlib._bootstrap._load(spec) + except BaseException as err: + raise ErrorDuringImport(path, err) + +def safeimport(path, forceload=0, cache={}): + """Import a module; handle errors; return None if the module isn't found. + + If the module *is* found but an exception occurs, it's wrapped in an + ErrorDuringImport exception and reraised. Unlike __import__, if a + package path is specified, the module at the end of the path is returned, + not the package at the beginning. If the optional 'forceload' argument + is 1, we reload the module from disk (unless it's a dynamic extension).""" + try: + # If forceload is 1 and the module has been previously loaded from + # disk, we always have to reload the module. Checking the file's + # mtime isn't good enough (e.g. the module could contain a class + # that inherits from another module that has changed). + if forceload and path in sys.modules: + if path not in sys.builtin_module_names: + # Remove the module from sys.modules and re-import to try + # and avoid problems with partially loaded modules. + # Also remove any submodules because they won't appear + # in the newly loaded module's namespace if they're already + # in sys.modules. + subs = [m for m in sys.modules if m.startswith(path + '.')] + for key in [path] + subs: + # Prevent garbage collection. + cache[key] = sys.modules[key] + del sys.modules[key] + module = importlib.import_module(path) + except BaseException as err: + # Did the error occur before or after the module was found? + if path in sys.modules: + # An error occurred while executing the imported module. + raise ErrorDuringImport(sys.modules[path].__file__, err) + elif type(err) is SyntaxError: + # A SyntaxError occurred before we could execute the module. + raise ErrorDuringImport(err.filename, err) + elif isinstance(err, ImportError) and err.name == path: + # No such module in the path. + return None + else: + # Some other error occurred during the importing process. + raise ErrorDuringImport(path, err) + return module + +# ---------------------------------------------------- formatter base class + +class Doc: + + PYTHONDOCS = os.environ.get("PYTHONDOCS", + "https://docs.python.org/%d.%d/library" + % sys.version_info[:2]) + + def document(self, object, name=None, *args): + """Generate documentation for an object.""" + args = (object, name) + args + # 'try' clause is to attempt to handle the possibility that inspect + # identifies something in a way that pydoc itself has issues handling; + # think 'super' and how it is a descriptor (which raises the exception + # by lacking a __name__ attribute) and an instance. + try: + if inspect.ismodule(object): return self.docmodule(*args) + if inspect.isclass(object): return self.docclass(*args) + if inspect.isroutine(object): return self.docroutine(*args) + except AttributeError: + pass + if inspect.isdatadescriptor(object): return self.docdata(*args) + return self.docother(*args) + + def fail(self, object, name=None, *args): + """Raise an exception for unimplemented types.""" + message = "don't know how to document object%s of type %s" % ( + name and ' ' + repr(name), type(object).__name__) + raise TypeError(message) + + docmodule = docclass = docroutine = docother = docproperty = docdata = fail + + def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): + """Return the location of module docs or None""" + + try: + file = inspect.getabsfile(object) + except TypeError: + file = '(built-in)' + + docloc = os.environ.get("PYTHONDOCS", self.PYTHONDOCS) + + basedir = os.path.normcase(basedir) + if (isinstance(object, type(os)) and + (object.__name__ in ('errno', 'exceptions', 'gc', + 'marshal', 'posix', 'signal', 'sys', + '_thread', 'zipimport') or + (file.startswith(basedir) and + not file.startswith(os.path.join(basedir, 'site-packages')))) and + object.__name__ not in ('xml.etree', 'test.test_pydoc.pydoc_mod')): + + try: + from pydoc_data import module_docs + except ImportError: + module_docs = None + + if module_docs and object.__name__ in module_docs.module_docs: + doc_name = module_docs.module_docs[object.__name__] + if docloc.startswith(("http://", "https://")): + docloc = "{}/{}".format(docloc.rstrip("/"), doc_name) + else: + docloc = os.path.join(docloc, doc_name) + else: + docloc = None + else: + docloc = None + return docloc + +# -------------------------------------------- HTML documentation generator + +class HTMLRepr(Repr): + """Class for safely making an HTML representation of a Python object.""" + def __init__(self): + Repr.__init__(self) + self.maxlist = self.maxtuple = 20 + self.maxdict = 10 + self.maxstring = self.maxother = 100 + + def escape(self, text): + return replace(text, '&', '&', '<', '<', '>', '>') + + def repr(self, object): + return Repr.repr(self, object) + + def repr1(self, x, level): + if hasattr(type(x), '__name__'): + methodname = 'repr_' + '_'.join(type(x).__name__.split()) + if hasattr(self, methodname): + return getattr(self, methodname)(x, level) + return self.escape(cram(stripid(repr(x)), self.maxother)) + + def repr_string(self, x, level): + test = cram(x, self.maxstring) + testrepr = repr(test) + if '\\' in test and '\\' not in replace(testrepr, r'\\', ''): + # Backslashes are only literal in the string and are never + # needed to make any special characters, so show a raw string. + return 'r' + testrepr[0] + self.escape(test) + testrepr[0] + return re.sub(r'((\\[\\abfnrtv\'"]|\\[0-9]..|\\x..|\\u....)+)', + r'\1', + self.escape(testrepr)) + + repr_str = repr_string + + def repr_instance(self, x, level): + try: + return self.escape(cram(stripid(repr(x)), self.maxstring)) + except: + return self.escape('<%s instance>' % x.__class__.__name__) + + repr_unicode = repr_string + +class HTMLDoc(Doc): + """Formatter class for HTML documentation.""" + + # ------------------------------------------- HTML formatting utilities + + _repr_instance = HTMLRepr() + repr = _repr_instance.repr + escape = _repr_instance.escape + + def page(self, title, contents): + """Format an HTML page.""" + return '''\ + + + + +Python: %s + +%s +''' % (title, contents) + + def heading(self, title, extras=''): + """Format a page heading.""" + return ''' + + + +
 
%s		%s
+ ''' % (title, extras or ' ') + + def section(self, title, cls, contents, width=6, + prelude='', marginalia=None, gap=' '): + """Format a section with a heading.""" + if marginalia is None: + marginalia = '' + ' ' * width + '' + result = '''

+ + + + ''' % (cls, title) + if prelude: + result = result + ''' + + +''' % (cls, marginalia, cls, prelude, gap) + else: + result = result + ''' +''' % (cls, marginalia, gap) + + return result + '\n
 
%s
%s%s
%s
%s%s%s
' % contents + + def bigsection(self, title, *args): + """Format a section with a big heading.""" + title = '%s' % title + return self.section(title, *args) + + def preformat(self, text): + """Format literal preformatted text.""" + text = self.escape(text.expandtabs()) + return replace(text, '\n\n', '\n \n', '\n\n', '\n \n', + ' ', ' ', '\n', '
\n') + + def multicolumn(self, list, format): + """Format a list of items into a multi-column list.""" + result = '' + rows = (len(list) + 3) // 4 + for col in range(4): + result = result + '' + for i in range(rows*col, rows*col+rows): + if i < len(list): + result = result + format(list[i]) + '
\n' + result = result + '' + return '%s
' % result + + def grey(self, text): return '%s' % text + + def namelink(self, name, *dicts): + """Make a link for an identifier, given name-to-URL mappings.""" + for dict in dicts: + if name in dict: + return '%s' % (dict[name], name) + return name + + def classlink(self, object, modname): + """Make a link for a class.""" + name, module = object.__name__, sys.modules.get(object.__module__) + if hasattr(module, name) and getattr(module, name) is object: + return '%s' % ( + module.__name__, name, classname(object, modname)) + return classname(object, modname) + + def parentlink(self, object, modname): + """Make a link for the enclosing class or module.""" + link = None + name, module = object.__name__, sys.modules.get(object.__module__) + if hasattr(module, name) and getattr(module, name) is object: + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname: + link = '%s.html#%s' % (module.__name__, name) + else: + link = '#%s' % name + else: + if object.__module__ != modname: + link = '%s.html' % module.__name__ + if link: + return '%s' % (link, parentname(object, modname)) + else: + return parentname(object, modname) + + def modulelink(self, object): + """Make a link for a module.""" + return '%s' % (object.__name__, object.__name__) + + def modpkglink(self, modpkginfo): + """Make a link for a module or package to display in an index.""" + name, path, ispackage, shadowed = modpkginfo + if shadowed: + return self.grey(name) + if path: + url = '%s.%s.html' % (path, name) + else: + url = '%s.html' % name + if ispackage: + text = '%s (package)' % name + else: + text = name + return '%s' % (url, text) + + def filelink(self, url, path): + """Make a link to source file.""" + return '%s' % (url, path) + + def markup(self, text, escape=None, funcs={}, classes={}, methods={}): + """Mark up some plain text, given a context of symbols to look for. + Each context dictionary maps object names to anchor names.""" + escape = escape or self.escape + results = [] + here = 0 + pattern = re.compile(r'\b((http|https|ftp)://\S+[\w/]|' + r'RFC[- ]?(\d+)|' + r'PEP[- ]?(\d+)|' + r'(self\.)?(\w+))') + while match := pattern.search(text, here): + start, end = match.span() + results.append(escape(text[here:start])) + + all, scheme, rfc, pep, selfdot, name = match.groups() + if scheme: + url = escape(all).replace('"', '"') + results.append('%s' % (url, url)) + elif rfc: + url = 'https://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) + results.append('%s' % (url, escape(all))) + elif pep: + url = 'https://peps.python.org/pep-%04d/' % int(pep) + results.append('%s' % (url, escape(all))) + elif selfdot: + # Create a link for methods like 'self.method(...)' + # and use for attributes like 'self.attr' + if text[end:end+1] == '(': + results.append('self.' + self.namelink(name, methods)) + else: + results.append('self.%s' % name) + elif text[end:end+1] == '(': + results.append(self.namelink(name, methods, funcs, classes)) + else: + results.append(self.namelink(name, classes)) + here = end + results.append(escape(text[here:])) + return ''.join(results) + + # ---------------------------------------------- type-specific routines + + def formattree(self, tree, modname, parent=None): + """Produce HTML for a class tree as given by inspect.getclasstree().""" + result = '' + for entry in tree: + if isinstance(entry, tuple): + c, bases = entry + result = result + '

' + result = result + self.classlink(c, modname) + if bases and bases != (parent,): + parents = [] + for base in bases: + parents.append(self.classlink(base, modname)) + result = result + '(' + ', '.join(parents) + ')' + result = result + '\n
' + elif isinstance(entry, list): + result = result + '
\n%s
\n' % self.formattree( + entry, modname, c) + return '
\n%s
\n' % result + + def docmodule(self, object, name=None, mod=None, *ignored): + """Produce HTML documentation for a module object.""" + name = object.__name__ # ignore the passed-in name + try: + all = object.__all__ + except AttributeError: + all = None + parts = name.split('.') + links = [] + for i in range(len(parts)-1): + links.append( + '%s' % + ('.'.join(parts[:i+1]), parts[i])) + linkedname = '.'.join(links + parts[-1:]) + head = '%s' % linkedname + try: + path = inspect.getabsfile(object) + url = urllib.parse.quote(path) + filelink = self.filelink(url, path) + except TypeError: + filelink = '(built-in)' + info = [] + if hasattr(object, '__version__'): + version = str(object.__version__) + if version[:11] == '$' + 'Revision: ' and version[-1:] == '$': + version = version[11:-1].strip() + info.append('version %s' % self.escape(version)) + if hasattr(object, '__date__'): + info.append(self.escape(str(object.__date__))) + if info: + head = head + ' (%s)' % ', '.join(info) + docloc = self.getdocloc(object) + if docloc is not None: + docloc = '
Module Reference' % locals() + else: + docloc = '' + result = self.heading(head, 'index
' + filelink + docloc) + + modules = inspect.getmembers(object, inspect.ismodule) + + classes, cdict = [], {} + for key, value in inspect.getmembers(object, inspect.isclass): + # if __all__ exists, believe it. Otherwise use old heuristic. + if (all is not None or + (inspect.getmodule(value) or object) is object): + if visiblename(key, all, object): + classes.append((key, value)) + cdict[key] = cdict[value] = '#' + key + for key, value in classes: + for base in value.__bases__: + key, modname = base.__name__, base.__module__ + module = sys.modules.get(modname) + if modname != name and module and hasattr(module, key): + if getattr(module, key) is base: + if not key in cdict: + cdict[key] = cdict[base] = modname + '.html#' + key + funcs, fdict = [], {} + for key, value in inspect.getmembers(object, inspect.isroutine): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or inspect.isbuiltin(value) + or (inspect.getmodule(value) or object) is object): + if visiblename(key, all, object): + funcs.append((key, value)) + fdict[key] = '#-' + key + if inspect.isfunction(value): fdict[value] = fdict[key] + data = [] + for key, value in inspect.getmembers(object, isdata): + if visiblename(key, all, object): + data.append((key, value)) + + doc = self.markup(getdoc(object), self.preformat, fdict, cdict) + doc = doc and '%s' % doc + result = result + '

%s

\n' % doc + + if hasattr(object, '__path__'): + modpkgs = [] + for importer, modname, ispkg in pkgutil.iter_modules(object.__path__): + modpkgs.append((modname, name, ispkg, 0)) + modpkgs.sort() + contents = self.multicolumn(modpkgs, self.modpkglink) + result = result + self.bigsection( + 'Package Contents', 'pkg-content', contents) + elif modules: + contents = self.multicolumn( + modules, lambda t: self.modulelink(t[1])) + result = result + self.bigsection( + 'Modules', 'pkg-content', contents) + + if classes: + classlist = [value for (key, value) in classes] + contents = [ + self.formattree(inspect.getclasstree(classlist, 1), name)] + for key, value in classes: + contents.append(self.document(value, key, name, fdict, cdict)) + result = result + self.bigsection( + 'Classes', 'index', ' '.join(contents)) + if funcs: + contents = [] + for key, value in funcs: + contents.append(self.document(value, key, name, fdict, cdict)) + result = result + self.bigsection( + 'Functions', 'functions', ' '.join(contents)) + if data: + contents = [] + for key, value in data: + contents.append(self.document(value, key)) + result = result + self.bigsection( + 'Data', 'data', '
\n'.join(contents)) + if hasattr(object, '__author__'): + contents = self.markup(str(object.__author__), self.preformat) + result = result + self.bigsection('Author', 'author', contents) + if hasattr(object, '__credits__'): + contents = self.markup(str(object.__credits__), self.preformat) + result = result + self.bigsection('Credits', 'credits', contents) + + return result + + def docclass(self, object, name=None, mod=None, funcs={}, classes={}, + *ignored): + """Produce HTML documentation for a class object.""" + realname = object.__name__ + name = name or realname + bases = object.__bases__ + + contents = [] + push = contents.append + + # Cute little class to pump out a horizontal rule between sections. + class HorizontalRule: + def __init__(self): + self.needone = 0 + def maybe(self): + if self.needone: + push('
\n') + self.needone = 1 + hr = HorizontalRule() + + # List the mro, if non-trivial. + mro = deque(inspect.getmro(object)) + if len(mro) > 2: + hr.maybe() + push('
Method resolution order:
\n') + for base in mro: + push('
%s
\n' % self.classlink(base, + object.__module__)) + push('
\n') + + def spill(msg, attrs, predicate): + ok, attrs = _split_list(attrs, predicate) + if ok: + hr.maybe() + push(msg) + for name, kind, homecls, value in ok: + try: + value = getattr(object, name) + except Exception: + # Some descriptors may meet a failure in their __get__. + # (bug #1785) + push(self.docdata(value, name, mod)) + else: + push(self.document(value, name, mod, + funcs, classes, mdict, object, homecls)) + push('\n') + return attrs + + def spilldescriptors(msg, attrs, predicate): + ok, attrs = _split_list(attrs, predicate) + if ok: + hr.maybe() + push(msg) + for name, kind, homecls, value in ok: + push(self.docdata(value, name, mod)) + return attrs + + def spilldata(msg, attrs, predicate): + ok, attrs = _split_list(attrs, predicate) + if ok: + hr.maybe() + push(msg) + for name, kind, homecls, value in ok: + base = self.docother(getattr(object, name), name, mod) + doc = getdoc(value) + if not doc: + push('
%s
\n' % base) + else: + doc = self.markup(getdoc(value), self.preformat, + funcs, classes, mdict) + doc = '
%s' % doc + push('
%s%s
\n' % (base, doc)) + push('\n') + return attrs + + attrs = [(name, kind, cls, value) + for name, kind, cls, value in classify_class_attrs(object) + if visiblename(name, obj=object)] + + mdict = {} + for key, kind, homecls, value in attrs: + mdict[key] = anchor = '#' + name + '-' + key + try: + value = getattr(object, name) + except Exception: + # Some descriptors may meet a failure in their __get__. + # (bug #1785) + pass + try: + # The value may not be hashable (e.g., a data attr with + # a dict or list value). + mdict[value] = anchor + except TypeError: + pass + + while attrs: + if mro: + thisclass = mro.popleft() + else: + thisclass = attrs[0][2] + attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) + + if object is not builtins.object and thisclass is builtins.object: + attrs = inherited + continue + elif thisclass is object: + tag = 'defined here' + else: + tag = 'inherited from %s' % self.classlink(thisclass, + object.__module__) + tag += ':
\n' + + sort_attributes(attrs, object) + + # Pump out the attrs, segregated by kind. + attrs = spill('Methods %s' % tag, attrs, + lambda t: t[1] == 'method') + attrs = spill('Class methods %s' % tag, attrs, + lambda t: t[1] == 'class method') + attrs = spill('Static methods %s' % tag, attrs, + lambda t: t[1] == 'static method') + attrs = spilldescriptors("Readonly properties %s" % tag, attrs, + lambda t: t[1] == 'readonly property') + attrs = spilldescriptors('Data descriptors %s' % tag, attrs, + lambda t: t[1] == 'data descriptor') + attrs = spilldata('Data and other attributes %s' % tag, attrs, + lambda t: t[1] == 'data') + assert attrs == [] + attrs = inherited + + contents = ''.join(contents) + + if name == realname: + title = 'class %s' % ( + name, realname) + else: + title = '%s = class %s' % ( + name, name, realname) + if bases: + parents = [] + for base in bases: + parents.append(self.classlink(base, object.__module__)) + title = title + '(%s)' % ', '.join(parents) + + decl = '' + argspec = _getargspec(object) + if argspec and argspec != '()': + decl = name + self.escape(argspec) + '\n\n' + + doc = getdoc(object) + if decl: + doc = decl + (doc or '') + doc = self.markup(doc, self.preformat, funcs, classes, mdict) + doc = doc and '%s
 ' % doc + + return self.section(title, 'title', contents, 3, doc) + + def formatvalue(self, object): + """Format an argument default value as text.""" + return self.grey('=' + self.repr(object)) + + def docroutine(self, object, name=None, mod=None, + funcs={}, classes={}, methods={}, cl=None, homecls=None): + """Produce HTML documentation for a function or method object.""" + realname = object.__name__ + name = name or realname + if homecls is None: + homecls = cl + anchor = ('' if cl is None else cl.__name__) + '-' + name + note = '' + skipdocs = False + imfunc = None + if _is_bound_method(object): + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % self.classlink(imself, mod) + else: + note = ' method of %s instance' % self.classlink( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % self.classlink(objclass, mod) + elif objclass is not homecls: + note = ' from ' + self.classlink(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = self.parentlink(imfunc, mod) + if pname: + note = ' from %s' % pname + + if (inspect.iscoroutinefunction(object) or + inspect.isasyncgenfunction(object)): + asyncqualifier = 'async ' + else: + asyncqualifier = '' + + if name == realname: + title = '%s' % (anchor, realname) + else: + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): + reallink = '%s' % ( + cl.__name__ + '-' + realname, realname) + skipdocs = True + if note.startswith(' from '): + note = '' + else: + reallink = realname + title = '%s = %s' % ( + anchor, name, reallink) + argspec = None + if inspect.isroutine(object): + argspec = _getargspec(object) + if argspec and realname == '': + title = '%s lambda ' % name + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + if not object.__annotations__: + argspec = argspec[1:-1] # remove parentheses + if not argspec: + argspec = '(...)' + + decl = asyncqualifier + title + self.escape(argspec) + (note and + self.grey('%s' % note)) + + if skipdocs: + return '
%s
\n' % decl + else: + doc = self.markup( + getdoc(object), self.preformat, funcs, classes, methods) + doc = doc and '
%s
' % doc + return '
%s
%s
\n' % (decl, doc) + + def docdata(self, object, name=None, mod=None, cl=None, *ignored): + """Produce html documentation for a data descriptor.""" + results = [] + push = results.append + + if name: + push('
%s
\n' % name) + doc = self.markup(getdoc(object), self.preformat) + if doc: + push('
%s
\n' % doc) + push('
\n') + + return ''.join(results) + + docproperty = docdata + + def docother(self, object, name=None, mod=None, *ignored): + """Produce HTML documentation for a data object.""" + lhs = name and '%s = ' % name or '' + return lhs + self.repr(object) + + def index(self, dir, shadowed=None): + """Generate an HTML index for a directory of modules.""" + modpkgs = [] + if shadowed is None: shadowed = {} + for importer, name, ispkg in pkgutil.iter_modules([dir]): + if any((0xD800 <= ord(ch) <= 0xDFFF) for ch in name): + # ignore a module if its name contains a surrogate character + continue + modpkgs.append((name, '', ispkg, name in shadowed)) + shadowed[name] = 1 + + modpkgs.sort() + contents = self.multicolumn(modpkgs, self.modpkglink) + return self.bigsection(dir, 'index', contents) + +# -------------------------------------------- text documentation generator + +class TextRepr(Repr): + """Class for safely making a text representation of a Python object.""" + def __init__(self): + Repr.__init__(self) + self.maxlist = self.maxtuple = 20 + self.maxdict = 10 + self.maxstring = self.maxother = 100 + + def repr1(self, x, level): + if hasattr(type(x), '__name__'): + methodname = 'repr_' + '_'.join(type(x).__name__.split()) + if hasattr(self, methodname): + return getattr(self, methodname)(x, level) + return cram(stripid(repr(x)), self.maxother) + + def repr_string(self, x, level): + test = cram(x, self.maxstring) + testrepr = repr(test) + if '\\' in test and '\\' not in replace(testrepr, r'\\', ''): + # Backslashes are only literal in the string and are never + # needed to make any special characters, so show a raw string. + return 'r' + testrepr[0] + test + testrepr[0] + return testrepr + + repr_str = repr_string + + def repr_instance(self, x, level): + try: + return cram(stripid(repr(x)), self.maxstring) + except: + return '<%s instance>' % x.__class__.__name__ + +class TextDoc(Doc): + """Formatter class for text documentation.""" + + # ------------------------------------------- text formatting utilities + + _repr_instance = TextRepr() + repr = _repr_instance.repr + + def bold(self, text): + """Format a string in bold by overstriking.""" + return ''.join(ch + '\b' + ch for ch in text) + + def indent(self, text, prefix=' '): + """Indent text by prepending a given prefix to each line.""" + if not text: return '' + lines = [(prefix + line).rstrip() for line in text.split('\n')] + return '\n'.join(lines) + + def section(self, title, contents): + """Format a section with a given heading.""" + clean_contents = self.indent(contents).rstrip() + return self.bold(title) + '\n' + clean_contents + '\n\n' + + # ---------------------------------------------- type-specific routines + + def formattree(self, tree, modname, parent=None, prefix=''): + """Render in text a class tree as returned by inspect.getclasstree().""" + result = '' + for entry in tree: + if isinstance(entry, tuple): + c, bases = entry + result = result + prefix + classname(c, modname) + if bases and bases != (parent,): + parents = (classname(c, modname) for c in bases) + result = result + '(%s)' % ', '.join(parents) + result = result + '\n' + elif isinstance(entry, list): + result = result + self.formattree( + entry, modname, c, prefix + ' ') + return result + + def docmodule(self, object, name=None, mod=None, *ignored): + """Produce text documentation for a given module object.""" + name = object.__name__ # ignore the passed-in name + synop, desc = splitdoc(getdoc(object)) + result = self.section('NAME', name + (synop and ' - ' + synop)) + all = getattr(object, '__all__', None) + docloc = self.getdocloc(object) + if docloc is not None: + result = result + self.section('MODULE REFERENCE', docloc + """ + +The following documentation is automatically generated from the Python +source files. It may be incomplete, incorrect or include features that +are considered implementation detail and may vary between Python +implementations. When in doubt, consult the module reference at the +location listed above. +""") + + if desc: + result = result + self.section('DESCRIPTION', desc) + + classes = [] + for key, value in inspect.getmembers(object, inspect.isclass): + # if __all__ exists, believe it. Otherwise use old heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): + if visiblename(key, all, object): + classes.append((key, value)) + funcs = [] + for key, value in inspect.getmembers(object, inspect.isroutine): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or inspect.isbuiltin(value) + or (inspect.getmodule(value) or object) is object): + if visiblename(key, all, object): + funcs.append((key, value)) + data = [] + for key, value in inspect.getmembers(object, isdata): + if visiblename(key, all, object): + data.append((key, value)) + + modpkgs = [] + modpkgs_names = set() + if hasattr(object, '__path__'): + for importer, modname, ispkg in pkgutil.iter_modules(object.__path__): + modpkgs_names.add(modname) + if ispkg: + modpkgs.append(modname + ' (package)') + else: + modpkgs.append(modname) + + modpkgs.sort() + result = result + self.section( + 'PACKAGE CONTENTS', '\n'.join(modpkgs)) + + # Detect submodules as sometimes created by C extensions + submodules = [] + for key, value in inspect.getmembers(object, inspect.ismodule): + if value.__name__.startswith(name + '.') and key not in modpkgs_names: + submodules.append(key) + if submodules: + submodules.sort() + result = result + self.section( + 'SUBMODULES', '\n'.join(submodules)) + + if classes: + classlist = [value for key, value in classes] + contents = [self.formattree( + inspect.getclasstree(classlist, 1), name)] + for key, value in classes: + contents.append(self.document(value, key, name)) + result = result + self.section('CLASSES', '\n'.join(contents)) + + if funcs: + contents = [] + for key, value in funcs: + contents.append(self.document(value, key, name)) + result = result + self.section('FUNCTIONS', '\n'.join(contents)) + + if data: + contents = [] + for key, value in data: + contents.append(self.docother(value, key, name, maxlen=70)) + result = result + self.section('DATA', '\n'.join(contents)) + + if hasattr(object, '__version__'): + version = str(object.__version__) + if version[:11] == '$' + 'Revision: ' and version[-1:] == '$': + version = version[11:-1].strip() + result = result + self.section('VERSION', version) + if hasattr(object, '__date__'): + result = result + self.section('DATE', str(object.__date__)) + if hasattr(object, '__author__'): + result = result + self.section('AUTHOR', str(object.__author__)) + if hasattr(object, '__credits__'): + result = result + self.section('CREDITS', str(object.__credits__)) + try: + file = inspect.getabsfile(object) + except TypeError: + file = '(built-in)' + result = result + self.section('FILE', file) + return result + + def docclass(self, object, name=None, mod=None, *ignored): + """Produce text documentation for a given class object.""" + realname = object.__name__ + name = name or realname + bases = object.__bases__ + + def makename(c, m=object.__module__): + return classname(c, m) + + if name == realname: + title = 'class ' + self.bold(realname) + else: + title = self.bold(name) + ' = class ' + realname + if bases: + parents = map(makename, bases) + title = title + '(%s)' % ', '.join(parents) + + contents = [] + push = contents.append + + argspec = _getargspec(object) + if argspec and argspec != '()': + push(name + argspec + '\n') + + doc = getdoc(object) + if doc: + push(doc + '\n') + + # List the mro, if non-trivial. + mro = deque(inspect.getmro(object)) + if len(mro) > 2: + push("Method resolution order:") + for base in mro: + push(' ' + makename(base)) + push('') + + # List the built-in subclasses, if any: + subclasses = sorted( + (str(cls.__name__) for cls in type.__subclasses__(object) + if not cls.__name__.startswith("_") and cls.__module__ == "builtins"), + key=str.lower + ) + no_of_subclasses = len(subclasses) + MAX_SUBCLASSES_TO_DISPLAY = 4 + if subclasses: + push("Built-in subclasses:") + for subclassname in subclasses[:MAX_SUBCLASSES_TO_DISPLAY]: + push(' ' + subclassname) + if no_of_subclasses > MAX_SUBCLASSES_TO_DISPLAY: + push(' ... and ' + + str(no_of_subclasses - MAX_SUBCLASSES_TO_DISPLAY) + + ' other subclasses') + push('') + + # Cute little class to pump out a horizontal rule between sections. + class HorizontalRule: + def __init__(self): + self.needone = 0 + def maybe(self): + if self.needone: + push('-' * 70) + self.needone = 1 + hr = HorizontalRule() + + def spill(msg, attrs, predicate): + ok, attrs = _split_list(attrs, predicate) + if ok: + hr.maybe() + push(msg) + for name, kind, homecls, value in ok: + try: + value = getattr(object, name) + except Exception: + # Some descriptors may meet a failure in their __get__. + # (bug #1785) + push(self.docdata(value, name, mod)) + else: + push(self.document(value, + name, mod, object, homecls)) + return attrs + + def spilldescriptors(msg, attrs, predicate): + ok, attrs = _split_list(attrs, predicate) + if ok: + hr.maybe() + push(msg) + for name, kind, homecls, value in ok: + push(self.docdata(value, name, mod)) + return attrs + + def spilldata(msg, attrs, predicate): + ok, attrs = _split_list(attrs, predicate) + if ok: + hr.maybe() + push(msg) + for name, kind, homecls, value in ok: + doc = getdoc(value) + try: + obj = getattr(object, name) + except AttributeError: + obj = homecls.__dict__[name] + push(self.docother(obj, name, mod, maxlen=70, doc=doc) + + '\n') + return attrs + + attrs = [(name, kind, cls, value) + for name, kind, cls, value in classify_class_attrs(object) + if visiblename(name, obj=object)] + + while attrs: + if mro: + thisclass = mro.popleft() + else: + thisclass = attrs[0][2] + attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) + + if object is not builtins.object and thisclass is builtins.object: + attrs = inherited + continue + elif thisclass is object: + tag = "defined here" + else: + tag = "inherited from %s" % classname(thisclass, + object.__module__) + + sort_attributes(attrs, object) + + # Pump out the attrs, segregated by kind. + attrs = spill("Methods %s:\n" % tag, attrs, + lambda t: t[1] == 'method') + attrs = spill("Class methods %s:\n" % tag, attrs, + lambda t: t[1] == 'class method') + attrs = spill("Static methods %s:\n" % tag, attrs, + lambda t: t[1] == 'static method') + attrs = spilldescriptors("Readonly properties %s:\n" % tag, attrs, + lambda t: t[1] == 'readonly property') + attrs = spilldescriptors("Data descriptors %s:\n" % tag, attrs, + lambda t: t[1] == 'data descriptor') + attrs = spilldata("Data and other attributes %s:\n" % tag, attrs, + lambda t: t[1] == 'data') + + assert attrs == [] + attrs = inherited + + contents = '\n'.join(contents) + if not contents: + return title + '\n' + return title + '\n' + self.indent(contents.rstrip(), ' | ') + '\n' + + def formatvalue(self, object): + """Format an argument default value as text.""" + return '=' + self.repr(object) + + def docroutine(self, object, name=None, mod=None, cl=None, homecls=None): + """Produce text documentation for a function or method object.""" + realname = object.__name__ + name = name or realname + if homecls is None: + homecls = cl + note = '' + skipdocs = False + imfunc = None + if _is_bound_method(object): + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % classname(imself, mod) + else: + note = ' method of %s instance' % classname( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % classname(objclass, mod) + elif objclass is not homecls: + note = ' from ' + classname(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = parentname(imfunc, mod) + if pname: + note = ' from %s' % pname + + if (inspect.iscoroutinefunction(object) or + inspect.isasyncgenfunction(object)): + asyncqualifier = 'async ' + else: + asyncqualifier = '' + + if name == realname: + title = self.bold(realname) + else: + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): + skipdocs = True + if note.startswith(' from '): + note = '' + title = self.bold(name) + ' = ' + realname + argspec = None + + if inspect.isroutine(object): + argspec = _getargspec(object) + if argspec and realname == '': + title = self.bold(name) + ' lambda ' + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + if not object.__annotations__: + argspec = argspec[1:-1] + if not argspec: + argspec = '(...)' + decl = asyncqualifier + title + argspec + note + + if skipdocs: + return decl + '\n' + else: + doc = getdoc(object) or '' + return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n') + + def docdata(self, object, name=None, mod=None, cl=None, *ignored): + """Produce text documentation for a data descriptor.""" + results = [] + push = results.append + + if name: + push(self.bold(name)) + push('\n') + doc = getdoc(object) or '' + if doc: + push(self.indent(doc)) + push('\n') + return ''.join(results) + + docproperty = docdata + + def docother(self, object, name=None, mod=None, parent=None, *ignored, + maxlen=None, doc=None): + """Produce text documentation for a data object.""" + repr = self.repr(object) + if maxlen: + line = (name and name + ' = ' or '') + repr + chop = maxlen - len(line) + if chop < 0: repr = repr[:chop] + '...' + line = (name and self.bold(name) + ' = ' or '') + repr + if not doc: + doc = getdoc(object) + if doc: + line += '\n' + self.indent(str(doc)) + '\n' + return line + +class _PlainTextDoc(TextDoc): + """Subclass of TextDoc which overrides string styling""" + def bold(self, text): + return text + +# --------------------------------------------------------- user interfaces + +def pager(text, title=''): + """The first time this is called, determine what kind of pager to use.""" + global pager + pager = get_pager() + pager(text, title) + +def describe(thing): + """Produce a short description of the given thing.""" + if inspect.ismodule(thing): + if thing.__name__ in sys.builtin_module_names: + return 'built-in module ' + thing.__name__ + if hasattr(thing, '__path__'): + return 'package ' + thing.__name__ + else: + return 'module ' + thing.__name__ + if inspect.isbuiltin(thing): + return 'built-in function ' + thing.__name__ + if inspect.isgetsetdescriptor(thing): + return 'getset descriptor %s.%s.%s' % ( + thing.__objclass__.__module__, thing.__objclass__.__name__, + thing.__name__) + if inspect.ismemberdescriptor(thing): + return 'member descriptor %s.%s.%s' % ( + thing.__objclass__.__module__, thing.__objclass__.__name__, + thing.__name__) + if inspect.isclass(thing): + return 'class ' + thing.__name__ + if inspect.isfunction(thing): + return 'function ' + thing.__name__ + if inspect.ismethod(thing): + return 'method ' + thing.__name__ + return type(thing).__name__ + +def locate(path, forceload=0): + """Locate an object by name or dotted path, importing as necessary.""" + parts = [part for part in path.split('.') if part] + module, n = None, 0 + while n < len(parts): + nextmodule = safeimport('.'.join(parts[:n+1]), forceload) + if nextmodule: module, n = nextmodule, n + 1 + else: break + if module: + object = module + else: + object = builtins + for part in parts[n:]: + try: + object = getattr(object, part) + except AttributeError: + return None + return object + +# --------------------------------------- interactive interpreter interface + +text = TextDoc() +plaintext = _PlainTextDoc() +html = HTMLDoc() + +def resolve(thing, forceload=0): + """Given an object or a path to an object, get the object and its name.""" + if isinstance(thing, str): + object = locate(thing, forceload) + if object is None: + raise ImportError('''\ +No Python documentation found for %r. +Use help() to get the interactive help utility. +Use help(str) for help on the str class.''' % thing) + return object, thing + else: + name = getattr(thing, '__name__', None) + return thing, name if isinstance(name, str) else None + +def render_doc(thing, title='Python Library Documentation: %s', forceload=0, + renderer=None): + """Render text documentation, given an object or a path to an object.""" + if renderer is None: + renderer = text + object, name = resolve(thing, forceload) + desc = describe(object) + module = inspect.getmodule(object) + if name and '.' in name: + desc += ' in ' + name[:name.rfind('.')] + elif module and module is not object: + desc += ' in module ' + module.__name__ + + if not (inspect.ismodule(object) or + inspect.isclass(object) or + inspect.isroutine(object) or + inspect.isdatadescriptor(object) or + _getdoc(object)): + # If the passed object is a piece of data or an instance, + # document its available methods instead of its value. + if hasattr(object, '__origin__'): + object = object.__origin__ + else: + object = type(object) + desc += ' object' + return title % desc + '\n\n' + renderer.document(object, name) + +def doc(thing, title='Python Library Documentation: %s', forceload=0, + output=None, is_cli=False): + """Display text documentation, given an object or a path to an object.""" + if output is None: + try: + if isinstance(thing, str): + what = thing + else: + what = getattr(thing, '__qualname__', None) + if not isinstance(what, str): + what = getattr(thing, '__name__', None) + if not isinstance(what, str): + what = type(thing).__name__ + ' object' + pager(render_doc(thing, title, forceload), f'Help on {what!s}') + except ImportError as exc: + if is_cli: + raise + print(exc) + else: + try: + s = render_doc(thing, title, forceload, plaintext) + except ImportError as exc: + s = str(exc) + output.write(s) + +def writedoc(thing, forceload=0): + """Write HTML documentation to a file in the current directory.""" + object, name = resolve(thing, forceload) + page = html.page(describe(object), html.document(object, name)) + with open(name + '.html', 'w', encoding='utf-8') as file: + file.write(page) + print('wrote', name + '.html') + +def writedocs(dir, pkgpath='', done=None): + """Write out HTML documentation for all modules in a directory tree.""" + if done is None: done = {} + for importer, modname, ispkg in pkgutil.walk_packages([dir], pkgpath): + writedoc(modname) + return + +class Helper: + + # These dictionaries map a topic name to either an alias, or a tuple + # (label, seealso-items). The "label" is the label of the corresponding + # section in the .rst file under Doc/ and an index into the dictionary + # in pydoc_data/topics.py. + # + # CAUTION: if you change one of these dictionaries, be sure to adapt the + # list of needed labels in Doc/tools/extensions/pyspecific.py and + # regenerate the pydoc_data/topics.py file by running + # make pydoc-topics + # in Doc/ and copying the output file into the Lib/ directory. + + keywords = { + 'False': '', + 'None': '', + 'True': '', + 'and': 'BOOLEAN', + 'as': 'with', + 'assert': ('assert', ''), + 'async': ('async', ''), + 'await': ('await', ''), + 'break': ('break', 'while for'), + 'class': ('class', 'CLASSES SPECIALMETHODS'), + 'continue': ('continue', 'while for'), + 'def': ('function', ''), + 'del': ('del', 'BASICMETHODS'), + 'elif': 'if', + 'else': ('else', 'while for'), + 'except': 'try', + 'finally': 'try', + 'for': ('for', 'break continue while'), + 'from': 'import', + 'global': ('global', 'nonlocal NAMESPACES'), + 'if': ('if', 'TRUTHVALUE'), + 'import': ('import', 'MODULES'), + 'in': ('in', 'SEQUENCEMETHODS'), + 'is': 'COMPARISON', + 'lambda': ('lambda', 'FUNCTIONS'), + 'nonlocal': ('nonlocal', 'global NAMESPACES'), + 'not': 'BOOLEAN', + 'or': 'BOOLEAN', + 'pass': ('pass', ''), + 'raise': ('raise', 'EXCEPTIONS'), + 'return': ('return', 'FUNCTIONS'), + 'try': ('try', 'EXCEPTIONS'), + 'while': ('while', 'break continue if TRUTHVALUE'), + 'with': ('with', 'CONTEXTMANAGERS EXCEPTIONS yield'), + 'yield': ('yield', ''), + } + # Either add symbols to this dictionary or to the symbols dictionary + # directly: Whichever is easier. They are merged later. + _strprefixes = [p + q for p in ('b', 'f', 'r', 'u') for q in ("'", '"')] + _symbols_inverse = { + 'STRINGS' : ("'", "'''", '"', '"""', *_strprefixes), + 'OPERATORS' : ('+', '-', '*', '**', '/', '//', '%', '<<', '>>', '&', + '|', '^', '~', '<', '>', '<=', '>=', '==', '!=', '<>'), + 'COMPARISON' : ('<', '>', '<=', '>=', '==', '!=', '<>'), + 'UNARY' : ('-', '~'), + 'AUGMENTEDASSIGNMENT' : ('+=', '-=', '*=', '/=', '%=', '&=', '|=', + '^=', '<<=', '>>=', '**=', '//='), + 'BITWISE' : ('<<', '>>', '&', '|', '^', '~'), + 'COMPLEX' : ('j', 'J') + } + symbols = { + '%': 'OPERATORS FORMATTING', + '**': 'POWER', + ',': 'TUPLES LISTS FUNCTIONS', + '.': 'ATTRIBUTES FLOAT MODULES OBJECTS', + '...': 'ELLIPSIS', + ':': 'SLICINGS DICTIONARYLITERALS', + '@': 'def class', + '\\': 'STRINGS', + ':=': 'ASSIGNMENTEXPRESSIONS', + '_': 'PRIVATENAMES', + '__': 'PRIVATENAMES SPECIALMETHODS', + '`': 'BACKQUOTES', + '(': 'TUPLES FUNCTIONS CALLS', + ')': 'TUPLES FUNCTIONS CALLS', + '[': 'LISTS SUBSCRIPTS SLICINGS', + ']': 'LISTS SUBSCRIPTS SLICINGS' + } + for topic, symbols_ in _symbols_inverse.items(): + for symbol in symbols_: + topics = symbols.get(symbol, topic) + if topic not in topics: + topics = topics + ' ' + topic + symbols[symbol] = topics + del topic, symbols_, symbol, topics + + topics = { + 'TYPES': ('types', 'STRINGS UNICODE NUMBERS SEQUENCES MAPPINGS ' + 'FUNCTIONS CLASSES MODULES FILES inspect'), + 'STRINGS': ('strings', 'str UNICODE SEQUENCES STRINGMETHODS ' + 'FORMATTING TYPES'), + 'STRINGMETHODS': ('string-methods', 'STRINGS FORMATTING'), + 'FORMATTING': ('formatstrings', 'OPERATORS'), + 'UNICODE': ('strings', 'encodings unicode SEQUENCES STRINGMETHODS ' + 'FORMATTING TYPES'), + 'NUMBERS': ('numbers', 'INTEGER FLOAT COMPLEX TYPES'), + 'INTEGER': ('integers', 'int range'), + 'FLOAT': ('floating', 'float math'), + 'COMPLEX': ('imaginary', 'complex cmath'), + 'SEQUENCES': ('typesseq', 'STRINGMETHODS FORMATTING range LISTS'), + 'MAPPINGS': 'DICTIONARIES', + 'FUNCTIONS': ('typesfunctions', 'def TYPES'), + 'METHODS': ('typesmethods', 'class def CLASSES TYPES'), + 'CODEOBJECTS': ('bltin-code-objects', 'compile FUNCTIONS TYPES'), + 'TYPEOBJECTS': ('bltin-type-objects', 'types TYPES'), + 'FRAMEOBJECTS': 'TYPES', + 'TRACEBACKS': 'TYPES', + 'NONE': ('bltin-null-object', ''), + 'ELLIPSIS': ('bltin-ellipsis-object', 'SLICINGS'), + 'SPECIALATTRIBUTES': ('specialattrs', ''), + 'CLASSES': ('types', 'class SPECIALMETHODS PRIVATENAMES'), + 'MODULES': ('typesmodules', 'import'), + 'PACKAGES': 'import', + 'EXPRESSIONS': ('operator-summary', 'lambda or and not in is BOOLEAN ' + 'COMPARISON BITWISE SHIFTING BINARY FORMATTING POWER ' + 'UNARY ATTRIBUTES SUBSCRIPTS SLICINGS CALLS TUPLES ' + 'LISTS DICTIONARIES'), + 'OPERATORS': 'EXPRESSIONS', + 'PRECEDENCE': 'EXPRESSIONS', + 'OBJECTS': ('objects', 'TYPES'), + 'SPECIALMETHODS': ('specialnames', 'BASICMETHODS ATTRIBUTEMETHODS ' + 'CALLABLEMETHODS SEQUENCEMETHODS MAPPINGMETHODS ' + 'NUMBERMETHODS CLASSES'), + 'BASICMETHODS': ('customization', 'hash repr str SPECIALMETHODS'), + 'ATTRIBUTEMETHODS': ('attribute-access', 'ATTRIBUTES SPECIALMETHODS'), + 'CALLABLEMETHODS': ('callable-types', 'CALLS SPECIALMETHODS'), + 'SEQUENCEMETHODS': ('sequence-types', 'SEQUENCES SEQUENCEMETHODS ' + 'SPECIALMETHODS'), + 'MAPPINGMETHODS': ('sequence-types', 'MAPPINGS SPECIALMETHODS'), + 'NUMBERMETHODS': ('numeric-types', 'NUMBERS AUGMENTEDASSIGNMENT ' + 'SPECIALMETHODS'), + 'EXECUTION': ('execmodel', 'NAMESPACES DYNAMICFEATURES EXCEPTIONS'), + 'NAMESPACES': ('naming', 'global nonlocal ASSIGNMENT DELETION DYNAMICFEATURES'), + 'DYNAMICFEATURES': ('dynamic-features', ''), + 'SCOPING': 'NAMESPACES', + 'FRAMES': 'NAMESPACES', + 'EXCEPTIONS': ('exceptions', 'try except finally raise'), + 'CONVERSIONS': ('conversions', ''), + 'IDENTIFIERS': ('identifiers', 'keywords SPECIALIDENTIFIERS'), + 'SPECIALIDENTIFIERS': ('id-classes', ''), + 'PRIVATENAMES': ('atom-identifiers', ''), + 'LITERALS': ('atom-literals', 'STRINGS NUMBERS TUPLELITERALS ' + 'LISTLITERALS DICTIONARYLITERALS'), + 'TUPLES': 'SEQUENCES', + 'TUPLELITERALS': ('exprlists', 'TUPLES LITERALS'), + 'LISTS': ('typesseq-mutable', 'LISTLITERALS'), + 'LISTLITERALS': ('lists', 'LISTS LITERALS'), + 'DICTIONARIES': ('typesmapping', 'DICTIONARYLITERALS'), + 'DICTIONARYLITERALS': ('dict', 'DICTIONARIES LITERALS'), + 'ATTRIBUTES': ('attribute-references', 'getattr hasattr setattr ATTRIBUTEMETHODS'), + 'SUBSCRIPTS': ('subscriptions', 'SEQUENCEMETHODS'), + 'SLICINGS': ('slicings', 'SEQUENCEMETHODS'), + 'CALLS': ('calls', 'EXPRESSIONS'), + 'POWER': ('power', 'EXPRESSIONS'), + 'UNARY': ('unary', 'EXPRESSIONS'), + 'BINARY': ('binary', 'EXPRESSIONS'), + 'SHIFTING': ('shifting', 'EXPRESSIONS'), + 'BITWISE': ('bitwise', 'EXPRESSIONS'), + 'COMPARISON': ('comparisons', 'EXPRESSIONS BASICMETHODS'), + 'BOOLEAN': ('booleans', 'EXPRESSIONS TRUTHVALUE'), + 'ASSERTION': 'assert', + 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), + 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), + 'ASSIGNMENTEXPRESSIONS': ('assignment-expressions', ''), + 'DELETION': 'del', + 'RETURNING': 'return', + 'IMPORTING': 'import', + 'CONDITIONAL': 'if', + 'LOOPING': ('compound', 'for while break continue'), + 'TRUTHVALUE': ('truth', 'if while and or not BASICMETHODS'), + 'DEBUGGING': ('debugger', 'pdb'), + 'CONTEXTMANAGERS': ('context-managers', 'with'), + } + + def __init__(self, input=None, output=None): + self._input = input + self._output = output + + @property + def input(self): + return self._input or sys.stdin + + @property + def output(self): + return self._output or sys.stdout + + def __repr__(self): + if inspect.stack()[1][3] == '?': + self() + return '' + return '<%s.%s instance>' % (self.__class__.__module__, + self.__class__.__qualname__) + + _GoInteractive = object() + def __call__(self, request=_GoInteractive): + if request is not self._GoInteractive: + try: + self.help(request) + except ImportError as err: + self.output.write(f'{err}\n') + else: + self.intro() + self.interact() + self.output.write(''' +You are now leaving help and returning to the Python interpreter. +If you want to ask for help on a particular object directly from the +interpreter, you can type "help(object)". Executing "help('string')" +has the same effect as typing a particular string at the help> prompt. +''') + + def interact(self): + self.output.write('\n') + while True: + try: + request = self.getline('help> ') + except (KeyboardInterrupt, EOFError): + break + request = request.strip() + if not request: + continue # back to the prompt + + # Make sure significant trailing quoting marks of literals don't + # get deleted while cleaning input + if (len(request) > 2 and request[0] == request[-1] in ("'", '"') + and request[0] not in request[1:-1]): + request = request[1:-1] + if request.lower() in ('q', 'quit', 'exit'): break + if request == 'help': + self.intro() + else: + self.help(request) + + def getline(self, prompt): + """Read one line, using input() when appropriate.""" + if self.input is sys.stdin: + return input(prompt) + else: + self.output.write(prompt) + self.output.flush() + return self.input.readline() + + def help(self, request, is_cli=False): + if isinstance(request, str): + request = request.strip() + if request == 'keywords': self.listkeywords() + elif request == 'symbols': self.listsymbols() + elif request == 'topics': self.listtopics() + elif request == 'modules': self.listmodules() + elif request[:8] == 'modules ': + self.listmodules(request.split()[1]) + elif request in self.symbols: self.showsymbol(request) + elif request in ['True', 'False', 'None']: + # special case these keywords since they are objects too + doc(eval(request), 'Help on %s:', output=self._output, is_cli=is_cli) + elif request in self.keywords: self.showtopic(request) + elif request in self.topics: self.showtopic(request) + elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) + else: doc(str, 'Help on %s:', output=self._output, is_cli=is_cli) + elif isinstance(request, Helper): self() + else: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) + self.output.write('\n') + + def intro(self): + self.output.write('''\ +Welcome to Python {0}'s help utility! If this is your first time using +Python, you should definitely check out the tutorial at +https://docs.python.org/{0}/tutorial/. + +Enter the name of any module, keyword, or topic to get help on writing +Python programs and using Python modules. To get a list of available +modules, keywords, symbols, or topics, enter "modules", "keywords", +"symbols", or "topics". + +Each module also comes with a one-line summary of what it does; to list +the modules whose name or summary contain a given string such as "spam", +enter "modules spam". + +To quit this help utility and return to the interpreter, +enter "q", "quit" or "exit". +'''.format('%d.%d' % sys.version_info[:2])) + + def list(self, items, columns=4, width=80): + items = sorted(items) + colw = width // columns + rows = (len(items) + columns - 1) // columns + for row in range(rows): + for col in range(columns): + i = col * rows + row + if i < len(items): + self.output.write(items[i]) + if col < columns - 1: + self.output.write(' ' + ' ' * (colw - 1 - len(items[i]))) + self.output.write('\n') + + def listkeywords(self): + self.output.write(''' +Here is a list of the Python keywords. Enter any keyword to get more help. + +''') + self.list(self.keywords.keys()) + + def listsymbols(self): + self.output.write(''' +Here is a list of the punctuation symbols which Python assigns special meaning +to. Enter any symbol to get more help. + +''') + self.list(self.symbols.keys()) + + def listtopics(self): + self.output.write(''' +Here is a list of available topics. Enter any topic name to get more help. + +''') + self.list(self.topics.keys(), columns=3) + + def showtopic(self, topic, more_xrefs=''): + try: + import pydoc_data.topics + except ImportError: + self.output.write(''' +Sorry, topic and keyword documentation is not available because the +module "pydoc_data.topics" could not be found. +''') + return + target = self.topics.get(topic, self.keywords.get(topic)) + if not target: + self.output.write('no documentation found for %s\n' % repr(topic)) + return + if isinstance(target, str): + return self.showtopic(target, more_xrefs) + + label, xrefs = target + try: + doc = pydoc_data.topics.topics[label] + except KeyError: + self.output.write('no documentation found for %s\n' % repr(topic)) + return + doc = doc.strip() + '\n' + if more_xrefs: + xrefs = (xrefs or '') + ' ' + more_xrefs + if xrefs: + import textwrap + text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' + wrapped_text = textwrap.wrap(text, 72) + doc += '\n%s\n' % '\n'.join(wrapped_text) + + if self._output is None: + pager(doc, f'Help on {topic!s}') + else: + self.output.write(doc) + + def _gettopic(self, topic, more_xrefs=''): + """Return unbuffered tuple of (topic, xrefs). + + If an error occurs here, the exception is caught and displayed by + the url handler. + + This function duplicates the showtopic method but returns its + result directly so it can be formatted for display in an html page. + """ + try: + import pydoc_data.topics + except ImportError: + return(''' +Sorry, topic and keyword documentation is not available because the +module "pydoc_data.topics" could not be found. +''' , '') + target = self.topics.get(topic, self.keywords.get(topic)) + if not target: + raise ValueError('could not find topic') + if isinstance(target, str): + return self._gettopic(target, more_xrefs) + label, xrefs = target + doc = pydoc_data.topics.topics[label] + if more_xrefs: + xrefs = (xrefs or '') + ' ' + more_xrefs + return doc, xrefs + + def showsymbol(self, symbol): + target = self.symbols[symbol] + topic, _, xrefs = target.partition(' ') + self.showtopic(topic, xrefs) + + def listmodules(self, key=''): + if key: + self.output.write(''' +Here is a list of modules whose name or summary contains '{}'. +If there are any, enter a module name to get more help. + +'''.format(key)) + apropos(key) + else: + self.output.write(''' +Please wait a moment while I gather a list of all available modules... + +''') + modules = {} + def callback(path, modname, desc, modules=modules): + if modname and modname[-9:] == '.__init__': + modname = modname[:-9] + ' (package)' + if modname.find('.') < 0: + modules[modname] = 1 + def onerror(modname): + callback(None, modname, None) + ModuleScanner().run(callback, onerror=onerror) + self.list(modules.keys()) + self.output.write(''' +Enter any module name to get more help. Or, type "modules spam" to search +for modules whose name or summary contain the string "spam". +''') + +help = Helper() + +class ModuleScanner: + """An interruptible scanner that searches module synopses.""" + + def run(self, callback, key=None, completer=None, onerror=None): + if key: key = key.lower() + self.quit = False + seen = {} + + for modname in sys.builtin_module_names: + if modname != '__main__': + seen[modname] = 1 + if key is None: + callback(None, modname, '') + else: + name = __import__(modname).__doc__ or '' + desc = name.split('\n')[0] + name = modname + ' - ' + desc + if name.lower().find(key) >= 0: + callback(None, modname, desc) + + for importer, modname, ispkg in pkgutil.walk_packages(onerror=onerror): + if self.quit: + break + + if key is None: + callback(None, modname, '') + else: + try: + spec = importer.find_spec(modname) + except SyntaxError: + # raised by tests for bad coding cookies or BOM + continue + loader = spec.loader + if hasattr(loader, 'get_source'): + try: + source = loader.get_source(modname) + except Exception: + if onerror: + onerror(modname) + continue + desc = source_synopsis(io.StringIO(source)) or '' + if hasattr(loader, 'get_filename'): + path = loader.get_filename(modname) + else: + path = None + else: + try: + module = importlib._bootstrap._load(spec) + except ImportError: + if onerror: + onerror(modname) + continue + desc = module.__doc__.splitlines()[0] if module.__doc__ else '' + path = getattr(module,'__file__',None) + name = modname + ' - ' + desc + if name.lower().find(key) >= 0: + callback(path, modname, desc) + + if completer: + completer() + +def apropos(key): + """Print all the one-line module summaries that contain a substring.""" + def callback(path, modname, desc): + if modname[-9:] == '.__init__': + modname = modname[:-9] + ' (package)' + print(modname, desc and '- ' + desc) + def onerror(modname): + pass + with warnings.catch_warnings(): + warnings.filterwarnings('ignore') # ignore problems during import + ModuleScanner().run(callback, key, onerror=onerror) + +# --------------------------------------- enhanced web browser interface + +def _start_server(urlhandler, hostname, port): + """Start an HTTP server thread on a specific port. + + Start an HTML/text server thread, so HTML or text documents can be + browsed dynamically and interactively with a web browser. Example use: + + >>> import time + >>> import pydoc + + Define a URL handler. To determine what the client is asking + for, check the URL and content_type. + + Then get or generate some text or HTML code and return it. + + >>> def my_url_handler(url, content_type): + ... text = 'the URL sent was: (%s, %s)' % (url, content_type) + ... return text + + Start server thread on port 0. + If you use port 0, the server will pick a random port number. + You can then use serverthread.port to get the port number. + + >>> port = 0 + >>> serverthread = pydoc._start_server(my_url_handler, port) + + Check that the server is really started. If it is, open browser + and get first page. Use serverthread.url as the starting page. + + >>> if serverthread.serving: + ... import webbrowser + + The next two lines are commented out so a browser doesn't open if + doctest is run on this module. + + #... webbrowser.open(serverthread.url) + #True + + Let the server do its thing. We just need to monitor its status. + Use time.sleep so the loop doesn't hog the CPU. + + >>> starttime = time.monotonic() + >>> timeout = 1 #seconds + + This is a short timeout for testing purposes. + + >>> while serverthread.serving: + ... time.sleep(.01) + ... if serverthread.serving and time.monotonic() - starttime > timeout: + ... serverthread.stop() + ... break + + Print any errors that may have occurred. + + >>> print(serverthread.error) + None + """ + import http.server + import email.message + import select + import threading + + class DocHandler(http.server.BaseHTTPRequestHandler): + + def do_GET(self): + """Process a request from an HTML browser. + + The URL received is in self.path. + Get an HTML page from self.urlhandler and send it. + """ + if self.path.endswith('.css'): + content_type = 'text/css' + else: + content_type = 'text/html' + self.send_response(200) + self.send_header('Content-Type', '%s; charset=UTF-8' % content_type) + self.end_headers() + self.wfile.write(self.urlhandler( + self.path, content_type).encode('utf-8')) + + def log_message(self, *args): + # Don't log messages. + pass + + class DocServer(http.server.HTTPServer): + + def __init__(self, host, port, callback): + self.host = host + self.address = (self.host, port) + self.callback = callback + self.base.__init__(self, self.address, self.handler) + self.quit = False + + def serve_until_quit(self): + while not self.quit: + rd, wr, ex = select.select([self.socket.fileno()], [], [], 1) + if rd: + self.handle_request() + self.server_close() + + def server_activate(self): + self.base.server_activate(self) + if self.callback: + self.callback(self) + + class ServerThread(threading.Thread): + + def __init__(self, urlhandler, host, port): + self.urlhandler = urlhandler + self.host = host + self.port = int(port) + threading.Thread.__init__(self) + self.serving = False + self.error = None + self.docserver = None + + def run(self): + """Start the server.""" + try: + DocServer.base = http.server.HTTPServer + DocServer.handler = DocHandler + DocHandler.MessageClass = email.message.Message + DocHandler.urlhandler = staticmethod(self.urlhandler) + docsvr = DocServer(self.host, self.port, self.ready) + self.docserver = docsvr + docsvr.serve_until_quit() + except Exception as err: + self.error = err + + def ready(self, server): + self.serving = True + self.host = server.host + self.port = server.server_port + self.url = 'http://%s:%d/' % (self.host, self.port) + + def stop(self): + """Stop the server and this thread nicely""" + self.docserver.quit = True + self.join() + # explicitly break a reference cycle: DocServer.callback + # has indirectly a reference to ServerThread. + self.docserver = None + self.serving = False + self.url = None + + thread = ServerThread(urlhandler, hostname, port) + thread.start() + # Wait until thread.serving is True and thread.docserver is set + # to make sure we are really up before returning. + while not thread.error and not (thread.serving and thread.docserver): + time.sleep(.01) + return thread + + +def _url_handler(url, content_type="text/html"): + """The pydoc url handler for use with the pydoc server. + + If the content_type is 'text/css', the _pydoc.css style + sheet is read and returned if it exits. + + If the content_type is 'text/html', then the result of + get_html_page(url) is returned. + """ + class _HTMLDoc(HTMLDoc): + + def page(self, title, contents): + """Format an HTML page.""" + css_path = "pydoc_data/_pydoc.css" + css_link = ( + '' % + css_path) + return '''\ + + + + +Pydoc: %s +%s%s
%s
+''' % (title, css_link, html_navbar(), contents) + + + html = _HTMLDoc() + + def html_navbar(): + version = html.escape("%s [%s, %s]" % (platform.python_version(), + platform.python_build()[0], + platform.python_compiler())) + return """ +
+ Python %s
%s +
+
+ +
+
+ + +
  +
+ + +
+
+
+ """ % (version, html.escape(platform.platform(terse=True))) + + def html_index(): + """Module Index page.""" + + def bltinlink(name): + return '%s' % (name, name) + + heading = html.heading( + 'Index of Modules' + ) + names = [name for name in sys.builtin_module_names + if name != '__main__'] + contents = html.multicolumn(names, bltinlink) + contents = [heading, '

' + html.bigsection( + 'Built-in Modules', 'index', contents)] + + seen = {} + for dir in sys.path: + contents.append(html.index(dir, seen)) + + contents.append( + '

pydoc by Ka-Ping Yee' + '<ping@lfw.org>

') + return 'Index of Modules', ''.join(contents) + + def html_search(key): + """Search results page.""" + # scan for modules + search_result = [] + + def callback(path, modname, desc): + if modname[-9:] == '.__init__': + modname = modname[:-9] + ' (package)' + search_result.append((modname, desc and '- ' + desc)) + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore') # ignore problems during import + def onerror(modname): + pass + ModuleScanner().run(callback, key, onerror=onerror) + + # format page + def bltinlink(name): + return '%s' % (name, name) + + results = [] + heading = html.heading( + 'Search Results', + ) + for name, desc in search_result: + results.append(bltinlink(name) + desc) + contents = heading + html.bigsection( + 'key = %s' % key, 'index', '
'.join(results)) + return 'Search Results', contents + + def html_topics(): + """Index of topic texts available.""" + + def bltinlink(name): + return '%s' % (name, name) + + heading = html.heading( + 'INDEX', + ) + names = sorted(Helper.topics.keys()) + + contents = html.multicolumn(names, bltinlink) + contents = heading + html.bigsection( + 'Topics', 'index', contents) + return 'Topics', contents + + def html_keywords(): + """Index of keywords.""" + heading = html.heading( + 'INDEX', + ) + names = sorted(Helper.keywords.keys()) + + def bltinlink(name): + return '%s' % (name, name) + + contents = html.multicolumn(names, bltinlink) + contents = heading + html.bigsection( + 'Keywords', 'index', contents) + return 'Keywords', contents + + def html_topicpage(topic): + """Topic or keyword help page.""" + buf = io.StringIO() + htmlhelp = Helper(buf, buf) + contents, xrefs = htmlhelp._gettopic(topic) + if topic in htmlhelp.keywords: + title = 'KEYWORD' + else: + title = 'TOPIC' + heading = html.heading( + '%s' % title, + ) + contents = '
%s
' % html.markup(contents) + contents = html.bigsection(topic , 'index', contents) + if xrefs: + xrefs = sorted(xrefs.split()) + + def bltinlink(name): + return '%s' % (name, name) + + xrefs = html.multicolumn(xrefs, bltinlink) + xrefs = html.section('Related help topics: ', 'index', xrefs) + return ('%s %s' % (title, topic), + ''.join((heading, contents, xrefs))) + + def html_getobj(url): + obj = locate(url, forceload=1) + if obj is None and url != 'None': + raise ValueError('could not find object') + title = describe(obj) + content = html.document(obj, url) + return title, content + + def html_error(url, exc): + heading = html.heading( + 'Error', + ) + contents = '
'.join(html.escape(line) for line in + format_exception_only(type(exc), exc)) + contents = heading + html.bigsection(url, 'error', contents) + return "Error - %s" % url, contents + + def get_html_page(url): + """Generate an HTML page for url.""" + complete_url = url + if url.endswith('.html'): + url = url[:-5] + try: + if url in ("", "index"): + title, content = html_index() + elif url == "topics": + title, content = html_topics() + elif url == "keywords": + title, content = html_keywords() + elif '=' in url: + op, _, url = url.partition('=') + if op == "search?key": + title, content = html_search(url) + elif op == "topic?key": + # try topics first, then objects. + try: + title, content = html_topicpage(url) + except ValueError: + title, content = html_getobj(url) + elif op == "get?key": + # try objects first, then topics. + if url in ("", "index"): + title, content = html_index() + else: + try: + title, content = html_getobj(url) + except ValueError: + title, content = html_topicpage(url) + else: + raise ValueError('bad pydoc url') + else: + title, content = html_getobj(url) + except Exception as exc: + # Catch any errors and display them in an error page. + title, content = html_error(complete_url, exc) + return html.page(title, content) + + if url.startswith('/'): + url = url[1:] + if content_type == 'text/css': + path_here = os.path.dirname(os.path.realpath(__file__)) + css_path = os.path.join(path_here, url) + with open(css_path) as fp: + return ''.join(fp.readlines()) + elif content_type == 'text/html': + return get_html_page(url) + # Errors outside the url handler are caught by the server. + raise TypeError('unknown content type %r for url %s' % (content_type, url)) + + +def browse(port=0, *, open_browser=True, hostname='localhost'): + """Start the enhanced pydoc web server and open a web browser. + + Use port '0' to start the server on an arbitrary port. + Set open_browser to False to suppress opening a browser. + """ + import webbrowser + serverthread = _start_server(_url_handler, hostname, port) + if serverthread.error: + print(serverthread.error) + return + if serverthread.serving: + server_help_msg = 'Server commands: [b]rowser, [q]uit' + if open_browser: + webbrowser.open(serverthread.url) + try: + print('Server ready at', serverthread.url) + print(server_help_msg) + while serverthread.serving: + cmd = input('server> ') + cmd = cmd.lower() + if cmd == 'q': + break + elif cmd == 'b': + webbrowser.open(serverthread.url) + else: + print(server_help_msg) + except (KeyboardInterrupt, EOFError): + print() + finally: + if serverthread.serving: + serverthread.stop() + print('Server stopped') + + +# -------------------------------------------------- command-line interface + +def ispath(x): + return isinstance(x, str) and x.find(os.sep) >= 0 + +def _get_revised_path(given_path, argv0): + """Ensures current directory is on returned path, and argv0 directory is not + + Exception: argv0 dir is left alone if it's also pydoc's directory. + + Returns a new path entry list, or None if no adjustment is needed. + """ + # Scripts may get the current directory in their path by default if they're + # run with the -m switch, or directly from the current directory. + # The interactive prompt also allows imports from the current directory. + + # Accordingly, if the current directory is already present, don't make + # any changes to the given_path + if '' in given_path or os.curdir in given_path or os.getcwd() in given_path: + return None + + # Otherwise, add the current directory to the given path, and remove the + # script directory (as long as the latter isn't also pydoc's directory. + stdlib_dir = os.path.dirname(__file__) + script_dir = os.path.dirname(argv0) + revised_path = given_path.copy() + if script_dir in given_path and not os.path.samefile(script_dir, stdlib_dir): + revised_path.remove(script_dir) + revised_path.insert(0, os.getcwd()) + return revised_path + + +# Note: the tests only cover _get_revised_path, not _adjust_cli_path itself +def _adjust_cli_sys_path(): + """Ensures current directory is on sys.path, and __main__ directory is not. + + Exception: __main__ dir is left alone if it's also pydoc's directory. + """ + revised_path = _get_revised_path(sys.path, sys.argv[0]) + if revised_path is not None: + sys.path[:] = revised_path + + +def cli(): + """Command-line interface (looks at sys.argv to decide what to do).""" + import getopt + class BadUsage(Exception): pass + + _adjust_cli_sys_path() + + try: + opts, args = getopt.getopt(sys.argv[1:], 'bk:n:p:w') + writing = False + start_server = False + open_browser = False + port = 0 + hostname = 'localhost' + for opt, val in opts: + if opt == '-b': + start_server = True + open_browser = True + if opt == '-k': + apropos(val) + return + if opt == '-p': + start_server = True + port = val + if opt == '-w': + writing = True + if opt == '-n': + start_server = True + hostname = val + + if start_server: + browse(port, hostname=hostname, open_browser=open_browser) + return + + if not args: raise BadUsage + for arg in args: + if ispath(arg) and not os.path.exists(arg): + print('file %r does not exist' % arg) + sys.exit(1) + try: + if ispath(arg) and os.path.isfile(arg): + arg = importfile(arg) + if writing: + if ispath(arg) and os.path.isdir(arg): + writedocs(arg) + else: + writedoc(arg) + else: + help.help(arg, is_cli=True) + except (ImportError, ErrorDuringImport) as value: + print(value) + sys.exit(1) + + except (getopt.error, BadUsage): + cmd = os.path.splitext(os.path.basename(sys.argv[0]))[0] + print("""pydoc - the Python documentation tool + +{cmd} ... + Show text documentation on something. may be the name of a + Python keyword, topic, function, module, or package, or a dotted + reference to a class or function within a module or module in a + package. If contains a '{sep}', it is used as the path to a + Python source file to document. If name is 'keywords', 'topics', + or 'modules', a listing of these things is displayed. + +{cmd} -k + Search for a keyword in the synopsis lines of all available modules. + +{cmd} -n + Start an HTTP server with the given hostname (default: localhost). + +{cmd} -p + Start an HTTP server on the given port on the local machine. Port + number 0 can be used to get an arbitrary unused port. + +{cmd} -b + Start an HTTP server on an arbitrary unused port and open a web browser + to interactively browse documentation. This option can be used in + combination with -n and/or -p. + +{cmd} -w ... + Write out the HTML documentation for a module to a file in the current + directory. If contains a '{sep}', it is treated as a filename; if + it names a directory, documentation is written for all the contents. +""".format(cmd=cmd, sep=os.sep)) + +if __name__ == '__main__': + cli() diff --git a/crates/weavepy-vm/src/stdlib/python/sysconfig.py b/crates/weavepy-vm/src/stdlib/python/sysconfig.py new file mode 100644 index 0000000..01489d1 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/sysconfig.py @@ -0,0 +1,184 @@ +"""Access to WeavePy's configuration information. + +A faithful-in-shape, minimal implementation of CPython's ``sysconfig`` +public API. ``sysconfig`` is *definitionally* implementation-provided — +its values describe the running interpreter's build — so the honest +WeavePy answer reports WeavePy's own layout: a self-contained binary +whose stdlib is frozen into the executable (no on-disk ``lib/pythonX.Y`` +tree). Consumers in the conformance surface (`pydoc.getdocloc`, +`platform`, `site`) only need the call shapes and sane strings. +""" + +import os +import sys + +__all__ = [ + 'get_config_h_filename', + 'get_config_var', + 'get_config_vars', + 'get_default_scheme', + 'get_makefile_filename', + 'get_path', + 'get_path_names', + 'get_paths', + 'get_platform', + 'get_python_version', + 'get_scheme_names', + 'is_python_build', + 'parse_config_h', +] + +_PY_VERSION = sys.version.split()[0] +_PY_VERSION_SHORT = '.'.join(_PY_VERSION.split('.')[:2]) +_PY_VERSION_SHORT_NO_DOT = _PY_VERSION_SHORT.replace('.', '') + +_PREFIX = getattr(sys, 'prefix', '') or os.path.dirname( + getattr(sys, 'executable', '') or '/usr/local/bin') +_EXEC_PREFIX = getattr(sys, 'exec_prefix', _PREFIX) or _PREFIX + +# One scheme; WeavePy is a single self-contained binary, so every +# location resolves under the executable's prefix. +_SCHEME = { + 'stdlib': '{installed_base}/lib/python{py_version_short}', + 'platstdlib': '{platbase}/lib/python{py_version_short}', + 'purelib': '{base}/lib/python{py_version_short}/site-packages', + 'platlib': '{platbase}/lib/python{py_version_short}/site-packages', + 'include': '{installed_base}/include/python{py_version_short}', + 'platinclude': '{installed_platbase}/include/python{py_version_short}', + 'scripts': '{base}/bin', + 'data': '{base}', +} + +_CONFIG_VARS = None + + +def _expand(template, vars): + out = template + for key, value in vars.items(): + out = out.replace('{%s}' % key, str(value)) + return out + + +def _init_config_vars(): + global _CONFIG_VARS + if _CONFIG_VARS is None: + _CONFIG_VARS = { + 'prefix': _PREFIX, + 'exec_prefix': _EXEC_PREFIX, + 'base': _PREFIX, + 'platbase': _EXEC_PREFIX, + 'installed_base': _PREFIX, + 'installed_platbase': _EXEC_PREFIX, + 'py_version': _PY_VERSION, + 'py_version_short': _PY_VERSION_SHORT, + 'py_version_nodot': _PY_VERSION_SHORT_NO_DOT, + 'abiflags': '', + 'EXT_SUFFIX': '.so', + 'SOABI': 'weavepy', + 'Py_DEBUG': 0, + 'Py_ENABLE_SHARED': 0, + 'Py_GIL_DISABLED': 0, + 'LIBDIR': os.path.join(_PREFIX, 'lib'), + 'INCLUDEPY': os.path.join(_PREFIX, 'include', + 'python' + _PY_VERSION_SHORT), + 'projectbase': os.path.dirname( + getattr(sys, 'executable', '') or _PREFIX), + 'platlibdir': 'lib', + 'userbase': os.path.expanduser('~/.local'), + } + return _CONFIG_VARS + + +def get_config_vars(*args): + vars = _init_config_vars() + if args: + return [vars.get(name) for name in args] + return vars + + +def get_config_var(name): + return get_config_vars().get(name) + + +def get_scheme_names(): + return ('weavepy',) + + +def get_default_scheme(): + return 'weavepy' + + +def get_path_names(): + return tuple(_SCHEME) + + +def get_paths(scheme=get_default_scheme(), vars=None, expand=True): + all_vars = dict(_init_config_vars()) + if vars is not None: + all_vars.update(vars) + if expand: + return {name: _expand(template, all_vars) + for name, template in _SCHEME.items()} + return dict(_SCHEME) + + +def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): + paths = get_paths(scheme, vars, expand) + try: + return paths[name] + except KeyError: + raise KeyError('unknown path name %r' % (name,)) from None + + +def get_python_version(): + return _PY_VERSION_SHORT + + +def get_platform(): + if sys.platform == 'darwin': + import platform as _platform + machine = _platform.machine() or 'arm64' + return 'macosx-11.0-%s' % machine + if sys.platform.startswith('linux'): + import platform as _platform + machine = _platform.machine() or 'x86_64' + return 'linux-%s' % machine + return sys.platform + + +def is_python_build(check_home=None): + return False + + +def get_makefile_filename(): + return os.path.join(get_path('stdlib'), 'config', 'Makefile') + + +def get_config_h_filename(): + return os.path.join(get_path('platinclude'), 'pyconfig.h') + + +def parse_config_h(fp, vars=None): + """Parse a config.h-style file (name/value pairs).""" + import re + if vars is None: + vars = {} + define_rx = re.compile('#define ([A-Z][A-Za-z0-9_]+) (.*)\n') + undef_rx = re.compile('/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n') + while True: + line = fp.readline() + if not line: + break + m = define_rx.match(line) + if m: + n, v = m.group(1, 2) + try: + v = int(v) + except ValueError: + pass + vars[n] = v + else: + m = undef_rx.match(line) + if m: + vars[m.group(1)] = 0 + return vars diff --git a/crates/weavepy-vm/src/stdlib/python/test_support_init.py b/crates/weavepy-vm/src/stdlib/python/test_support_init.py index 7e3a103..e8cdd53 100644 --- a/crates/weavepy-vm/src/stdlib/python/test_support_init.py +++ b/crates/weavepy-vm/src/stdlib/python/test_support_init.py @@ -849,6 +849,7 @@ def _requires_module(name): TEST_HOME_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_SUPPORT_DIR = TEST_HOME_DIR STDLIB_DIR = os.path.dirname(os.path.dirname(TEST_HOME_DIR)) +REPO_ROOT = os.path.dirname(STDLIB_DIR) # --------------------------------------------------------------------------- diff --git a/crates/weavepy-vm/src/stdlib/python/token.py b/crates/weavepy-vm/src/stdlib/python/token.py new file mode 100644 index 0000000..54d7cdc --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/token.py @@ -0,0 +1,141 @@ +"""Token constants.""" +# Auto-generated by Tools/build/generate_token.py + +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] + +ENDMARKER = 0 +NAME = 1 +NUMBER = 2 +STRING = 3 +NEWLINE = 4 +INDENT = 5 +DEDENT = 6 +LPAR = 7 +RPAR = 8 +LSQB = 9 +RSQB = 10 +COLON = 11 +COMMA = 12 +SEMI = 13 +PLUS = 14 +MINUS = 15 +STAR = 16 +SLASH = 17 +VBAR = 18 +AMPER = 19 +LESS = 20 +GREATER = 21 +EQUAL = 22 +DOT = 23 +PERCENT = 24 +LBRACE = 25 +RBRACE = 26 +EQEQUAL = 27 +NOTEQUAL = 28 +LESSEQUAL = 29 +GREATEREQUAL = 30 +TILDE = 31 +CIRCUMFLEX = 32 +LEFTSHIFT = 33 +RIGHTSHIFT = 34 +DOUBLESTAR = 35 +PLUSEQUAL = 36 +MINEQUAL = 37 +STAREQUAL = 38 +SLASHEQUAL = 39 +PERCENTEQUAL = 40 +AMPEREQUAL = 41 +VBAREQUAL = 42 +CIRCUMFLEXEQUAL = 43 +LEFTSHIFTEQUAL = 44 +RIGHTSHIFTEQUAL = 45 +DOUBLESTAREQUAL = 46 +DOUBLESLASH = 47 +DOUBLESLASHEQUAL = 48 +AT = 49 +ATEQUAL = 50 +RARROW = 51 +ELLIPSIS = 52 +COLONEQUAL = 53 +EXCLAMATION = 54 +OP = 55 +TYPE_IGNORE = 56 +TYPE_COMMENT = 57 +SOFT_KEYWORD = 58 +FSTRING_START = 59 +FSTRING_MIDDLE = 60 +FSTRING_END = 61 +COMMENT = 62 +NL = 63 +# These aren't used by the C tokenizer but are needed for tokenize.py +ERRORTOKEN = 64 +ENCODING = 65 +N_TOKENS = 66 +# Special definitions for cooperation with parser +NT_OFFSET = 256 + +tok_name = {value: name + for name, value in globals().items() + if isinstance(value, int) and not name.startswith('_')} +__all__.extend(tok_name.values()) + +EXACT_TOKEN_TYPES = { + '!': EXCLAMATION, + '!=': NOTEQUAL, + '%': PERCENT, + '%=': PERCENTEQUAL, + '&': AMPER, + '&=': AMPEREQUAL, + '(': LPAR, + ')': RPAR, + '*': STAR, + '**': DOUBLESTAR, + '**=': DOUBLESTAREQUAL, + '*=': STAREQUAL, + '+': PLUS, + '+=': PLUSEQUAL, + ',': COMMA, + '-': MINUS, + '-=': MINEQUAL, + '->': RARROW, + '.': DOT, + '...': ELLIPSIS, + '/': SLASH, + '//': DOUBLESLASH, + '//=': DOUBLESLASHEQUAL, + '/=': SLASHEQUAL, + ':': COLON, + ':=': COLONEQUAL, + ';': SEMI, + '<': LESS, + '<<': LEFTSHIFT, + '<<=': LEFTSHIFTEQUAL, + '<=': LESSEQUAL, + '=': EQUAL, + '==': EQEQUAL, + '>': GREATER, + '>=': GREATEREQUAL, + '>>': RIGHTSHIFT, + '>>=': RIGHTSHIFTEQUAL, + '@': AT, + '@=': ATEQUAL, + '[': LSQB, + ']': RSQB, + '^': CIRCUMFLEX, + '^=': CIRCUMFLEXEQUAL, + '{': LBRACE, + '|': VBAR, + '|=': VBAREQUAL, + '}': RBRACE, + '~': TILDE, +} + +def ISTERMINAL(x): + return x < NT_OFFSET + +def ISNONTERMINAL(x): + return x >= NT_OFFSET + +def ISEOF(x): + return x == ENDMARKER diff --git a/crates/weavepy-vm/src/stdlib/python/tokenize.py b/crates/weavepy-vm/src/stdlib/python/tokenize.py new file mode 100644 index 0000000..69284ca --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/tokenize.py @@ -0,0 +1,613 @@ +"""Tokenization help for Python programs — WeavePy port. + +CPython 3.13's ``tokenize`` is a thin shell over the C ``_tokenize`` +accelerator (``TokenizerIter``). WeavePy has no C extensions, so — +following the same approach as the RFC 0035 pure-Python ``re`` engine — +this module is the *classic* pure-Python tokenizer (the reference +implementation that shipped in ``Lib/tokenize.py`` through CPython 3.11) +exposing the 3.13 public surface: ``TokenInfo``, ``tokenize``, +``generate_tokens``, ``detect_encoding``, ``untokenize``, ``open``, +``TokenError`` and the ``token`` constants. + +Known fidelity gap vs. 3.13: f-strings are produced as single ``STRING`` +tokens (the pre-PEP-701 tokenization) rather than ``FSTRING_START`` / +``FSTRING_MIDDLE`` / ``FSTRING_END`` triples. +""" + +from builtins import open as _builtin_open +import collections +import itertools as _itertools +import re +import sys + +from token import * +from token import EXACT_TOKEN_TYPES +import token + +__all__ = token.__all__ + ["tokenize", "generate_tokens", "detect_encoding", + "untokenize", "TokenInfo", "open", "TokenError"] + +cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) +blank_re = re.compile(rb'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) + + +class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): + def __repr__(self): + annotated_type = '%d (%s)' % (self.type, tok_name[self.type]) + return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % + self._replace(type=annotated_type)) + + @property + def exact_type(self): + if self.type == OP and self.string in EXACT_TOKEN_TYPES: + return EXACT_TOKEN_TYPES[self.string] + else: + return self.type + + +def group(*choices): return '(' + '|'.join(choices) + ')' +def any(*choices): return group(*choices) + '*' +def maybe(*choices): return group(*choices) + '?' + +# Note: we use unicode matching for names ("\w") but ascii matching for +# number literals. +Whitespace = r'[ \f\t]*' +Comment = r'#[^\r\n]*' +Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) +Name = r'\w+' + +Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+' +Binnumber = r'0[bB](?:_?[01])+' +Octnumber = r'0[oO](?:_?[0-7])+' +Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)' +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) +Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*' +Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?', + r'\.[0-9](?:_?[0-9])*') + maybe(Exponent) +Expfloat = r'[0-9](?:_?[0-9])*' + Exponent +Floatnumber = group(Pointfloat, Expfloat) +Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]') +Number = group(Imagnumber, Floatnumber, Intnumber) + + +# Return the empty string, plus all of the valid string prefixes. +def _all_string_prefixes(): + # The valid string prefixes. Only contain the lowercase versions, + # and don't contain any permutations (include 'fr', but not + # 'rf'). The various permutations will be generated. + _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + # if we add binary f-strings, add: ['fb', 'fbr'] + result = {''} + for prefix in _valid_string_prefixes: + for t in _itertools.permutations(prefix): + # create a list with upper and lower versions of each + # character + for u in _itertools.product(*[(c, c.upper()) for c in t]): + result.add(''.join(u)) + return result + +StringPrefix = group(*_all_string_prefixes()) + +# Tail end of ' string. +Single = r"[^'\\]*(?:\\.[^'\\]*)*'" +# Tail end of " string. +Double = r'[^"\\]*(?:\\.[^"\\]*)*"' +# Tail end of ''' string. +Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" +# Tail end of """ string. +Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' +Triple = group(StringPrefix + "'''", StringPrefix + '"""') +# Single-line ' or " string. +String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') + +# Sorting in reverse order puts the long operators before their prefixes. +# Otherwise if = came before ==, == would get interpreted as +# two instances of =. +Special = group(*map(re.escape, sorted(EXACT_TOKEN_TYPES, reverse=True))) +Funny = group(r'\r?\n', Special) + +PlainToken = group(Number, Funny, String, Name) +Token = Ignore + PlainToken + +# First (or only) line of ' or " string. +ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + + group("'", r'\\\r?\n'), + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + group('"', r'\\\r?\n')) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) +PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) + +# For a given string prefix plus quotes, endpats maps it to a regex +# to match the remainder of that string. _prefixes can be empty, for +# a normal single or triple quoted string (with no prefix). +endpats = {} +for _prefix in _all_string_prefixes(): + endpats[_prefix + "'"] = Single + endpats[_prefix + '"'] = Double + endpats[_prefix + "'''"] = Single3 + endpats[_prefix + '"""'] = Double3 +del _prefix + +# A set of all of the single and triple quoted string prefixes, +# including the opening quotes. +single_quoted = set() +triple_quoted = set() +for t in _all_string_prefixes(): + for u in (t + '"', t + "'"): + single_quoted.add(u) + for u in (t + '"""', t + "'''"): + triple_quoted.add(u) +del t, u + +tabsize = 8 + +# Compile the workhorse patterns once at import; the tokenizer loop +# matches one of these per token. +_pseudo_prog = re.compile(PseudoToken) +_endprogs = {pat: re.compile(pat) for pat in set(endpats.values())} + + +class TokenError(Exception): pass + + +class StopTokenizing(Exception): pass + + +class Untokenizer: + + def __init__(self): + self.tokens = [] + self.prev_row = 1 + self.prev_col = 0 + self.encoding = None + + def add_whitespace(self, start): + row, col = start + if row < self.prev_row or row == self.prev_row and col < self.prev_col: + raise ValueError("start ({},{}) precedes previous end ({},{})" + .format(row, col, self.prev_row, self.prev_col)) + row_offset = row - self.prev_row + if row_offset: + self.tokens.append("\\\n" * row_offset) + self.prev_col = 0 + col_offset = col - self.prev_col + if col_offset: + self.tokens.append(" " * col_offset) + + def untokenize(self, iterable): + it = iter(iterable) + indents = [] + startline = False + for t in it: + if len(t) == 2: + self.compat(t, it) + break + tok_type, token, start, end, line = t + if tok_type == ENCODING: + self.encoding = token + continue + if tok_type == ENDMARKER: + break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + start_row, start_col = start + if start_col >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False + self.add_whitespace(start) + self.tokens.append(token) + self.prev_row, self.prev_col = end + if tok_type in (NEWLINE, NL): + self.prev_row += 1 + self.prev_col = 0 + return "".join(self.tokens) + + def compat(self, token, iterable): + indents = [] + toks_append = self.tokens.append + startline = token[0] in (NEWLINE, NL) + prevstring = False + + for tok in _itertools.chain([token], iterable): + toknum, tokval = tok[:2] + if toknum == ENCODING: + self.encoding = tokval + continue + + if toknum in (NAME, NUMBER): + tokval += ' ' + + # Insert a space between two consecutive strings + if toknum == STRING: + if prevstring: + tokval = ' ' + tokval + prevstring = True + else: + prevstring = False + + if toknum == INDENT: + indents.append(tokval) + continue + elif toknum == DEDENT: + indents.pop() + continue + elif toknum in (NEWLINE, NL): + startline = True + elif startline and indents: + toks_append(indents[-1]) + startline = False + toks_append(tokval) + + +def untokenize(iterable): + """Transform tokens back into Python source code. + It returns a bytes object, encoded using the ENCODING + token, which is the first token sequence output by tokenize. + + Each element returned by the iterable must be a token sequence + with at least two elements, a token number and token value. If + only two tokens are passed, the resulting output is poor. + + The result is guaranteed to tokenize back to match the input so + that the conversion is lossless and round-trips are assured. + The guarantee applies only to the token type and token string as + the spacing between tokens (column positions) may change. + """ + ut = Untokenizer() + out = ut.untokenize(iterable) + if ut.encoding is not None: + out = out.encode(ut.encoding) + return out + + +def _get_normal_name(orig_enc): + """Imitates get_normal_name in Parser/tokenizer/helpers.c.""" + # Only care about the first 12 characters. + enc = orig_enc[:12].lower().replace("_", "-") + if enc == "utf-8" or enc.startswith("utf-8-"): + return "utf-8" + if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \ + enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")): + return "iso-8859-1" + return orig_enc + + +def detect_encoding(readline): + """ + The detect_encoding() function is used to detect the encoding that should + be used to decode a Python source file. It requires one argument, readline, + in the same way as the tokenize() generator. + + It will call readline a maximum of twice, and return the encoding used + (as a string) and a list of any lines (left as bytes) it has read in. + + It detects the encoding from the presence of a UTF-8 BOM or an encoding + cookie as specified in PEP-0263. If both a BOM and a cookie are present, + but disagree, a SyntaxError will be raised. If the encoding cookie is an + invalid charset, raise a SyntaxError. Note that if a UTF-8 BOM is found, + 'utf-8-sig' is returned. + + If no encoding is specified, then the default of 'utf-8' will be returned. + """ + try: + filename = readline.__self__.name + except AttributeError: + filename = None + bom_found = False + encoding = None + default = 'utf-8' + + def read_or_stop(): + try: + return readline() + except StopIteration: + return b'' + + def find_cookie(line): + try: + # Decode as UTF-8. Either the line is an encoding declaration, + # in which case it should be pure ASCII, or it must be UTF-8 + # per default encoding. + line_string = line.decode('utf-8') + except UnicodeDecodeError: + msg = "invalid or missing encoding declaration" + if filename is not None: + msg = '{} for {!r}'.format(msg, filename) + raise SyntaxError(msg) + + match = cookie_re.match(line_string) + if not match: + return None + encoding = _get_normal_name(match.group(1)) + try: + import codecs + codecs.lookup(encoding) + except LookupError: + # This behaviour mimics the Python interpreter + if filename is None: + msg = "unknown encoding: " + encoding + else: + msg = "unknown encoding for {!r}: {}".format(filename, + encoding) + raise SyntaxError(msg) + + if bom_found: + if encoding != 'utf-8': + # This behaviour mimics the Python interpreter + if filename is None: + msg = 'encoding problem: utf-8' + else: + msg = 'encoding problem for {!r}: utf-8'.format(filename) + raise SyntaxError(msg) + encoding += '-sig' + return encoding + + first = read_or_stop() + if first.startswith(b'\xef\xbb\xbf'): + bom_found = True + first = first[3:] + default = 'utf-8-sig' + if not first: + return default, [] + + encoding = find_cookie(first) + if encoding: + return encoding, [first] + if not blank_re.match(first): + return default, [first] + + second = read_or_stop() + if not second: + return default, [first] + + encoding = find_cookie(second) + if encoding: + return encoding, [first, second] + + return default, [first, second] + + +def open(filename): + """Open a file in read only mode using the encoding detected by + detect_encoding(). + """ + # CPython wraps the original binary buffer in a TextIOWrapper; + # WeavePy's `io` has no public TextIOWrapper-over-buffer, so we + # detect on a first binary pass and reopen in text mode with the + # detected encoding — same observable contract. + buffer = _builtin_open(filename, 'rb') + try: + encoding, lines = detect_encoding(buffer.readline) + finally: + buffer.close() + if encoding == 'utf-8-sig': + encoding = 'utf-8' + text = _builtin_open(filename, 'r', encoding=encoding) + return text + + +def tokenize(readline): + """ + The tokenize() generator requires one argument, readline, which + must be a callable object which provides the same interface as the + readline() method of built-in file objects. Each call to the function + should return one line of input as bytes. Alternatively, readline + can be a callable function terminating with StopIteration: + readline = open(myfile, 'rb').__next__ # Example of alternate readline + + The generator produces 5-tuples with these members: the token type; the + token string; a 2-tuple (srow, scol) of ints specifying the row and + column where the token begins in the source; a 2-tuple (erow, ecol) of + ints specifying the row and column where the token ends in the source; + and the line on which the token was found. The line passed is the + physical line. + + The first token sequence will always be an ENCODING token + which tells you which encoding was used to decode the bytes stream. + """ + encoding, consumed = detect_encoding(readline) + empty = _itertools.repeat(b"") + rl_gen = _itertools.chain(consumed, iter(readline, b""), empty) + return _tokenize(rl_gen.__next__, encoding) + + +def _tokenize(readline, encoding): + lnum = parenlev = continued = 0 + numchars = '0123456789' + contstr, needcont = '', 0 + contline = None + indents = [0] + + if encoding is not None: + if encoding == "utf-8-sig": + # BOM will already have been stripped. + encoding = "utf-8" + yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') + last_line = b'' + line = b'' + while True: # loop over lines in stream + try: + # We capture the value of the line variable here because + # readline uses the empty string `''` to signal end of input, + # unlike the `None` default of `line` we use here. + last_line = line + line = readline() + except StopIteration: + line = b'' + + if encoding is not None and isinstance(line, bytes): + line = line.decode(encoding) + lnum += 1 + pos, max = 0, len(line) + + if contstr: # continued string + if not line: + raise TokenError("EOF in multi-line string", strstart) + endmatch = endprog.match(line) + if endmatch: + pos = end = endmatch.end(0) + yield TokenInfo(STRING, contstr + line[:end], + strstart, (lnum, end), contline + line) + contstr, needcont = '', 0 + contline = None + elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': + yield TokenInfo(ERRORTOKEN, contstr + line, + strstart, (lnum, len(line)), contline) + contstr = '' + contline = None + continue + else: + contstr = contstr + line + contline = contline + line + continue + + elif parenlev == 0 and not continued: # new statement + if not line: break + column = 0 + while pos < max: # measure leading whitespace + if line[pos] == ' ': + column += 1 + elif line[pos] == '\t': + column = (column//tabsize + 1)*tabsize + elif line[pos] == '\f': + column = 0 + else: + break + pos += 1 + if pos == max: + break + + if line[pos] in '#\r\n': # skip comments or blank lines + if line[pos] == '#': + comment_token = line[pos:].rstrip('\r\n') + yield TokenInfo(COMMENT, comment_token, + (lnum, pos), (lnum, pos + len(comment_token)), line) + pos += len(comment_token) + + yield TokenInfo(NL, line[pos:], + (lnum, pos), (lnum, len(line)), line) + continue + + if column > indents[-1]: # count indents or dedents + indents.append(column) + yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) + while column < indents[-1]: + if column not in indents: + raise IndentationError( + "unindent does not match any outer indentation level", + ("", lnum, pos, line)) + indents = indents[:-1] + + yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) + + else: # continued statement + if not line: + raise TokenError("EOF in multi-line statement", (lnum, 0)) + continued = 0 + + while pos < max: + pseudomatch = _pseudo_prog.match(line, pos) + if pseudomatch: # scan for tokens + start, end = pseudomatch.span(1) + spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue + token, initial = line[start:end], line[start] + + if (initial in numchars or + (initial == '.' and token != '.' and token != '...')): # ordinary number + yield TokenInfo(NUMBER, token, spos, epos, line) + elif initial in '\r\n': + if parenlev > 0: + yield TokenInfo(NL, token, spos, epos, line) + else: + yield TokenInfo(NEWLINE, token, spos, epos, line) + + elif initial == '#': + assert not token.endswith("\n") + yield TokenInfo(COMMENT, token, spos, epos, line) + + elif token in triple_quoted: + endprog = _endprogs[endpats[token]] + endmatch = endprog.match(line, pos) + if endmatch: # all on one line + pos = endmatch.end(0) + token = line[start:pos] + yield TokenInfo(STRING, token, spos, (lnum, pos), line) + else: + strstart = (lnum, start) # multiple lines + contstr = line[start:] + contline = line + break + + # Check up to the first 3 chars of the token to see if + # they're in the single_quoted set. If so, they start + # a string. + # We're using the first 3, because we're looking for + # "rb'" (for example) at the start of the token. If + # we switch to longer prefixes, this needs to be + # adjusted. + # Note that initial == token[:1]. + # Also note that single quote checking must come after + # triple quote checking (above). + elif (initial in single_quoted or + token[:2] in single_quoted or + token[:3] in single_quoted): + if token[-1] == '\n': # continued string + strstart = (lnum, start) + # Again, we're computing the matching regex here + # by using the first few chars of the token to + # find the corresponding endpat. + endpat = (endpats.get(initial) or + endpats.get(token[1]) or + endpats.get(token[2])) + endprog = _endprogs[endpat] + contstr, needcont = line[start:], 1 + contline = line + break + else: # ordinary string + yield TokenInfo(STRING, token, spos, epos, line) + + elif initial.isidentifier(): # ordinary name + yield TokenInfo(NAME, token, spos, epos, line) + elif initial == '\\': # continued stmt + continued = 1 + else: + if initial in '([{': + parenlev += 1 + elif initial in ')]}': + parenlev -= 1 + yield TokenInfo(OP, token, spos, epos, line) + else: + yield TokenInfo(ERRORTOKEN, line[pos], + (lnum, pos), (lnum, pos+1), line) + pos += 1 + + # Add an implicit NEWLINE if the input doesn't end in one + if last_line and last_line[-1] not in '\r\n' and \ + not last_line.strip().startswith('#'): + yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), + (lnum - 1, len(last_line) + 1), '') + for indent in indents[1:]: # pop remaining indent levels + yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') + yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') + + +def generate_tokens(readline): + """Tokenize a source reading Python code as unicode strings. + + This has the same API as tokenize(), except that it expects the *readline* + callable to return str objects instead of bytes. + """ + return _tokenize(readline, None) diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index 0ca0968..69d01ad 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -415,6 +415,13 @@ pub fn build(cache: &ModuleCache) -> Rc { DictKey(Object::from_static("platform")), Object::from_static(host_platform()), ); + // CPython-on-macOS build detail: the framework name when built + // as a macOS framework, `""` otherwise (the common case, and + // ours). `pydoc`/`platform`/`site` read it unconditionally. + d.insert( + DictKey(Object::from_static("_framework")), + Object::from_static(""), + ); d.insert( DictKey(Object::from_static("byteorder")), Object::from_static(if cfg!(target_endian = "little") { diff --git a/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs b/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs new file mode 100644 index 0000000..24bedf0 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs @@ -0,0 +1,67 @@ +//! Native stand-in for CPython's `_testinternalcapi` C test helper. +//! +//! CPython's regression suite imports this extension to observe +//! interpreter internals. WeavePy implements the handful of probes the +//! conformance targets use, mapped onto *our* equivalent internal +//! state rather than faked answers: +//! +//! - `has_inline_values(obj)` — CPython 3.13 reports whether an +//! instance's attributes still live in the object's inline value +//! array (no materialised dict escape). WeavePy instances always +//! carry a dict, but the *observable lifecycle* CPython tests — +//! fresh managed-dict instances are inline, `del obj.__dict__` / +//! `obj.__dict__ = d` and attribute-count blowups de-inline — is +//! tracked faithfully via [`PyInstance::inline_values`] plus a +//! capacity check mirroring CPython's shared-keys limit (30). + +use crate::sync::Rc; +use crate::sync::RefCell; + +use crate::error::RuntimeError; +use crate::import::ModuleCache; +use crate::object::{BuiltinFn, DictData, DictKey, Object, PyModule}; + +/// CPython's `SHARED_KEYS_MAX_SIZE`: instances whose dict outgrows the +/// shared-keys capacity stop using inline values. +const INLINE_CAPACITY: usize = 30; + +fn has_inline_values(args: &[Object]) -> Result { + let inline = match args.first() { + Some(Object::Instance(inst)) => { + inst.class.has_managed_dict() + && !inst.class.has_var_sized_base() + && inst.inline_values.get() + && inst.dict.borrow().len() <= INLINE_CAPACITY + } + _ => false, + }; + Ok(Object::Bool(inline)) +} + +pub fn build(_cache: &ModuleCache) -> Rc { + let dict = Rc::new(RefCell::new(DictData::new())); + { + let mut d = dict.borrow_mut(); + d.insert( + DictKey(Object::from_static("__name__")), + Object::from_static("_testinternalcapi"), + ); + d.insert( + DictKey(Object::from_static("__doc__")), + Object::from_static("WeavePy stand-in for CPython internal-API test probes."), + ); + d.insert( + DictKey(Object::from_static("has_inline_values")), + Object::Builtin(Rc::new(BuiltinFn { + name: "has_inline_values", + call: Box::new(has_inline_values), + call_kw: None, + })), + ); + } + Rc::new(PyModule { + name: "_testinternalcapi".to_owned(), + filename: None, + dict, + }) +} diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs index 6c57dd3..02be674 100644 --- a/crates/weavepy-vm/src/stdlib/thread_real.rs +++ b/crates/weavepy-vm/src/stdlib/thread_real.rs @@ -278,6 +278,7 @@ fn make_lock_object(lock: Arc) -> Object { class: lock_type(), dict, native: None, + inline_values: crate::sync::Cell::new(true), }); Object::Instance(inst) } @@ -378,6 +379,7 @@ fn make_rlock_object(rlock: Arc) -> Object { class: rlock_type(), dict, native: None, + inline_values: crate::sync::Cell::new(true), }); Object::Instance(inst) } diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs index 900e8b5..7453865 100644 --- a/crates/weavepy-vm/src/stdlib/weakref_real.rs +++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs @@ -338,6 +338,7 @@ fn make_ref_object(target: Object, callback: Option, kind_tag: u8) -> Ob class, dict, native: None, + inline_values: crate::sync::Cell::new(true), })) } diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index 3565800..bd582ff 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -149,6 +149,115 @@ impl TypeObject { Ok(ty) } + /// Does this type have a CPython "managed `__dict__`" — i.e. do its + /// instances carry an attribute dict? True for user-defined classes + /// whose MRO doesn't declare slots-without-dict the whole way down. + pub fn has_managed_dict(&self) -> bool { + !self.flags.is_builtin && !self.forbids_dict + } + + /// Does this type inherit from a *variable-sized* built-in + /// (`tp_itemsize != 0` in CPython: `int`, `tuple`, `str`, `bytes`, + /// `type`)? Such types get a managed dict but no inline values. + pub fn has_var_sized_base(&self) -> bool { + self.mro.borrow().iter().any(|t| { + t.flags.is_builtin + && matches!(t.name.as_str(), "int" | "tuple" | "str" | "bytes" | "type") + }) + } + + /// CPython `type.__flags__` (`tp_flags`), computed from this type's + /// observable properties. Covers the documented/queried bits: + /// inline-values + managed-dict (`test_class`), heap/base/ready/gc, + /// abstractness, and the `*_SUBCLASS` fast-classification bits. + pub fn flags_bits(&self) -> i64 { + const INLINE_VALUES: i64 = 1 << 2; + const MANAGED_WEAKREF: i64 = 1 << 3; + const MANAGED_DICT: i64 = 1 << 4; + const IMMUTABLETYPE: i64 = 1 << 8; + const HEAPTYPE: i64 = 1 << 9; + const BASETYPE: i64 = 1 << 10; + const READY: i64 = 1 << 12; + const HAVE_GC: i64 = 1 << 14; + const IS_ABSTRACT: i64 = 1 << 20; + const LONG_SUBCLASS: i64 = 1 << 24; + const LIST_SUBCLASS: i64 = 1 << 25; + const TUPLE_SUBCLASS: i64 = 1 << 26; + const BYTES_SUBCLASS: i64 = 1 << 27; + const UNICODE_SUBCLASS: i64 = 1 << 28; + const DICT_SUBCLASS: i64 = 1 << 29; + const BASE_EXC_SUBCLASS: i64 = 1 << 30; + const TYPE_SUBCLASS: i64 = 1 << 31; + + let mut bits = READY; + if self.flags.is_builtin { + bits |= IMMUTABLETYPE; + // Built-ins that refuse subclassing. + let is_final = matches!( + self.name.as_str(), + "bool" + | "NoneType" + | "NotImplementedType" + | "ellipsis" + | "range" + | "slice" + | "memoryview" + | "generator" + | "coroutine" + | "async_generator" + | "function" + | "builtin_function_or_method" + | "method_wrapper" + | "mappingproxy" + ); + if !is_final { + bits |= BASETYPE; + } + if matches!( + self.name.as_str(), + "list" | "dict" | "set" | "frozenset" | "tuple" | "type" + ) || self.flags.is_exception + { + bits |= HAVE_GC; + } + } else { + bits |= HEAPTYPE | BASETYPE | HAVE_GC | MANAGED_WEAKREF; + if self.has_managed_dict() { + bits |= MANAGED_DICT; + if !self.has_var_sized_base() { + bits |= INLINE_VALUES; + } + } + } + match self + .dict + .borrow() + .get(&DictKey(Object::from_static("__abstractmethods__"))) + { + Some(Object::Set(s)) if !s.borrow().is_empty() => bits |= IS_ABSTRACT, + Some(Object::FrozenSet(s)) if !s.is_empty() => bits |= IS_ABSTRACT, + _ => {} + } + for t in self.mro.borrow().iter() { + if t.flags.is_builtin { + match t.name.as_str() { + "int" => bits |= LONG_SUBCLASS, + "list" => bits |= LIST_SUBCLASS, + "tuple" => bits |= TUPLE_SUBCLASS, + "bytes" => bits |= BYTES_SUBCLASS, + "str" => bits |= UNICODE_SUBCLASS, + "dict" => bits |= DICT_SUBCLASS, + "type" => bits |= TYPE_SUBCLASS, + _ => {} + } + } + } + if self.flags.is_exception { + bits |= BASE_EXC_SUBCLASS; + } + bits + } + /// Reset the cached `__getattribute__` classification for this type and /// every (transitive) subclass. Called when `__getattribute__` is /// assigned to or deleted from a type's dict, since that can change the @@ -271,6 +380,13 @@ pub struct PyInstance { /// by the numeric / comparison / hashing / conversion fast paths /// so e.g. `class C(int)` instances behave like real ints. pub native: Option, + /// Mirrors CPython 3.13's "inline values" state observable through + /// `_testinternalcapi.has_inline_values`: starts `true` and is + /// permanently cleared when the instance's `__dict__` is deleted or + /// replaced wholesale (`del obj.__dict__` / `obj.__dict__ = d`). + /// The capacity-overflow half of the state (too many attributes) + /// is computed at query time from the dict size. + pub inline_values: Cell, } impl PyInstance { @@ -279,6 +395,7 @@ impl PyInstance { class, dict: Rc::new(RefCell::new(DictData::new())), native: None, + inline_values: Cell::new(true), } } @@ -289,6 +406,7 @@ impl PyInstance { class, dict: Rc::new(RefCell::new(DictData::new())), native: Some(native), + inline_values: Cell::new(true), } } } diff --git a/crates/weavepy-vm/src/weakref_registry.rs b/crates/weavepy-vm/src/weakref_registry.rs index 782839a..029018f 100644 --- a/crates/weavepy-vm/src/weakref_registry.rs +++ b/crates/weavepy-vm/src/weakref_registry.rs @@ -324,6 +324,7 @@ pub fn id_of(obj: &Object) -> ObjectId { Object::Generator(g) => Rc::as_ptr(g) as usize as u64, Object::Coroutine(g) => Rc::as_ptr(g) as usize as u64, Object::AsyncGenerator(g) => Rc::as_ptr(g) as usize as u64, + Object::AsyncGenAwait(a) => Rc::as_ptr(a) as usize as u64, Object::Iter(i) => Rc::as_ptr(i) as usize as u64, Object::Range(r) => Rc::as_ptr(r) as usize as u64, Object::Cell(c) => Rc::as_ptr(c) as usize as u64, From 245cf3943067c856e0b396b4e19a46b14369358d Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:06:49 -0700 Subject: [PATCH 7/9] feat: advance CPython Lib/test conformance wave 2 --- crates/weavepy-capi/src/abstract_.rs | 8 +- crates/weavepy-capi/src/argparse.rs | 2 +- crates/weavepy-capi/src/datetime_api.rs | 4 +- crates/weavepy-capi/src/errors.rs | 8 +- crates/weavepy-capi/src/genericalloc.rs | 2 +- crates/weavepy-capi/src/module.rs | 2 +- crates/weavepy-capi/src/types.rs | 2 +- crates/weavepy-compiler/src/lib.rs | 148 +- crates/weavepy-parser/src/parser.rs | 9 + crates/weavepy-vm/src/builtin_types.rs | 571 ++++- crates/weavepy-vm/src/builtins.rs | 367 ++- crates/weavepy-vm/src/error.rs | 10 +- crates/weavepy-vm/src/gc_trace.rs | 69 +- crates/weavepy-vm/src/lib.rs | 2194 ++++++++++++++--- crates/weavepy-vm/src/object.rs | 133 +- crates/weavepy-vm/src/specialize.rs | 13 +- crates/weavepy-vm/src/stdlib/abc_mod.rs | 4 +- crates/weavepy-vm/src/stdlib/ast_mod.rs | 4 +- crates/weavepy-vm/src/stdlib/marshal_mod.rs | 7 +- crates/weavepy-vm/src/stdlib/mod.rs | 11 +- .../src/stdlib/python/_testlimitedcapi.py | 8 + .../weavepy-vm/src/stdlib/python/asyncio.py | 8 + .../src/stdlib/python/collections.py | 14 +- .../weavepy-vm/src/stdlib/python/inspect.py | 89 +- .../weavepy-vm/src/stdlib/python/linecache.py | 16 + crates/weavepy-vm/src/stdlib/python/pickle.py | 16 +- .../src/stdlib/python/random_mod.py | 1070 ++++++++ .../weavepy-vm/src/stdlib/python/traceback.py | 40 + .../weavepy-vm/src/stdlib/python/types_mod.py | 85 +- crates/weavepy-vm/src/stdlib/random.rs | 407 --- crates/weavepy-vm/src/stdlib/random_core.rs | 449 +++- crates/weavepy-vm/src/stdlib/socket_mod.rs | 2 +- crates/weavepy-vm/src/stdlib/sys.rs | 29 +- .../src/stdlib/testinternalcapi_mod.rs | 4 +- crates/weavepy-vm/src/stdlib/thread_real.rs | 10 +- crates/weavepy-vm/src/stdlib/weakref_real.rs | 100 +- crates/weavepy-vm/src/types.rs | 42 +- crates/weavepy-vm/src/vm_singletons.rs | 34 +- 38 files changed, 4946 insertions(+), 1045 deletions(-) create mode 100644 crates/weavepy-vm/src/stdlib/python/random_mod.py delete mode 100644 crates/weavepy-vm/src/stdlib/random.rs diff --git a/crates/weavepy-capi/src/abstract_.rs b/crates/weavepy-capi/src/abstract_.rs index 105171b..5172724 100644 --- a/crates/weavepy-capi/src/abstract_.rs +++ b/crates/weavepy-capi/src/abstract_.rs @@ -159,7 +159,7 @@ fn attr_lookup(o: &Object, key: &str) -> Option { // Walk the MRO and invoke descriptor protocol if the // resolved attribute is a property, classmethod, or // staticmethod. Mirror the VM's `LOAD_ATTR` dispatcher. - let raw = inst.class.lookup(key)?; + let raw = inst.cls().lookup(key)?; match &raw { Object::Property(p) => { let getter = p.fget.clone(); @@ -177,7 +177,7 @@ fn attr_lookup(o: &Object, key: &str) -> Option { } Object::StaticMethod(inner) => Some((**inner).clone()), Object::ClassMethod(inner) => { - let class = Object::Type(inst.class.clone()); + let class = Object::Type(inst.cls()); Some(Object::BoundMethod(weavepy_vm::sync::Rc::new( weavepy_vm::object::BoundMethod { receiver: class, @@ -495,7 +495,7 @@ fn install_runtime_error(err: RuntimeError) { match err { RuntimeError::PyException(pe) => { let cls = match &pe.instance { - Object::Instance(inst) => Some(inst.class.clone()), + Object::Instance(inst) => Some(inst.cls()), _ => None, }; crate::errors::set_pending(cls, Object::from_str(pe.message())); @@ -644,7 +644,7 @@ pub unsafe extern "C" fn PyObject_IsInstance(o: *mut PyObject, cls: *mut PyObjec _ => return 0, }; let actual = match &ob { - Object::Instance(inst) => Some(inst.class.clone()), + Object::Instance(inst) => Some(inst.cls()), Object::Type(_) => Some(weavepy_vm::builtin_types::builtin_types().type_.clone()), _ => weavepy_vm::builtin_types::builtin_types() .by_name(type_name(&ob)) diff --git a/crates/weavepy-capi/src/argparse.rs b/crates/weavepy-capi/src/argparse.rs index 3d08cc8..8351053 100644 --- a/crates/weavepy-capi/src/argparse.rs +++ b/crates/weavepy-capi/src/argparse.rs @@ -497,7 +497,7 @@ pub unsafe extern "C" fn _WeavePy_Format_Set(ty: *mut PyObject, msg: *const c_ch } else { match unsafe { crate::object::clone_object(ty) } { Object::Type(t) => Some(t), - Object::Instance(inst) => Some(inst.class.clone()), + Object::Instance(inst) => Some(inst.cls()), _ => None, } }; diff --git a/crates/weavepy-capi/src/datetime_api.rs b/crates/weavepy-capi/src/datetime_api.rs index 4b99e21..8fe832e 100644 --- a/crates/weavepy-capi/src/datetime_api.rs +++ b/crates/weavepy-capi/src/datetime_api.rs @@ -645,7 +645,7 @@ fn is_class_named(o: *mut PyObject, name: &str) -> c_int { } match unsafe { crate::object::clone_object(o) } { Object::Instance(inst) => { - for cls in inst.class.mro.borrow().iter() { + for cls in inst.cls().mro.borrow().iter() { if cls.name == name { return 1; } @@ -662,7 +662,7 @@ fn is_class_named_exact(o: *mut PyObject, name: &str) -> c_int { } match unsafe { crate::object::clone_object(o) } { Object::Instance(inst) => { - if inst.class.name == name { + if inst.cls().name == name { 1 } else { 0 diff --git a/crates/weavepy-capi/src/errors.rs b/crates/weavepy-capi/src/errors.rs index b2c457a..acaf971 100644 --- a/crates/weavepy-capi/src/errors.rs +++ b/crates/weavepy-capi/src/errors.rs @@ -97,7 +97,7 @@ fn message_for(o: &Object) -> String { } } } - format!("<{}>", inst.class.name) + format!("<{}>", inst.cls().name) } Object::None => String::new(), _ => format!("{o:?}"), @@ -122,7 +122,7 @@ pub fn set_pending_from_runtime(err: RuntimeError) { match err { RuntimeError::PyException(pe) => { let cls = match &pe.instance { - Object::Instance(inst) => Some(inst.class.clone()), + Object::Instance(inst) => Some(inst.cls()), _ => None, }; set_pending(cls, Object::from_str(pe.message())); @@ -554,7 +554,7 @@ pub unsafe extern "C" fn PyErr_GivenExceptionMatches( } let given_ty = match unsafe { crate::object::clone_object(given) } { Object::Type(t) => t, - Object::Instance(inst) => inst.class.clone(), + Object::Instance(inst) => inst.cls(), _ => return 0, }; let exc_obj = unsafe { crate::object::clone_object(exc) }; @@ -684,7 +684,7 @@ fn type_object_for(p: *mut PyObject) -> Option> { } match unsafe { crate::object::clone_object(p) } { Object::Type(t) => Some(t), - Object::Instance(inst) => Some(inst.class.clone()), + Object::Instance(inst) => Some(inst.cls()), _ => None, } } diff --git a/crates/weavepy-capi/src/genericalloc.rs b/crates/weavepy-capi/src/genericalloc.rs index 134ede0..8ff510c 100644 --- a/crates/weavepy-capi/src/genericalloc.rs +++ b/crates/weavepy-capi/src/genericalloc.rs @@ -253,7 +253,7 @@ pub unsafe extern "C" fn PyObject_HashNotImplemented(o: *mut PyObject) -> isize } let obj = unsafe { crate::object::clone_object(o) }; let name = match &obj { - Object::Instance(inst) => inst.class.name.clone(), + Object::Instance(inst) => inst.cls().name.clone(), _ => "object".to_owned(), }; crate::errors::set_type_error(format!("unhashable type: '{name}'")); diff --git a/crates/weavepy-capi/src/module.rs b/crates/weavepy-capi/src/module.rs index cde16ca..ca12464 100644 --- a/crates/weavepy-capi/src/module.rs +++ b/crates/weavepy-capi/src/module.rs @@ -631,7 +631,7 @@ fn install_runtime_error(err: RuntimeError) { match err { RuntimeError::PyException(pe) => { let cls = match &pe.instance { - Object::Instance(inst) => Some(inst.class.clone()), + Object::Instance(inst) => Some(inst.cls()), _ => None, }; crate::errors::set_pending(cls, Object::from_str(pe.message())); diff --git a/crates/weavepy-capi/src/types.rs b/crates/weavepy-capi/src/types.rs index 4742aaf..93e57a4 100644 --- a/crates/weavepy-capi/src/types.rs +++ b/crates/weavepy-capi/src/types.rs @@ -298,7 +298,7 @@ pub fn type_for_object(o: &Object) -> *mut PyTypeObject { O::Slice(_) => PySlice_Type.as_ptr(), O::Type(t) => find_type_ptr(t).unwrap_or_else(|| PyType_Type.as_ptr()), O::Instance(inst) => { - find_type_ptr(&inst.class).unwrap_or_else(|| PyBaseObject_Type.as_ptr()) + find_type_ptr(&inst.cls()).unwrap_or_else(|| PyBaseObject_Type.as_ptr()) } _ => PyBaseObject_Type.as_ptr(), } diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index 2fb4e2c..92f7eee 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -46,6 +46,10 @@ pub use cpython_code::{CpythonCode, Position}; pub enum CompileError { #[error("`{0}` is not a valid assignment target")] BadAssignmentTarget(String), + /// A syntax error whose message must match CPython verbatim + /// (doctests assert on these strings). + #[error("{0}")] + SyntaxExact(String), #[error("`break` outside loop")] BreakOutsideLoop, #[error("`continue` outside loop")] @@ -128,6 +132,12 @@ pub struct CodeObject { /// that *also* contains `yield`. Calling such a function returns /// an `Object::AsyncGenerator`. pub is_async_generator: bool, + /// `True` when a generator code object was marked with + /// `types.coroutine` (CPython's `CO_ITERABLE_COROUTINE`). Such a + /// generator is accepted by `await` and may `yield from` a + /// coroutine. Never set by the compiler — only by the runtime + /// marking helper and marshal round-trips. + pub is_iterable_coroutine: bool, } /// A per-instruction source-column span (PEP-657). `col`/`end_col` are @@ -936,8 +946,22 @@ impl Compiler { self.patch_jump(skip, end); } StmtKind::Assign { targets, value } => { - self.compile_expr(value)?; let n = targets.len(); + for t in targets.iter() { + if matches!(t.kind, ExprKind::Yield(_) | ExprKind::YieldFrom(_)) { + // CPython distinguishes a bare `yield` in a chained + // assignment (`x = yield = y`) from a parenthesised + // sole target (`(yield x) = y`). + return Err(CompileError::SyntaxExact(if n > 1 { + "assignment to yield expression not possible".to_owned() + } else { + "cannot assign to yield expression here. Maybe you meant '==' \ + instead of '='?" + .to_owned() + })); + } + } + self.compile_expr(value)?; for (i, t) in targets.iter().enumerate() { if i + 1 < n { self.emit(OpCode::CopyTop, 0); @@ -946,6 +970,12 @@ impl Compiler { } } StmtKind::AugAssign { target, op, value } => { + if matches!(target.kind, ExprKind::Yield(_) | ExprKind::YieldFrom(_)) { + return Err(CompileError::SyntaxExact( + "'yield expression' is an illegal expression for augmented assignment" + .to_owned(), + )); + } self.compile_load_target(target)?; self.compile_expr(value)?; self.emit( @@ -2153,7 +2183,7 @@ impl Compiler { self.emit(OpCode::LoadConst, none_idx); self.emit(OpCode::LoadConst, none_idx); self.emit(OpCode::Call, 3); - self.compile_await_dance(2); + self.compile_await_dance(3); self.emit(OpCode::PopTop, 0); Ok(()) } @@ -2197,11 +2227,17 @@ impl Compiler { for s in body { self.compile_stmt(s)?; } - // Else clause runs only on normal body completion. + let body_end = self.next_offset(); + // Else clause runs only on normal body completion. It sits + // *outside* the handled range: an exception raised in `else` + // does not reach this statement's own `except` clauses (it + // still passes through `finally` via the cleanup entries + // registered below). + let orelse_start = self.next_offset(); for s in orelse { self.compile_stmt(s)?; } - let body_end = self.next_offset(); + let orelse_end = self.next_offset(); // Normal-exit finally + jump to end. Falls through to the // finally body, then skips past the exception handlers. @@ -2346,6 +2382,13 @@ impl Compiler { // Stack on entry: [exc] (pushed by dispatch loop). let mut next_handler_sites: Vec = Vec::new(); let mut handler_exit_jumps: Vec = Vec::new(); + // With a `finally`, an exception raised *inside* an except + // clause (match check, bind, or body — e.g. a bare + // `raise`) must still run the finally before propagating. + // We record each clause's covered range (excluding the + // inline finally copies) and point them at a shared + // cleanup block emitted after the re-raise path. + let mut cleanup_ranges: Vec<(u32, u32)> = Vec::new(); // Each except clause's body lives between the body and the // catch-all `RERAISE` at the bottom. If a clause's `type_` // doesn't match we fall through to the next clause via the @@ -2360,6 +2403,7 @@ impl Compiler { self.patch_jump(site, cur); } } + let clause_start = self.next_offset(); match &h.type_ { Some(t) => { // Stack: [exc] → [exc, type] → [exc, bool] @@ -2388,6 +2432,9 @@ impl Compiler { self.compile_stmt(s)?; } self.emit(OpCode::PopExcept, 0); + if has_finally { + cleanup_ranges.push((clause_start, self.next_offset())); + } // Run finally on the matched path. let saved = if pushed_finally { self.finally_stack.pop() @@ -2421,6 +2468,38 @@ impl Compiler { self.finally_stack.push(f); } self.emit(OpCode::Reraise, 0); + // Shared finally-cleanup block for exceptions escaping an + // except clause or the `else` body. Reached only through + // the exception-table entries registered below; normal + // flow jumps past it (handler exits patch to `end`). + if has_finally { + let cleanup_start = self.next_offset(); + let cleanup_push = self.emit(OpCode::PushExcInfo, 0); + let saved = self.finally_stack.pop(); + for s in finalbody { + self.compile_stmt(s)?; + } + if let Some(f) = saved { + self.finally_stack.push(f); + } + self.emit(OpCode::Reraise, 0); + let cleanup_end = self.next_offset(); + self.co.instructions[cleanup_push as usize].arg = cleanup_end; + if orelse_end > orelse_start { + cleanup_ranges.push((orelse_start, orelse_end)); + } + // Appended after any entries pushed while compiling + // nested statements, so the forward "innermost-first" + // scan in the VM still prefers those. + for (s, e) in cleanup_ranges { + self.co.exception_table.push(ExcHandler { + start: s, + end: e, + handler: cleanup_start, + depth: body_depth, + }); + } + } // Patch handler-exit jumps to end. let end = self.next_offset(); for site in handler_exit_jumps { @@ -2498,6 +2577,12 @@ impl Compiler { return self.compile_with(&items[..1], &inner); } let item = &items[0]; + // The `with` statement's own line — the cleanup/`__exit__` + // sequences below are attributed to it (CPython does the same: + // a traceback through `__exit__` shows the `with` line, not the + // last body statement). + let with_line = self.current_line; + let with_span = self.current_span; let cm_name = format!(".with_cm{}", self.with_counter); self.with_counter += 1; let cm_idx = self.var_index_or_add(&cm_name); @@ -2540,6 +2625,10 @@ impl Compiler { // below emits the same call inline. self.finally_stack.pop(); + // Attribute the whole exit path to the `with` line. + self.current_line = with_line; + self.current_span = with_span; + // Normal exit: cm.__exit__(None, None, None). self.emit(OpCode::LoadFast, cm_idx); let exit_name = self.co.intern_name("__exit__"); @@ -2585,10 +2674,12 @@ impl Compiler { self.emit(OpCode::WithExceptStart, 0); // Stack: [__exit__, exc, result] let swallow = self.emit(OpCode::PopJumpIfTrue, 0); - // Falsy: re-raise. Stack: [__exit__, exc] + // Falsy: re-raise. Stack: [__exit__, exc]. CPython uses RERAISE + // here: the original traceback is preserved and no entry is + // recorded for the re-raise site. self.emit(OpCode::Swap, 2); self.emit(OpCode::PopTop, 0); - self.emit(OpCode::RaiseVarargs, 1); + self.emit(OpCode::Reraise, 0); let swallow_target = self.next_offset(); self.patch_jump(swallow, swallow_target); // Swallowed: Stack: [__exit__, exc]. Drop the active handled-exc @@ -3342,6 +3433,13 @@ impl Compiler { if self.kind != CodeKind::Function { return Err(CompileError::YieldOutsideFunction("yield from")); } + // PEP 525: `yield from` is forbidden in `async def` + // (only plain `yield` makes an async generator). + if self.in_async_context() { + return Err(CompileError::SyntaxExact( + "'yield from' inside async function".to_owned(), + )); + } // CPython 3.13 pattern: // // GET_YIELD_FROM_ITER @@ -3382,7 +3480,9 @@ impl Compiler { /// Emit the "drive awaitable to completion" instruction sequence /// CPython 3.13 uses for `await`. Stack on entry: `[awaitable]`; /// stack on exit: `[result]`. `awaitable_arg` is passed to - /// `GET_AWAITABLE` (0 = plain, 1 = aiter, 2 = aenter). + /// `GET_AWAITABLE` and selects the error message: 0 = plain + /// `await`, 1 = `async for`'s `__anext__` result, 2 = `async + /// with`'s `__aenter__` result, 3 = its `__aexit__` result. fn compile_await_dance(&mut self, awaitable_arg: u32) { self.emit(OpCode::GetAwaitable, awaitable_arg); let none_idx = self.co.intern_constant(Constant::None); @@ -3425,6 +3525,11 @@ impl Compiler { let anext_site = self.emit(OpCode::GetAnext, 0); let _ = anext_site; self.compile_await_dance(1); + // The StopAsyncIteration window closes here: only the + // `__anext__` await may end the loop. An exception raised by + // the assignment target or the body — even a + // StopAsyncIteration — propagates (bpo-44895). + let dance_end = self.next_offset(); // Stack: [aiter, value]. Move the value into the target. self.compile_assign(target)?; self.loop_stack.push(LoopFrame { @@ -3438,14 +3543,15 @@ impl Compiler { let back = self.emit(OpCode::JumpBackward, 0); self.patch_jump(back, loop_top); let frame = self.loop_stack.pop().expect("loop frame"); - // Register an exception-table handler covering the loop body - // so `StopAsyncIteration` lands at the cleanup label. The + // Register an exception-table handler covering only the + // `__anext__` await (loop header) so its `StopAsyncIteration` + // lands at the cleanup label; body exceptions propagate. The // aiter stays at stack depth 1 across the whole loop body — // every per-iteration push lives above it. let cleanup_target = self.next_offset(); self.co.exception_table.push(ExcHandler { start: loop_top, - end: back, + end: dance_end, handler: cleanup_target, depth: 1, }); @@ -3470,6 +3576,10 @@ impl Compiler { return Ok(()); } let (head, rest) = items.split_first().expect("nonempty"); + // See `compile_with`: the exit paths are attributed to the + // `async with` statement's own line. + let with_line = self.current_line; + let with_span = self.current_span; self.compile_expr(&head.context_expr)?; // BEFORE_ASYNC_WITH leaves [aexit, awaitable(aenter)]. self.emit(OpCode::BeforeAsyncWith, 0); @@ -3515,6 +3625,10 @@ impl Compiler { // exception-cleanup paths below emit their own `__aexit__` call. self.finally_stack.pop(); + // Attribute the whole exit path to the `async with` line. + self.current_line = with_line; + self.current_span = with_span; + // Normal exit: `await aexit(None, None, None)`. self.emit(OpCode::LoadFast, slot_idx); let none_idx = self.co.intern_constant(Constant::None); @@ -3522,7 +3636,7 @@ impl Compiler { self.emit(OpCode::LoadConst, none_idx); self.emit(OpCode::LoadConst, none_idx); self.emit(OpCode::Call, 3); - self.compile_await_dance(2); + self.compile_await_dance(3); self.emit(OpCode::PopTop, 0); let end_jump = self.emit(OpCode::JumpForward, 0); @@ -3559,13 +3673,14 @@ impl Compiler { // Stack: [aexit, exc] self.emit(OpCode::WithExceptStart, 0); // Stack: [aexit, exc, awaitable] — await the `__aexit__` coroutine. - self.compile_await_dance(2); + self.compile_await_dance(3); // Stack: [aexit, exc, result] let swallow = self.emit(OpCode::PopJumpIfTrue, 0); - // Falsy: re-raise. Stack: [aexit, exc] + // Falsy: re-raise. Stack: [aexit, exc]. RERAISE preserves the + // original traceback (no entry for the re-raise site). self.emit(OpCode::Swap, 2); self.emit(OpCode::PopTop, 0); - self.emit(OpCode::RaiseVarargs, 1); + self.emit(OpCode::Reraise, 0); let swallow_target = self.next_offset(); self.patch_jump(swallow, swallow_target); // Swallowed: Stack: [aexit, exc]. Drop the active handled-exc @@ -3977,6 +4092,9 @@ fn compile_comp_body( let loop_top = inner.next_offset(); inner.emit(OpCode::GetAnext, 0); inner.compile_await_dance(1); + // As in `compile_async_for`: only the `__anext__` await may end + // the loop via StopAsyncIteration (bpo-44895). + let dance_end = inner.next_offset(); inner.compile_assign(&gen.target)?; let mut filter_jumps = Vec::new(); for cond in &gen.ifs { @@ -3994,7 +4112,7 @@ fn compile_comp_body( let cleanup_target = inner.next_offset(); inner.co.exception_table.push(ExcHandler { start: loop_top, - end: back, + end: dance_end, handler: cleanup_target, depth: cleanup_depth, }); diff --git a/crates/weavepy-parser/src/parser.rs b/crates/weavepy-parser/src/parser.rs index 0103d78..18ebec0 100644 --- a/crates/weavepy-parser/src/parser.rs +++ b/crates/weavepy-parser/src/parser.rs @@ -2261,6 +2261,15 @@ impl<'src> Parser<'src> { // matching CPython. if self.at_keyword(Keyword::Await) { let kw = self.bump(); + // CPython grammar: `await_primary: AWAIT primary` — a + // directly chained `await await x` is invalid syntax + // (`await (await x)` is fine: the parens make a primary). + if self.at_keyword(Keyword::Await) { + return Err(ParseError::Unexpected { + span: kw.span, + message: "invalid syntax".to_owned(), + }); + } let operand = self.parse_unary()?; let span = kw.span.merge(operand.span); return Ok(Expr { diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index b96151c..23035b8 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -62,6 +62,9 @@ pub struct BuiltinTypes { pub generator_: Rc, pub coroutine_: Rc, pub async_generator_: Rc, + /// `types.FrameType` / `types.TracebackType`. + pub frame_: Rc, + pub traceback_: Rc, pub module_: Rc, @@ -117,6 +120,8 @@ pub struct BuiltinTypes { pub eof_error: Rc, pub buffer_error: Rc, + /// Raised on access through a dead weak proxy. + pub reference_error: Rc, pub memory_error: Rc, /// PEP 654 / RFC 0018 — exception group hierarchy. pub base_exception_group: Rc, @@ -202,7 +207,13 @@ impl BuiltinTypes { let generator_ = mk("generator", vec![object_.clone()]); let coroutine_ = mk("coroutine", vec![object_.clone()]); let async_generator_ = mk("async_generator", vec![object_.clone()]); + install_gen_name_getsets(&generator_, "generator"); + install_gen_name_getsets(&coroutine_, "coroutine"); + install_gen_name_getsets(&async_generator_, "async generator"); + let frame_ = mk("frame", vec![object_.clone()]); + let traceback_ = mk("traceback", vec![object_.clone()]); let module_ = mk("module", vec![object_.clone()]); + install_module_init(&module_); let base_exception = exc("BaseException", object_.clone()); let exception = exc("Exception", base_exception.clone()); @@ -286,6 +297,7 @@ impl BuiltinTypes { let eof_error = exc("EOFError", exception.clone()); let buffer_error = exc("BufferError", exception.clone()); + let reference_error = exc("ReferenceError", exception.clone()); let memory_error = exc("MemoryError", exception.clone()); // RFC 0018 — Warning hierarchy. @@ -361,6 +373,8 @@ impl BuiltinTypes { generator_, coroutine_, async_generator_, + frame_, + traceback_, module_, base_exception, exception, @@ -409,6 +423,7 @@ impl BuiltinTypes { process_lookup_error, eof_error, buffer_error, + reference_error, memory_error, base_exception_group, exception_group, @@ -519,6 +534,7 @@ impl BuiltinTypes { pair!(process_lookup_error, "ProcessLookupError"), pair!(eof_error, "EOFError"), pair!(buffer_error, "BufferError"), + pair!(reference_error, "ReferenceError"), pair!(memory_error, "MemoryError"), pair!(base_exception_group, "BaseExceptionGroup"), pair!(exception_group, "ExceptionGroup"), @@ -563,6 +579,8 @@ impl BuiltinTypes { "dict_keys" => Some(self.dict_keys_.clone()), "dict_values" => Some(self.dict_values_.clone()), "dict_items" => Some(self.dict_items_.clone()), + "frame" => Some(self.frame_.clone()), + "traceback" => Some(self.traceback_.clone()), "BaseException" => Some(self.base_exception.clone()), "Exception" => Some(self.exception.clone()), "ArithmeticError" => Some(self.arithmetic_error.clone()), @@ -610,6 +628,7 @@ impl BuiltinTypes { "ProcessLookupError" => Some(self.process_lookup_error.clone()), "EOFError" => Some(self.eof_error.clone()), "BufferError" => Some(self.buffer_error.clone()), + "ReferenceError" => Some(self.reference_error.clone()), "MemoryError" => Some(self.memory_error.clone()), "BaseExceptionGroup" => Some(self.base_exception_group.clone()), "ExceptionGroup" => Some(self.exception_group.clone()), @@ -839,6 +858,23 @@ pub(crate) fn object_new(args: &[Object]) -> Result { )) } }; + // CPython `object_new` arity policy (bpo-31506): excess arguments + // are an error unless exactly one of `__new__`/`__init__` is + // overridden (the overriding side owns the signature). + if args.len() > 1 && !cls.flags.is_builtin && native_seed_for_new(&cls, None).is_none() { + if overrides_dunder_new(&cls) { + return Err(crate::error::type_error( + "object.__new__() takes exactly one argument (the type to instantiate)" + .to_owned(), + )); + } + if !overrides_dunder_init(&cls) { + return Err(crate::error::type_error(format!( + "{}() takes no arguments", + cls.name + ))); + } + } // When `cls` derives from a value/container built-in (`int`, `float`, // `str`, `tuple`, `list`, `dict`, …) capture the native payload the // instance wraps so the inherited protocols keep firing through the @@ -852,6 +888,38 @@ pub(crate) fn object_new(args: &[Object]) -> Result { Ok(Object::Instance(Rc::new(PyInstance::new(cls)))) } +/// Does `cls` inherit `__new__` from somewhere other than `object`? +/// The value built-ins (`int`, `str`, …) install their own `__new__` +/// (CPython `int_new` etc.), which counts as an override for the +/// `object_new`/`object_init` arity policy even though WeavePy routes +/// it through the same default allocator. +pub(crate) fn overrides_dunder_new(cls: &Rc) -> bool { + for ty in cls.mro.borrow().iter() { + if ty + .dict + .borrow() + .contains_key(&DictKey(Object::from_static("__new__"))) + { + return ty.name != "object"; + } + } + false +} + +/// Does `cls` (or a non-`object` base) define a *user* `__init__`? +pub(crate) fn overrides_dunder_init(cls: &Rc) -> bool { + for ty in cls.mro.borrow().iter() { + if ty + .dict + .borrow() + .contains_key(&DictKey(Object::from_static("__init__"))) + { + return ty.name != "object"; + } + } + false +} + /// A fresh `Object::StaticMethod(Builtin "__new__")` wrapping [`object_new`]. /// Each call returns a *distinct* object so `int.__new__ is object.__new__` /// is `False` (matching CPython) while the instantiation path still treats it @@ -865,20 +933,149 @@ fn make_default_new() -> Object { })))) } +/// `module.__init__(self, name, doc=None)` — CPython's `module_init`. +/// `types.ModuleType("m")` (runpy, importlib, test doubles) reaches this; +/// it must accept the name/doc arguments rather than fall back to the +/// strict `object.__init__`. +/// Install the `__name__` / `__qualname__` getset descriptors on the +/// generator-family types (CPython's `gen_getsetlist` / +/// `coro_getsetlist` / `async_gen_getsetlist`). Tests read their +/// docstrings out of the type dict (`test_corotype_1`); reads on the +/// type itself still report the type's own name via the metaclass +/// precedence in `load_attr_type`. +fn install_gen_name_getsets(ty: &Rc, kind: &'static str) { + use crate::object::{BuiltinFn, PyProperty}; + fn gen_of(args: &[Object]) -> Result<&crate::sync::Rc, RuntimeError> + { + match args.first() { + Some( + Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g), + ) => Ok(g), + _ => Err(crate::error::type_error( + "descriptor requires a generator-family object", + )), + } + } + fn get_name(args: &[Object]) -> Result { + Ok(Object::from_str(gen_of(args)?.name.borrow().clone())) + } + fn get_qualname(args: &[Object]) -> Result { + Ok(Object::from_str(gen_of(args)?.qualname.borrow().clone())) + } + let docs = [ + ("__name__", get_name as fn(&[Object]) -> Result, format!("name of the {kind}")), + ("__qualname__", get_qualname, format!("qualified name of the {kind}")), + ]; + for (attr, f, doc) in docs { + ty.dict.borrow_mut().insert( + DictKey(Object::from_static(attr)), + Object::Property(Rc::new(PyProperty::new( + Object::Builtin(Rc::new(BuiltinFn { + name: attr, + call: Box::new(f), + call_kw: None, + })), + Object::None, + Object::None, + Object::from_str(doc), + ))), + ); + } +} + +fn install_module_init(module_: &Rc) { + use crate::object::BuiltinFn; + fn module_init(args: &[Object]) -> Result { + let inst = match args.first() { + Some(Object::Instance(i)) => i.clone(), + _ => { + return Err(crate::error::type_error( + "module.__init__() requires a module instance".to_owned(), + )) + } + }; + if args.len() > 3 { + return Err(crate::error::type_error(format!( + "module.__init__() takes at most 2 arguments ({} given)", + args.len() - 1 + ))); + } + let name = match args.get(1) { + Some(Object::Str(s)) => Object::Str(s.clone()), + Some(_) => { + return Err(crate::error::type_error( + "module.__init__() argument 1 must be str".to_owned(), + )) + } + None => { + return Err(crate::error::type_error( + "module.__init__() missing required argument: 'name' (pos 1)".to_owned(), + )) + } + }; + let doc = args.get(2).cloned().unwrap_or(Object::None); + let mut dict = inst.dict.borrow_mut(); + dict.insert(DictKey(Object::from_static("__name__")), name); + dict.insert(DictKey(Object::from_static("__doc__")), doc); + dict.insert(DictKey(Object::from_static("__package__")), Object::None); + dict.insert(DictKey(Object::from_static("__loader__")), Object::None); + dict.insert(DictKey(Object::from_static("__spec__")), Object::None); + Ok(Object::None) + } + module_.dict.borrow_mut().insert( + DictKey(Object::from_static("__init__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__init__", + call: Box::new(module_init), + call_kw: None, + })), + ); +} + /// Install `object.__new__`, `object.__init__`, `object.__setattr__` /// and `object.__delattr__` on the root class. These are the implicit /// base methods every user class inherits. fn install_object_dunders(object_: &Rc) { use crate::object::BuiltinFn; - fn object_init(_args: &[Object]) -> Result { + fn object_init(args: &[Object]) -> Result { + // CPython `object_init` arity policy (bpo-31506): excess + // arguments are an error unless `__new__` is overridden while + // `__init__` is not (then `__new__` owns the signature and the + // default `__init__` stays lenient). + if args.len() > 1 { + if let Some(Object::Instance(inst)) = args.first() { + let cls = &inst.cls(); + // A native payload means a built-in base's constructor + // (`int_new`, `property_init`, …) owns the signature — + // CPython's tp_new/tp_init for those types aren't + // `object_new`/`object_init`, so the strict arity + // policy doesn't apply. + if inst.native.is_none() { + if overrides_dunder_init(cls) { + // An overriding `__init__` delegated here + // (`super().__init__(*args)`) — blame object.__init__. + return Err(crate::error::type_error( + "object.__init__() takes exactly one argument (the instance to initialize)" + .to_owned(), + )); + } + if !overrides_dunder_new(cls) { + return Err(crate::error::type_error(format!( + "{}.__init__() takes exactly one argument (the instance to initialize)", + cls.name + ))); + } + } + } + } // No-op; honours `super().__init__()` chains. Ok(Object::None) } fn object_setattr(args: &[Object]) -> Result { - // `object.__setattr__(self, name, value)` — write directly - // to the instance dict, bypassing any user `__setattr__` - // override on the receiver's class (used by dataclasses' - // frozen __init__ to populate fields). + // `object.__setattr__(self, name, value)` — CPython's + // `PyObject_GenericSetAttr`: descriptors, `__slots__` and + // `__class__` handling, but *no* user-`__setattr__` dispatch + // (this is the default that overrides chain up to). if args.len() != 3 { return Err(crate::error::type_error( "object.__setattr__() takes 3 arguments".to_owned(), @@ -890,9 +1087,16 @@ fn install_object_dunders(object_: &Rc) { }; match &args[0] { Object::Instance(inst) => { - inst.dict - .borrow_mut() - .insert(DictKey(Object::from_str(name)), args[2].clone()); + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still + // live on this thread; the GIL keeps access exclusive. + let interp = unsafe { &mut *ptr }; + interp.generic_setattr_instance(inst, &args[0], &name, args[2].clone())?; + } else { + inst.dict + .borrow_mut() + .insert(DictKey(Object::from_str(name)), args[2].clone()); + } Ok(Object::None) } // `type.__setattr__` semantics for a class receiver — reached @@ -931,6 +1135,13 @@ fn install_object_dunders(object_: &Rc) { }; match &args[0] { Object::Instance(inst) => { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still + // live on this thread; the GIL keeps access exclusive. + let interp = unsafe { &mut *ptr }; + interp.generic_delattr_instance(inst, &args[0], &name)?; + return Ok(Object::None); + } let removed = inst .dict .borrow_mut() @@ -939,7 +1150,7 @@ fn install_object_dunders(object_: &Rc) { if !removed { return Err(crate::error::attribute_error(format!( "'{}' object has no attribute '{}'", - inst.class.name, name + inst.cls().name, name ))); } Ok(Object::None) @@ -1514,6 +1725,15 @@ fn install_exception_str_repr(base_exception: &Rc) { } else { Vec::new() }; + // PEP 380: `StopIteration.value` mirrors args[0] for the + // built-in class and any user subclass (CPython stores it + // in `StopIteration.__init__`). + if is_subclass_by_name(&inst_rc.cls(), "StopIteration") { + inst_rc.dict.borrow_mut().insert( + DictKey(Object::from_static("value")), + rest.first().cloned().unwrap_or(Object::None), + ); + } inst_rc.dict.borrow_mut().insert( DictKey(Object::from_static("args")), Object::new_tuple(rest), @@ -1531,7 +1751,7 @@ fn install_exception_str_repr(base_exception: &Rc) { // is ``"'x'"`` not ``'x'``. We special-case KeyError here // because the runtime constructs them from Rust and we // can't easily install a per-subclass ``__str__``. - let is_key_error = is_subclass_by_name(&inst_rc.class, "KeyError"); + let is_key_error = is_subclass_by_name(&inst_rc.cls(), "KeyError"); let dict = inst_rc.dict.borrow(); if let Some(Object::Tuple(items)) = dict.get(&DictKey(Object::from_static("args"))) { return Ok(match items.as_ref() { @@ -1539,6 +1759,14 @@ fn install_exception_str_repr(base_exception: &Rc) { [single] => { if is_key_error { Object::from_str(single.repr()) + } else if matches!(single, Object::Instance(_)) { + // A nested exception (or other instance) needs + // its own __str__ dispatched: CPython's + // BaseException.__str__ is `str(args[0])`. + Object::from_str( + crate::builtins::str_reentrant(single) + .unwrap_or_else(|| single.to_str()), + ) } else { Object::from_str(single.to_str()) } @@ -1561,7 +1789,7 @@ fn install_exception_str_repr(base_exception: &Rc) { .first() .ok_or_else(|| crate::error::type_error("expected exception instance".to_owned()))?; if let Object::Instance(inst_rc) = inst { - let cls = inst_rc.class.name.clone(); + let cls = inst_rc.cls().name.clone(); let dict = inst_rc.dict.borrow(); let args_repr = if let Some(Object::Tuple(items)) = dict.get(&DictKey(Object::from_static("args"))) @@ -1668,7 +1896,13 @@ pub fn make_exception_with_class(class: Rc, message: impl Into) { Ok(Object::from_static("")) } fn eg_derive(args: &[Object]) -> Result { - // derive(self, excs) -> new EG of the same class + // Default `derive(self, excs)` — CPython's returns a *plain* + // `BaseExceptionGroup(self.message, excs)` (not `type(self)`), + // which `__new__`'s PEP 654 magic lowers to `ExceptionGroup` + // when every leaf is an `Exception`. Subclasses that want to + // survive `split`/`subgroup` must override `derive`. let inst = args .first() .ok_or_else(|| crate::error::type_error("expected exception instance"))?; @@ -1794,14 +2032,14 @@ fn install_exception_group_init(base: &Rc) { .cloned() .unwrap_or(Object::from_static("")); drop(dict); - let cls = inst_rc.class.clone(); + let excs_tuple: Rc<[Object]> = match excs { + Object::Tuple(t) => t, + Object::List(l) => Rc::from(l.borrow().clone().into_boxed_slice()), + _ => Rc::from(Vec::::new().into_boxed_slice()), + }; + let cls = exception_group_class_for(&excs_tuple); let new_inst = make_exception_with_class(cls, ""); if let Object::Instance(ni) = &new_inst { - let excs_tuple = match excs { - Object::Tuple(t) => t, - Object::List(l) => Rc::from(l.borrow().clone().into_boxed_slice()), - _ => Rc::from(Vec::::new().into_boxed_slice()), - }; let mut d = ni.dict.borrow_mut(); d.insert( DictKey(Object::from_static("args")), @@ -1882,15 +2120,85 @@ fn install_exception_group_init(base: &Rc) { ); } +/// PEP 654 class-selection rule for a derived/constructed group: a +/// plain `BaseExceptionGroup` whose leaves are all `Exception`s +/// materialises as `ExceptionGroup`. +fn exception_group_class_for(items: &[Object]) -> Rc { + let bt = builtin_types(); + let all_exceptions = items + .iter() + .all(|e| instance_is_subclass(e, &bt.exception)); + if all_exceptions { + bt.exception_group.clone() + } else { + bt.base_exception_group.clone() + } +} + +/// Enforce PEP 654's construction rules when instantiating exception +/// classes: lower a plain `BaseExceptionGroup` to `ExceptionGroup` +/// when every contained exception is an `Exception`, and refuse to +/// nest a bare `BaseException` inside an `ExceptionGroup` (subclass). +pub fn resolve_exception_group_class( + cls: Rc, + args: &[Object], +) -> Result, RuntimeError> { + let bt = builtin_types(); + if !cls.is_subclass_of(&bt.base_exception_group) { + return Ok(cls); + } + let items: Vec = match args.get(1) { + Some(Object::Tuple(t)) => t.to_vec(), + Some(Object::List(l)) => l.borrow().clone(), + _ => return Ok(cls), + }; + let all_exceptions = items + .iter() + .all(|e| instance_is_subclass(e, &bt.exception)); + if Rc::ptr_eq(&cls, &bt.base_exception_group) { + if all_exceptions { + return Ok(bt.exception_group.clone()); + } + return Ok(cls); + } + if cls.is_subclass_of(&bt.exception_group) && !all_exceptions { + return Err(crate::error::type_error( + "Cannot nest BaseExceptions in an ExceptionGroup", + )); + } + Ok(cls) +} + +/// `True` if `class` overrides `derive` somewhere below the builtin +/// `BaseExceptionGroup` implementation in its MRO. +fn overrides_eg_derive(class: &Rc) -> bool { + let bt = builtin_types(); + for t in class.mro.borrow().iter() { + if Rc::ptr_eq(t, &bt.base_exception_group) { + return false; + } + if t.dict.borrow().contains_key(&DictKey(Object::from_static("derive"))) { + return true; + } + } + false +} + /// Split an exception group instance against a type predicate. Used /// by the VM's `CheckEGMatch` opcode and exposed via /// `BaseExceptionGroup.split(typ)`. /// /// Returns `(matched, rest)` where: /// - `matched` is `None` if no contained exception matches, otherwise -/// a new exception group of the same class containing the matches. +/// a new exception group containing the matches. /// - `rest` is `None` if every contained exception matches, otherwise /// a new group with the non-matching ones. +/// +/// New groups are produced via `derive` semantics: the *default* +/// derive returns a plain group (auto-lowered per PEP 654); a +/// user-overridden `derive` is dispatched through the interpreter. +/// `__cause__`, `__context__`, `__traceback__` and `__notes__` are +/// copied onto the derived parts, mirroring CPython's split. pub fn split_exception_group( group: &Object, type_pred: &Object, @@ -1906,7 +2214,7 @@ pub fn split_exception_group( Some(Object::Tuple(t)) => t.to_vec(), _ => Vec::new(), }; - (inst.class.clone(), msg, excs) + (inst.cls(), msg, excs) } _ => { return Err(crate::error::type_error( @@ -1919,7 +2227,7 @@ pub fn split_exception_group( for exc in excs { // For nested groups, recurse. let is_group = match &exc { - Object::Instance(i) => is_subclass_by_name(&i.class, "BaseExceptionGroup"), + Object::Instance(i) => is_subclass_by_name(&i.cls(), "BaseExceptionGroup"), _ => false, }; if is_group { @@ -1936,24 +2244,46 @@ pub fn split_exception_group( rest.push(exc); } } - let mk = |items: Vec| -> Object { + let derive_override = overrides_eg_derive(&cls); + let mk = |items: Vec| -> Result { if items.is_empty() { - return Object::None; + return Ok(Object::None); } - let new_inst = make_exception_with_class(cls.clone(), ""); - if let Object::Instance(ni) = &new_inst { - let mut d = ni.dict.borrow_mut(); - let items_t = Object::new_tuple(items); - d.insert( - DictKey(Object::from_static("args")), - Object::new_tuple(vec![message.clone(), items_t.clone()]), - ); - d.insert(DictKey(Object::from_static("message")), message.clone()); - d.insert(DictKey(Object::from_static("exceptions")), items_t); + let items_t = Object::new_tuple(items.clone()); + let new_inst = if derive_override { + // Dispatch the subclass's own `derive(self, excs)`. + let derive = cls + .lookup("derive") + .ok_or_else(|| crate::error::type_error("exception group lost its derive"))?; + crate::builtins::reentrant_call(&derive, &[group.clone(), items_t.clone()])? + } else { + let new_cls = exception_group_class_for(&items); + let ni = make_exception_with_class(new_cls, ""); + if let Object::Instance(inst_rc) = &ni { + let mut d = inst_rc.dict.borrow_mut(); + d.insert( + DictKey(Object::from_static("args")), + Object::new_tuple(vec![message.clone(), items_t.clone()]), + ); + d.insert(DictKey(Object::from_static("message")), message.clone()); + d.insert(DictKey(Object::from_static("exceptions")), items_t.clone()); + } + ni + }; + // CPython copies the chaining/traceback metadata from the + // original group onto each derived part. + if let (Object::Instance(src), Object::Instance(dst)) = (group, &new_inst) { + let src_d = src.dict.borrow(); + let mut dst_d = dst.dict.borrow_mut(); + for key in ["__cause__", "__context__", "__traceback__", "__notes__"] { + if let Some(v) = src_d.get(&DictKey(Object::from_static(key))) { + dst_d.insert(DictKey(Object::from_static(key)), v.clone()); + } + } } - new_inst + Ok(new_inst) }; - Ok((mk(matched), mk(rest))) + Ok((mk(matched)?, mk(rest)?)) } fn exception_matches_type(exc: &Object, type_pred: &Object) -> bool { @@ -1998,7 +2328,7 @@ pub fn exception_message(obj: &Object) -> Option { /// `True` when `obj` is an instance whose class derives from `cls`. pub fn instance_is_subclass(obj: &Object, cls: &TypeObject) -> bool { match obj { - Object::Instance(inst) => inst.class.is_subclass_of(cls), + Object::Instance(inst) => inst.cls().is_subclass_of(cls), _ => false, } } @@ -2030,6 +2360,173 @@ fn install_value_type_new(bt: &BuiltinTypes) { .borrow_mut() .insert(DictKey(Object::from_static("__new__")), make_default_new()); } + install_mutable_container_init(bt); +} + +/// The mutable containers own a real `tp_init` in CPython: `dict.__init__` +/// merges a mapping/iterable + kwargs, `list.__init__` clears and extends, +/// `set.__init__` clears and unions. `super().__init__(src)` from a +/// subclass must reach these (not the strict `object.__init__`). +fn install_mutable_container_init(bt: &BuiltinTypes) { + use crate::object::BuiltinFn; + + fn self_payload(args: &[Object]) -> Result { + match args.first() { + Some(o @ (Object::Dict(_) | Object::List(_) | Object::Set(_))) => Ok(o.clone()), + Some(Object::Instance(inst)) => match &inst.native { + Some(n @ (Object::Dict(_) | Object::List(_) | Object::Set(_))) => Ok(n.clone()), + _ => Err(crate::error::type_error( + "descriptor '__init__' requires a container instance".to_owned(), + )), + }, + _ => Err(crate::error::type_error( + "descriptor '__init__' requires a container instance".to_owned(), + )), + } + } + + fn reenter() -> Result<&'static mut crate::Interpreter, RuntimeError> { + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| crate::error::runtime_error("no running interpreter"))?; + // SAFETY: published by an enclosing VM frame still live on this + // thread; the GIL keeps the access exclusive. + Ok(unsafe { &mut *ptr }) + } + + fn dict_init_kw( + args: &[Object], + kwargs: &[(String, Object)], + ) -> Result { + let payload = self_payload(args)?; + let Object::Dict(target) = &payload else { + return Err(crate::error::type_error( + "descriptor '__init__' requires a 'dict' object".to_owned(), + )); + }; + if args.len() > 2 { + return Err(crate::error::type_error(format!( + "dict expected at most 1 argument, got {}", + args.len() - 1 + ))); + } + if let Some(src) = args.get(1) { + let interp = reenter()?; + let globals = interp.builtins_dict(); + let merged: Vec<(DictKey, Object)> = + if let Some(Object::Dict(d)) = interp.try_dict_from_mapping(src, &globals)? { + let view = d.borrow(); + view.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } else { + let items = interp.collect_iterable(src, &globals)?; + let mut out = Vec::with_capacity(items.len()); + for (i, pair) in items.into_iter().enumerate() { + let kv = interp.collect_iterable(&pair, &globals)?; + if kv.len() != 2 { + return Err(crate::error::type_error(format!( + "dictionary update sequence element #{i} has length {}; 2 is required", + kv.len() + ))); + } + out.push((DictKey(kv[0].clone()), kv[1].clone())); + } + out + }; + let mut t = target.borrow_mut(); + for (k, v) in merged { + t.insert(k, v); + } + } + let mut t = target.borrow_mut(); + for (k, v) in kwargs { + t.insert(DictKey(Object::from_str(k.clone())), v.clone()); + } + Ok(Object::None) + } + + fn list_init(args: &[Object]) -> Result { + let payload = self_payload(args)?; + let Object::List(target) = &payload else { + return Err(crate::error::type_error( + "descriptor '__init__' requires a 'list' object".to_owned(), + )); + }; + if args.len() > 2 { + return Err(crate::error::type_error(format!( + "list expected at most 1 argument, got {}", + args.len() - 1 + ))); + } + let items = match args.get(1) { + Some(src) => { + let interp = reenter()?; + let globals = interp.builtins_dict(); + interp.collect_iterable(src, &globals)? + } + None => Vec::new(), + }; + let mut t = target.borrow_mut(); + t.clear(); + t.extend(items); + Ok(Object::None) + } + + fn set_init(args: &[Object]) -> Result { + let payload = self_payload(args)?; + let Object::Set(target) = &payload else { + return Err(crate::error::type_error( + "descriptor '__init__' requires a 'set' object".to_owned(), + )); + }; + if args.len() > 2 { + return Err(crate::error::type_error(format!( + "set expected at most 1 argument, got {}", + args.len() - 1 + ))); + } + let items = match args.get(1) { + Some(src) => { + let interp = reenter()?; + let globals = interp.builtins_dict(); + interp.collect_iterable(src, &globals)? + } + None => Vec::new(), + }; + let mut t = target.borrow_mut(); + t.clear(); + for item in items { + t.insert(DictKey(item)); + } + Ok(Object::None) + } + + fn dict_init(args: &[Object]) -> Result { + dict_init_kw(args, &[]) + } + + bt.dict_.dict.borrow_mut().insert( + DictKey(Object::from_static("__init__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__init__", + call: Box::new(dict_init), + call_kw: Some(Box::new(dict_init_kw)), + })), + ); + bt.list_.dict.borrow_mut().insert( + DictKey(Object::from_static("__init__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__init__", + call: Box::new(list_init), + call_kw: None, + })), + ); + bt.set_.dict.borrow_mut().insert( + DictKey(Object::from_static("__init__")), + Object::Builtin(Rc::new(BuiltinFn { + name: "__init__", + call: Box::new(set_init), + call_kw: None, + })), + ); } /// RFC 0019 — install class methods on the numeric / bytes types. diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index f0342dd..2218004 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -186,6 +186,10 @@ pub fn default_builtins() -> DictData { reg!("iter", b_iter); reg!("aiter", b_aiter); reg!("anext", b_anext); + reg!( + "_weavepy_mark_iterable_coroutine", + b_mark_iterable_coroutine + ); reg!("divmod", b_divmod); reg!("round", b_round); reg!("format", b_format); @@ -1248,6 +1252,18 @@ fn slot_repr(args: &[Object]) -> Result { let o = args .first() .ok_or_else(|| type_error("__repr__() takes exactly one argument (0 given)"))?; + // CPython guards `PyObject_Repr` with `Py_EnterRecursiveCall`; the + // native repr can re-enter the VM (user `__repr__`), and rebinding + // `__repr__ = __str__` (test_descr.test_repr_as_str) creates a + // native-only cycle that must raise instead of overflowing. + let _guard = match crate::recursion::enter() { + crate::recursion::Enter::Ok(g) => g, + crate::recursion::Enter::Overflow => { + return Err(crate::error::recursion_error( + "maximum recursion depth exceeded while getting the repr of an object", + )) + } + }; let native = o.native_value(); Ok(Object::from_str(native.as_ref().unwrap_or(o).repr())) } @@ -1259,8 +1275,44 @@ fn slot_str(args: &[Object]) -> Result { let o = args .first() .ok_or_else(|| type_error("__str__() takes exactly one argument (0 given)"))?; + // See `slot_repr`: participate in the recursion limit so + // `__repr__`/`__str__` rebinding cycles raise `RecursionError`. + let _guard = match crate::recursion::enter() { + crate::recursion::Enter::Ok(g) => g, + crate::recursion::Enter::Overflow => { + return Err(crate::error::recursion_error( + "maximum recursion depth exceeded while getting the str of an object", + )) + } + }; let native = o.native_value(); - Ok(Object::from_str(native.as_ref().unwrap_or(o).to_str())) + let target = native.as_ref().unwrap_or(o); + // CPython `object.__str__` is `PyObject_Repr(self)`: a user-defined + // `__repr__` is dispatched through the VM so its exceptions (and + // RecursionError from `__repr__ = __str__` cycles) *propagate*, + // rather than being swallowed by the native fallback rendering. + if let Object::Instance(inst) = target { + let key = crate::object::DictKey(Object::from_static("__repr__")); + let has_user_repr = inst + .cls() + .mro + .borrow() + .iter() + .any(|t| t.dict.borrow().contains_key(&key)); + if has_user_repr { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still live + // on this thread; the GIL keeps it exclusive. + let interp = unsafe { &mut *ptr }; + if let Some(method) = crate::instance_method(target, "__repr__") { + let globals = interp.builtins_dict(); + let r = interp.call_object_with_globals(&method, &[], &[], &globals)?; + return Ok(Object::from_str(r.to_str())); + } + } + } + } + Ok(Object::from_str(target.to_str())) } /// `object.__format__(self, spec)` / `str.__format__(self, spec)` — format @@ -1295,6 +1347,12 @@ fn slot_call(args: &[Object], kwargs: &[(String, Object)]) -> Result Option { { return Some(Object::Builtin(Rc::new(method_kw("__call__", slot_call)))); } - let f: fn(&[Object]) -> Result = match name { - "__repr__" => slot_repr, - "__format__" => slot_format, - // Every built-in value type has its own `tp_str` in CPython - // (`int.__str__ is not object.__str__` — enum's ReprEnum wiring - // tests that identity), and `slot_str` already stringifies the - // receiver's native payload per type. - "__str__" => slot_str, - _ => return None, + let (static_name, f): (&'static str, fn(&[Object]) -> Result) = + match name { + "__repr__" => ("__repr__", slot_repr), + "__format__" => ("__format__", slot_format), + // Every built-in value type has its own `tp_str` in CPython + // (`int.__str__ is not object.__str__` — enum's ReprEnum wiring + // tests that identity), and `slot_str` already stringifies the + // receiver's native payload per type. + "__str__" => ("__str__", slot_str), + // `object`'s default rich comparisons: `==`/`!=` compare by + // identity (value identity for primitives) and return + // `NotImplemented` otherwise; the orderings are always + // `NotImplemented` at the `object` level. + "__eq__" => ("__eq__", slot_obj_eq), + "__ne__" => ("__ne__", slot_obj_ne), + "__lt__" => ("__lt__", slot_obj_ordering), + "__le__" => ("__le__", slot_obj_ordering), + "__gt__" => ("__gt__", slot_obj_ordering), + "__ge__" => ("__ge__", slot_obj_ordering), + "__dir__" => ("__dir__", b_dir), + "__sizeof__" => ("__sizeof__", slot_sizeof), + "__getstate__" => ("__getstate__", slot_getstate), + _ => return None, + }; + Some(Object::Builtin(Rc::new(method(static_name, f)))) +} + +/// `object.__eq__(self, other)` — identity (payload equality for the +/// primitive value types), `NotImplemented` otherwise. +fn slot_obj_eq(args: &[Object]) -> Result { + let (a, b) = match args { + [a, b] => (a, b), + _ => return Err(type_error("expected 2 arguments")), }; - let static_name = match name { - "__repr__" => "__repr__", - "__str__" => "__str__", - "__format__" => "__format__", - _ => return None, + if object_identity(a) == object_identity(b) { + Ok(Object::Bool(true)) + } else { + Ok(crate::vm_singletons::not_implemented()) + } +} + +/// `object.__ne__(self, other)` — the negation of `__eq__`, staying +/// `NotImplemented` when equality is undecided. +fn slot_obj_ne(args: &[Object]) -> Result { + let (a, b) = match args { + [a, b] => (a, b), + _ => return Err(type_error("expected 2 arguments")), }; - Some(Object::Builtin(Rc::new(method(static_name, f)))) + if object_identity(a) == object_identity(b) { + Ok(Object::Bool(false)) + } else { + Ok(crate::vm_singletons::not_implemented()) + } +} + +/// `object.__lt__` / `__le__` / `__gt__` / `__ge__` — `object` defines +/// no ordering: always `NotImplemented`. +fn slot_obj_ordering(_args: &[Object]) -> Result { + Ok(crate::vm_singletons::not_implemented()) +} + +/// `object.__sizeof__(self)` — a coarse byte size. WeavePy objects +/// don't share CPython's memory layout; report the CPython-typical +/// fixed header size so the protocol surface exists and returns a +/// plausible positive int. +fn slot_sizeof(args: &[Object]) -> Result { + let o = one(args, "__sizeof__")?; + let size: i64 = match o { + Object::Instance(inst) => 16 + 8 * inst.dict.borrow().len() as i64, + Object::Str(s) => 49 + s.len() as i64, + Object::Bytes(b) => 33 + b.len() as i64, + Object::List(items) => 56 + 8 * items.borrow().len() as i64, + Object::Tuple(items) => 40 + 8 * items.len() as i64, + Object::Dict(d) => 64 + 24 * d.borrow().len() as i64, + _ => 16, + }; + Ok(Object::Int(size)) +} + +/// `object.__getstate__(self)` — PEP 307 default pickling state: the +/// instance `__dict__` when non-empty, else `None`. +fn slot_getstate(args: &[Object]) -> Result { + let o = one(args, "__getstate__")?; + if let Object::Instance(inst) = o { + if !inst.dict.borrow().is_empty() { + return Ok(Object::Dict(inst.dict.clone())); + } + } + Ok(Object::None) } // ---------- free builtins ---------- @@ -1694,7 +1824,19 @@ fn property_deleter(args: &[Object]) -> Result { /// builtin context. Shared by the explicit descriptor-protocol slots /// (`property.__get__` / `__set__` / `__delete__`), whose accessors are /// ordinary Python functions. -fn reentrant_call(callable: &Object, args: &[Object]) -> Result { +/// `str(obj)` through the running interpreter (so user `__str__` / +/// nested-exception rendering dispatches). `None` when no interpreter +/// is live on this thread. +pub(crate) fn str_reentrant(obj: &Object) -> Option { + let ptr = crate::vm_singletons::current_interpreter_ptr()?; + // SAFETY: the pointer was published by an enclosing VM frame still + // live on this thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + interp.stringify_public(obj, &globals).ok() +} + +pub(crate) fn reentrant_call(callable: &Object, args: &[Object]) -> Result { let ptr = crate::vm_singletons::current_interpreter_ptr() .ok_or_else(|| crate::error::runtime_error("no running interpreter"))?; // SAFETY: the pointer was published by an enclosing VM frame still @@ -1765,9 +1907,9 @@ fn b_getattr(args: &[Object]) -> Result { if args.len() < 2 { return Err(type_error("getattr() requires at least 2 arguments")); } - let name = match &args[1] { - Object::Str(s) => s.to_string(), - _ => return Err(type_error("attribute name must be string")), + let name = match crate::attr_name_of(&args[1]) { + Some(n) => n, + None => return Err(type_error("attribute name must be string")), }; let default = args.get(2).cloned(); match attr_get(&args[0], &name) { @@ -1787,9 +1929,9 @@ fn b_setattr(args: &[Object]) -> Result { if args.len() != 3 { return Err(type_error("setattr() takes exactly 3 arguments")); } - let name = match &args[1] { - Object::Str(s) => s.to_string(), - _ => return Err(type_error("attribute name must be string")), + let name = match crate::attr_name_of(&args[1]) { + Some(n) => n, + None => return Err(type_error("attribute name must be string")), }; attr_set(&args[0], &name, args[2].clone())?; Ok(Object::None) @@ -1799,9 +1941,9 @@ fn b_delattr(args: &[Object]) -> Result { if args.len() != 2 { return Err(type_error("delattr() takes exactly 2 arguments")); } - let name = match &args[1] { - Object::Str(s) => s.to_string(), - _ => return Err(type_error("attribute name must be string")), + let name = match crate::attr_name_of(&args[1]) { + Some(n) => n, + None => return Err(type_error("attribute name must be string")), }; attr_delete(&args[0], &name)?; Ok(Object::None) @@ -1811,9 +1953,9 @@ fn b_hasattr(args: &[Object]) -> Result { if args.len() != 2 { return Err(type_error("hasattr() takes exactly 2 arguments")); } - let name = match &args[1] { - Object::Str(s) => s.to_string(), - _ => return Err(type_error("attribute name must be string")), + let name = match crate::attr_name_of(&args[1]) { + Some(n) => n, + None => return Err(type_error("attribute name must be string")), }; Ok(Object::Bool(attr_get(&args[0], &name).is_some())) } @@ -1869,7 +2011,7 @@ fn b_callable(args: &[Object]) -> Result { } // Instances are callable when their class exposes `__call__`. if let Object::Instance(inst) = v { - return Ok(Object::Bool(inst.class.lookup("__call__").is_some())); + return Ok(Object::Bool(inst.cls().lookup("__call__").is_some())); } Ok(Object::Bool(false)) } @@ -1909,7 +2051,7 @@ fn bind_descriptor(value: &Object, receiver: &Object) -> Object { Object::StaticMethod(inner) => (**inner).clone(), Object::ClassMethod(inner) => { let cls = match receiver { - Object::Instance(inst) => Object::Type(inst.class.clone()), + Object::Instance(inst) => Object::Type(inst.cls()), Object::Type(_) => receiver.clone(), _ => receiver.clone(), }; @@ -1933,7 +2075,7 @@ fn attr_get(obj: &Object, name: &str) -> Option { { return Some(v); } - if let Some(v) = inst.class.lookup(name) { + if let Some(v) = inst.cls().lookup(name) { // Bind functions to the receiver so `getattr(inst, 'm')()` // works the same as `inst.m()`. Other descriptors are // left to the VM's full `descriptor_get` path. @@ -1941,7 +2083,7 @@ fn attr_get(obj: &Object, name: &str) -> Option { } match name { "__dict__" => Some(Object::Dict(inst.dict.clone())), - "__class__" => Some(Object::Type(inst.class.clone())), + "__class__" => Some(Object::Type(inst.cls())), _ => None, } } @@ -2371,7 +2513,10 @@ pub(crate) fn code_flags(c: &weavepy_compiler::CodeObject) -> u32 { f |= CO_GENERATOR; } if c.is_coroutine { - f |= CO_COROUTINE | CO_ITERABLE_COROUTINE; + f |= CO_COROUTINE; + } + if c.is_iterable_coroutine { + f |= CO_ITERABLE_COROUTINE; } if c.is_async_generator { f |= CO_ASYNC_GENERATOR; @@ -4289,7 +4434,7 @@ pub fn make_super(class: Rc, receiver: Object) -> Obje // always `C`'s MRO) breaks either diamond `__init_subclass__` or // `super().__init__()` inside a metaclass, respectively. let receiver_class = match &receiver { - Object::Instance(inst) => inst.class.clone(), + Object::Instance(inst) => inst.cls(), Object::Type(t) if t.is_subclass_of(&class) => t.clone(), Object::Type(t) => t.metaclass_or_type(), _ => class.clone(), @@ -4314,7 +4459,7 @@ pub fn make_super(class: Rc, receiver: Object) -> Obje getattribute_kind: crate::sync::Cell::new(0), }); let inst = crate::types::PyInstance { - class: proxy, + class: RefCell::new(proxy), dict: Rc::new(RefCell::new({ let mut d = DictData::new(); d.insert(DictKey(Object::from_static("__self__")), receiver); @@ -4345,18 +4490,42 @@ fn b_issubclass(args: &[Object]) -> Result { _ => return Err(type_error("issubclass() arg 1 must be a class")), }; let info = &args[1]; - Ok(Object::Bool(class_matches_classinfo(&cls, info)?)) + Ok(Object::Bool(class_matches_classinfo_named( + &cls, + info, + "issubclass", + )?)) } /// Walk `cls`'s MRO against a single type or tuple of types. pub fn class_matches_classinfo( cls: &crate::types::TypeObject, info: &Object, +) -> Result { + class_matches_classinfo_named(cls, info, "isinstance") +} + +/// As [`class_matches_classinfo`], with the caller's function name +/// (`isinstance`/`issubclass`) threaded through for CPython-exact +/// error messages. +pub fn class_matches_classinfo_named( + cls: &crate::types::TypeObject, + info: &Object, + func: &str, ) -> Result { // PEP 604 union (`int | str`) — succeed if any union arm matches. + // A *parameterized* arm (`list[int] | int`) is not runtime- + // checkable: CPython's `union_instancecheck` raises TypeError. if let Some(args) = crate::is_pep604_union(info) { for arg in &args { - if class_matches_classinfo(cls, arg)? { + if generic_alias_origin(arg).is_some() { + return Err(type_error(format!( + "{func}() argument 2 cannot contain a parameterized generic" + ))); + } + } + for arg in &args { + if class_matches_classinfo_named(cls, arg, func)? { return Ok(true); } } @@ -4421,7 +4590,7 @@ fn generic_alias_origin(info: &Object) -> Option { pub fn class_of(obj: &Object) -> crate::sync::Rc { let bt = builtin_types(); match obj { - Object::Instance(inst) => inst.class.clone(), + Object::Instance(inst) => inst.cls(), Object::None => bt.none_type.clone(), Object::Bool(_) => bt.bool_.clone(), Object::Int(_) => bt.int_.clone(), @@ -4485,7 +4654,8 @@ pub fn class_of(obj: &Object) -> crate::sync::Rc { Object::Code(_) | Object::Cell(_) | Object::SlotDescriptor(_) | Object::File(_) => { bt.object_.clone() } - Object::Frame(_) | Object::Traceback(_) => bt.object_.clone(), + Object::Frame(_) => bt.frame_.clone(), + Object::Traceback(_) => bt.traceback_.clone(), } } @@ -4652,7 +4822,7 @@ pub(crate) fn b_dir(args: &[Object]) -> Result { names.insert(s.to_string()); } } - for t in inst.class.mro.borrow().iter() { + for t in inst.cls().mro.borrow().iter() { for k in t.dict.borrow().keys() { if let Object::Str(s) = &k.0 { names.insert(s.to_string()); @@ -4676,7 +4846,70 @@ pub(crate) fn b_dir(args: &[Object]) -> Result { } } } - _ => {} + other => { + // Generic objects: `object.__dir__` ≈ the type's attributes. + for t in class_of(other).mro.borrow().iter() { + for k in t.dict.borrow().keys() { + if let Object::Str(s) = &k.0 { + names.insert(s.to_string()); + } + } + } + // The generator family's methods and introspection attrs are + // synthesized in `load_attr` rather than stored in type + // dicts; surface the same names CPython's type dicts hold. + let extra: &[&str] = match other { + Object::Generator(_) => &[ + "close", + "send", + "throw", + "gi_code", + "gi_frame", + "gi_running", + "gi_suspended", + "gi_yieldfrom", + "__next__", + "__iter__", + "__name__", + "__qualname__", + "__del__", + ], + Object::Coroutine(_) => &[ + "close", + "send", + "throw", + "cr_await", + "cr_code", + "cr_frame", + "cr_origin", + "cr_running", + "cr_suspended", + "__await__", + "__name__", + "__qualname__", + "__del__", + ], + Object::AsyncGenerator(_) => &[ + "aclose", + "asend", + "athrow", + "ag_await", + "ag_code", + "ag_frame", + "ag_running", + "ag_suspended", + "__aiter__", + "__anext__", + "__name__", + "__qualname__", + "__del__", + ], + _ => &[], + }; + for n in extra { + names.insert((*n).to_string()); + } + } } Ok(Object::new_list( names.into_iter().map(Object::from_str).collect(), @@ -4701,10 +4934,8 @@ fn b_hex(args: &[Object]) -> Result { } } Object::Bool(b) => Ok(Object::from_str(format!("0x{}", i64::from(*b)))), - other => Err(type_error(format!( - "'{}' object cannot be interpreted as an integer", - other.type_name() - ))), + // The `__index__` protocol (CPython `PyNumber_Index`). + other => b_hex(&[Object::Int(coerce_index_i64(other)?)]), } } @@ -4726,7 +4957,8 @@ fn b_oct(args: &[Object]) -> Result { } } Object::Bool(b) => Ok(Object::from_str(format!("0o{}", i64::from(*b)))), - _ => Err(type_error("expected int")), + // The `__index__` protocol (CPython `PyNumber_Index`). + other => b_oct(&[Object::Int(coerce_index_i64(other)?)]), } } @@ -4748,7 +4980,8 @@ fn b_bin(args: &[Object]) -> Result { } } Object::Bool(b) => Ok(Object::from_str(format!("0b{}", i64::from(*b)))), - _ => Err(type_error("expected int")), + // The `__index__` protocol (CPython `PyNumber_Index`). + other => b_bin(&[Object::Int(coerce_index_i64(other)?)]), } } @@ -5073,6 +5306,33 @@ fn b_aiter(_args: &[Object]) -> Result { Err(type_error("aiter() must be called through the VM")) } +/// Runtime support for `types.coroutine`: return a copy of a generator +/// function whose code carries `CO_ITERABLE_COROUTINE` (CPython sets +/// the flag by replacing `func.__code__`). Generators created by the +/// returned function are accepted by `await` and may `yield from` a +/// coroutine. +fn b_mark_iterable_coroutine(args: &[Object]) -> Result { + let Some(Object::Function(f)) = args.first() else { + return Err(type_error( + "_weavepy_mark_iterable_coroutine() expects a function", + )); + }; + let mut code = (*f.code).clone(); + code.is_iterable_coroutine = true; + let marked = crate::object::PyFunction { + name: f.name.clone(), + code: Rc::new(code), + globals: f.globals.clone(), + defaults: f.defaults.clone(), + kw_defaults: f.kw_defaults.clone(), + closure: f.closure.clone(), + // Shared, not copied: `func.__dict__` mutations stay visible on + // both, matching CPython where the function object is the same. + attrs: f.attrs.clone(), + }; + Ok(Object::Function(Rc::new(marked))) +} + /// `anext(async_iterator[, default])` — return the awaitable from /// `__anext__` (3.10+). VM-routed through [`crate::Vm::get_anext`]. fn b_anext(_args: &[Object]) -> Result { @@ -7395,7 +7655,16 @@ fn bytearray_extend(args: &[Object]) -> Result { .ok_or_else(|| type_error("extend() requires iterable"))?; match other { Object::Bytes(buf) => b.borrow_mut().extend_from_slice(buf), - Object::ByteArray(buf) => b.borrow_mut().extend_from_slice(&buf.borrow()), + Object::ByteArray(buf) => { + // `b.extend(b)` — self-extension must not double-borrow. + if Rc::ptr_eq(&b, buf) { + let mut t = b.borrow_mut(); + let copy = t.clone(); + t.extend_from_slice(©); + } else { + b.borrow_mut().extend_from_slice(&buf.borrow()); + } + } Object::List(items) => { let items = items.borrow(); for o in items.iter() { diff --git a/crates/weavepy-vm/src/error.rs b/crates/weavepy-vm/src/error.rs index 8340a8f..336cc12 100644 --- a/crates/weavepy-vm/src/error.rs +++ b/crates/weavepy-vm/src/error.rs @@ -31,6 +31,11 @@ pub struct PyException { pub context: Option>, /// Explicit chaining via `raise X from Y`. pub cause: Option>, + /// One-shot marker set by a bare `raise` / `RERAISE`: the *next* + /// frame-level unwind must not add a traceback entry (CPython + /// re-raises preserve the original traceback — the re-raise + /// location is not recorded). + pub suppress_tb_once: bool, } impl PyException { @@ -40,6 +45,7 @@ impl PyException { traceback: Vec::new(), context: None, cause: None, + suppress_tb_once: false, } } @@ -54,7 +60,7 @@ impl PyException { /// The class name of the wrapped instance. pub fn type_name(&self) -> String { match &self.instance { - Object::Instance(inst) => inst.class.name.clone(), + Object::Instance(inst) => inst.cls().name.clone(), _ => "BaseException".to_owned(), } } @@ -104,7 +110,7 @@ impl PyException { return None; }; if !inst - .class + .cls() .is_subclass_of(&crate::builtin_types::builtin_types().system_exit) { return None; diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 51b7b95..417dad8 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -448,6 +448,15 @@ impl GcState { handle.color.store(color::White, Ordering::Release); } + // Index the candidate set by id so the per-child lookups in + // phases 3 and 4 are O(1) — a linear `find` here makes the + // whole collection quadratic, which generator-heavy programs + // (itertools pipelines) hit hard. + let by_id: std::collections::HashMap> = candidate_set + .iter() + .map(|h| (h.id, h.clone())) + .collect(); + // Phase 3: subtract internal refs by walking each // tracked object's children. Self-references count too — // a `self.self = self` instance has one internal ref to @@ -455,7 +464,7 @@ impl GcState { // collapses to gc_refs == 0. for handle in &candidate_set { traverse_object(&handle.object, &mut |child| { - if let Some(target) = candidate_set.iter().find(|h| h.id == id_of(child)) { + if let Some(target) = by_id.get(&id_of(child)) { target.gc_refs.fetch_sub(1, Ordering::AcqRel); } }); @@ -473,7 +482,7 @@ impl GcState { while let Some(h) = grey.pop() { h.color.store(color::Black, Ordering::Release); traverse_object(&h.object, &mut |child| { - if let Some(target) = candidate_set.iter().find(|t| t.id == id_of(child)) { + if let Some(target) = by_id.get(&id_of(child)) { if target.color.load(Ordering::Acquire) == color::White { target.color.store(color::Grey, Ordering::Release); grey.push(target.clone()); @@ -779,6 +788,14 @@ pub fn clear_object_fields(obj: &Object) { Object::Cell(c) => { *c.borrow_mut() = Object::None; } + Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { + // Dropping the suspended frame box breaks the cycle + // (the finalizer — close() — has already run by the + // time clear is reached; see collect phase 5c). + if let Ok(mut st) = g.state.try_borrow_mut() { + *st = crate::object::GeneratorState::Finished; + } + } _ => {} } } @@ -794,12 +811,17 @@ fn run_finalizer(obj: &Object) { } } -/// True iff `obj`'s class defines `__del__`. +/// True iff `obj` needs finalization when it becomes garbage: +/// instances whose class defines `__del__`, and generator-family +/// objects that haven't finished (closing them runs `finally` +/// blocks — CPython's `gen_dealloc` behavior). fn has_finalizer(obj: &Object) -> bool { - if let Object::Instance(inst) = obj { - inst.class.lookup("__del__").is_some() - } else { - false + match obj { + Object::Instance(inst) => inst.cls().lookup("__del__").is_some(), + Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { + !g.is_finished() + } + _ => false, } } @@ -819,6 +841,39 @@ pub fn track(obj: Object) { with_state(|s| s.track(obj)); } +/// Convenience: find a tracked handle by object id (scans all +/// generations plus the frozen set). +pub fn find_handle(id: ObjectId) -> Option> { + with_state(|s| { + { + let gens = s.generations.borrow(); + for g in gens.iter() { + if let Some(h) = g.handles.iter().find(|h| h.id == id) { + return Some(h.clone()); + } + } + } + s.frozen.borrow().iter().find(|h| h.id == id).cloned() + }) +} + +/// Convenience: is `id` currently tracked by the cycle GC? Used +/// by refcount-emulation paths to discount the registry's own +/// strong handle. +pub fn is_tracked(id: ObjectId) -> bool { + find_handle(id).is_some() +} + +/// Convenience: claim `id`'s finalizer (so a later collection +/// won't double-run `__del__`). Returns false if it was already +/// claimed or the object isn't tracked. +pub fn mark_finalized(id: ObjectId) -> bool { + match find_handle(id) { + Some(h) => !h.finalized.swap(true, Ordering::AcqRel), + None => false, + } +} + /// Convenience: snapshot all tracked objects with an unrun `__del__` /// on the current thread's GC (see [`GcState::finalization_candidates`]). pub fn finalization_candidates() -> Vec> { diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 2619b85..55f923d 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -157,6 +157,85 @@ impl Frame { } } +/// Cycle-GC traversal hook for generator-family objects: walk the +/// suspended frame's locals, evaluation stack, and cells so cycles +/// running through a generator frame (`g.send(g)`) are detectable. +/// Registered with `gc_trace::register_traverse` at interpreter init +/// because `Frame` is private to this module. +fn generator_frame_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) { + let g = match obj { + Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => g, + _ => return, + }; + let Ok(state) = g.state.try_borrow() else { + return; + }; + let boxed = match &*state { + GeneratorState::Created(b) | GeneratorState::Suspended(b) => b, + _ => return, + }; + if let Some(frame) = boxed.downcast_ref::() { + for v in &frame.locals { + visit(v); + } + for v in &frame.stack { + visit(v); + } + for c in &frame.cells { + if let Ok(v) = c.try_borrow() { + visit(&v); + } + } + // The cached Python-visible frame snapshot keeps its own strong + // clones of the locals (the live mirror) plus a materialised + // f_locals dict; both are edges out of this generator. + if let Some(py) = &frame.py_frame { + if let Ok(mirror) = py.locals_mirror.try_borrow() { + if let Some(mirror) = mirror.as_ref() { + if let Ok(vals) = mirror.try_borrow() { + for v in vals.iter() { + visit(v); + } + } + } + } + if let Ok(cache) = py.locals_cache.try_borrow() { + if let Some(d) = cache.as_ref() { + visit(d); + } + } + } + } +} + +/// Snapshot the objects in a generator's suspended frame that will +/// need finalization when the frame is dropped (instances with a +/// `__del__`). Captured *before* `close()` tears the frame down so +/// the caller can emulate CPython's prompt refcount-driven +/// finalization afterwards. +fn frame_finalizables(g: &Rc) -> Vec { + let Ok(state) = g.state.try_borrow() else { + return Vec::new(); + }; + let boxed = match &*state { + GeneratorState::Created(b) | GeneratorState::Suspended(b) => b, + _ => return Vec::new(), + }; + let Some(frame) = boxed.downcast_ref::() else { + return Vec::new(); + }; + let finalizable = |o: &&Object| -> bool { + matches!(o, Object::Instance(i) if i.cls().lookup("__del__").is_some()) + }; + frame + .locals + .iter() + .filter(finalizable) + .chain(frame.stack.iter().filter(finalizable)) + .cloned() + .collect() +} + /// RFC 0032 — render the tier-2 JIT's counters as a markdown block for /// the `WEAVEPY_VM_STATS` report, or `None` when the `jit` feature is /// disabled or the JIT was never exercised on this thread. @@ -265,6 +344,20 @@ impl Default for Interpreter { let builtins = Rc::new(RefCell::new(builtins_dict)); let cache = ModuleCache::default(); stdlib::register_all(&cache); + // RFC 0024: teach the cycle GC to walk suspended generator + // frames (their `Frame` type is private to this module). + static GEN_TRAVERSE: std::sync::Once = std::sync::Once::new(); + GEN_TRAVERSE.call_once(|| { + crate::gc_trace::register_traverse( + |o| { + matches!( + o, + Object::Generator(_) | Object::Coroutine(_) | Object::AsyncGenerator(_) + ) + }, + generator_frame_traverse, + ); + }); let excepthook = Rc::new(RefCell::new(Object::None)); let unraisable_hook = Rc::new(RefCell::new(Object::None)); let frame_stack: Rc>>> = Rc::new(RefCell::new(Vec::new())); @@ -661,6 +754,49 @@ impl Interpreter { result } + /// Emulate CPython's prompt refcount-driven death for an object + /// whose binding was just dropped (a `del` statement, or a frame + /// torn down by `close()`). If `dropped` holds the last + /// program-visible reference — everything else is the GC + /// registry's handle and weakref slots' strong clones — run its + /// finalizer now and clear its weakrefs, exactly as CPython's + /// `tp_dealloc` would. Conservative: any extra reference anywhere + /// (containers, caches, other bindings) skips the reap and leaves + /// the object to the cycle collector. + fn prompt_reap_dropped(&mut self, dropped: Object) { + let finalizable = match &dropped { + Object::Instance(i) => i.cls().lookup("__del__").is_some(), + Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { + !g.is_finished() + } + _ => return, + }; + let id = crate::weakref_registry::id_of(&dropped); + let watched = crate::weakref_registry::count_for(id) > 0; + if !finalizable && !watched { + return; + } + let strong = gc_trace::strong_count_for(&dropped); + let registry_holds = usize::from(gc_trace::is_tracked(id)); + let weak_clones = crate::weakref_registry::strong_clone_count(id); + // `dropped` itself accounts for one reference. + if strong > 1 + registry_holds + weak_clones { + return; + } + // Dead in refcount terms. CPython order: __del__ first, then + // weakrefs are cleared. + if finalizable && gc_trace::mark_finalized(id) { + crate::vm_singletons::push_pending_finalizer(dropped.clone()); + self.run_pending_finalizers(); + } else if finalizable && registry_holds == 0 { + // Not tracked (shouldn't happen for instances/generators, + // but stay safe): run the finalizer directly. + crate::vm_singletons::push_pending_finalizer(dropped.clone()); + self.run_pending_finalizers(); + } + let _ = crate::weakref_registry::notify_clear(id); + } + /// Invoke any `__del__` finalizers queued by the cycle GC. /// Each finalizer runs at most once. Exceptions from a /// finalizer are routed through `sys.unraisablehook` (the @@ -711,28 +847,67 @@ impl Interpreter { /// through the unraisable hook. Used by both the cycle-GC drain and /// the shutdown pass. fn invoke_finalizer(&mut self, obj: &Object) { + // A coroutine that was created but never driven: CPython's + // `_PyGen_Finalize` emits the "was never awaited" + // RuntimeWarning instead of closing. With warnings-as-errors + // the raised warning routes through the unraisable hook, + // with the coroutine itself as the hook's object. + if let Object::Coroutine(g) = obj { + if matches!(&*g.state.borrow(), GeneratorState::Created(_)) { + *g.state.borrow_mut() = GeneratorState::Finished; + let message = + format!("coroutine '{}' was never awaited", g.qualname.borrow()); + if let Err(err) = self.emit_runtime_warning(message) { + let outer = Rc::new(RefCell::new(DictData::new())); + let context_repr = self + .repr_of(obj, &outer) + .unwrap_or_else(|_| obj.repr()); + self.write_unraisable(&err, obj, &context_repr); + } + return; + } + } + // A generator/coroutine dropped while suspended: deliver + // `GeneratorExit` (CPython's `gen_dealloc` → `gen_close`). + // A generator that ignores the exit, or raises, reports via + // `sys.unraisablehook` like any finalizer error. + if matches!( + obj, + Object::Generator(_) | Object::Coroutine(_) | Object::AsyncGenerator(_) + ) { + if let Err(err) = self.gen_method_close(obj) { + let outer = Rc::new(RefCell::new(DictData::new())); + let context_repr = self + .repr_of(obj, &outer) + .unwrap_or_else(|_| obj.repr()); + self.write_unraisable(&err, obj, &context_repr); + } + return; + } let Object::Instance(inst) = obj else { return; }; - let Some(del) = inst.class.lookup("__del__") else { + let Some(del) = inst.cls().lookup("__del__") else { return; }; - let class_name = inst.class.name.clone(); + let class_name = inst.cls().name.clone(); let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: obj.clone(), - function: del, + function: del.clone(), })); let kwargs: Vec<(String, Object)> = Vec::new(); let outer = Rc::new(RefCell::new(DictData::new())); if let Err(err) = self.call(&bound, &[], &kwargs, &outer) { - // The finalizer is the `object` reported to the hook; the - // printed context mirrors CPython's bound-method repr so a + // CPython's `slot_tp_finalize` reports the *looked-up* + // `__del__` (the plain function) as the hook's `object` + // (`cm.unraisable.object == C.__del__` holds); the printed + // context still mirrors the bound-method repr so the // default hook emits `… `. let receiver_repr = self .repr_of(obj, &outer) .unwrap_or_else(|_| obj.repr()); let context_repr = format!(""); - self.write_unraisable(&err, &bound, &context_repr); + self.write_unraisable(&err, &del, &context_repr); } } @@ -748,7 +923,7 @@ impl Interpreter { RuntimeError::PyException(pyexc) => { let inst = pyexc.instance.clone(); let ty = match &inst { - Object::Instance(i) => Object::Type(i.class.clone()), + Object::Instance(i) => Object::Type(i.cls()), _ => Object::None, }; (inst, ty, pyexc.traceback.clone()) @@ -757,7 +932,7 @@ impl Interpreter { let inst = crate::builtin_types::make_exception("RuntimeError", other.to_string()); let ty = match &inst { - Object::Instance(i) => Object::Type(i.class.clone()), + Object::Instance(i) => Object::Type(i.cls()), _ => Object::None, }; (inst, ty, Vec::new()) @@ -816,10 +991,19 @@ impl Interpreter { let mut d = DictData::new(); d.insert(DictKey(Object::from_static("exc_type")), exc_type.clone()); d.insert(DictKey(Object::from_static("exc_value")), exc_value.clone()); - d.insert( - DictKey(Object::from_static("exc_traceback")), - Object::None, - ); + // The exception instance carries its traceback in + // `__traceback__`; surface it as `exc_traceback` like + // CPython's `UnraisableHookArgs`. + let tb = match exc_value { + Object::Instance(inst) => inst + .dict + .borrow() + .get(&DictKey(Object::from_static("__traceback__"))) + .cloned() + .unwrap_or(Object::None), + _ => Object::None, + }; + d.insert(DictKey(Object::from_static("exc_traceback")), tb); d.insert(DictKey(Object::from_static("err_msg")), Object::None); d.insert(DictKey(Object::from_static("object")), object.clone()); Object::SimpleNamespace(Rc::new(RefCell::new(d))) @@ -855,7 +1039,7 @@ impl Interpreter { } let (kind, msg) = match exc_value { Object::Instance(i) => ( - i.class.name.clone(), + i.cls().name.clone(), crate::builtin_types::exception_message(exc_value).unwrap_or_default(), ), _ => ("Exception".to_owned(), String::new()), @@ -1162,7 +1346,27 @@ impl Interpreter { } Ok(StepOutcome::StartGenerator) => break Ok(FrameOutcome::StartGenerator), Err(err) => { - if let RuntimeError::PyException(exc) = err { + if let RuntimeError::PyException(mut exc) = err { + // CPython's `_PyErr_SetObject` chains *every* + // fresh exception — including ones raised from C + // (here: Rust opcodes and builtins) — to the + // currently handled exception. `RAISE_VARARGS` + // already did this at the raise site; a fresh + // Rust-raised error is recognisable by its empty + // traceback and unset context/cause. Re-raises + // carry `suppress_tb_once` or an existing dict + // `__context__` and are left alone. + if exc.context.is_none() + && exc.cause.is_none() + && exc.traceback.is_empty() + && !exc.suppress_tb_once + && !instance_has_nonnull_attr(&exc.instance, "__context__") + { + self.attach_implicit_context(&mut exc); + if exc.context.is_some() { + Self::sync_exc_attrs(&exc); + } + } if crate::trace::any_observers_active() { self.fire_exception_event(&py_frame, &exc)?; } @@ -1342,6 +1546,7 @@ impl Interpreter { locals_provider: RefCell::new(Some(provider)), locals_mirror: RefCell::new(Some(locals_snapshot)), trace: RefCell::new(Object::None), + gen_owner: RefCell::new(None), override_lineno: Cell::new(None), last_line: Cell::new(None), trace_lines: Cell::new(true), @@ -1349,6 +1554,62 @@ impl Interpreter { }) } + /// `frame.clear()` — drop the references a no-longer-executing + /// frame holds (CPython `frame_clear`). Clearing the frame of a + /// *created* (never-started) generator finalizes the generator; + /// executing and suspended frames raise `RuntimeError` (gh-79932). + fn frame_clear(&mut self, py: &Rc) -> Result { + if self.frame_stack.borrow().iter().any(|f| Rc::ptr_eq(f, py)) { + return Err(crate::error::runtime_error( + "cannot clear an executing frame", + )); + } + let owner = py.gen_owner.borrow().as_ref().and_then(|w| w.upgrade()); + if let Some(g) = owner { + let state_kind = match &*g.state.borrow() { + GeneratorState::Running => 0u8, + GeneratorState::Suspended(_) => 1, + GeneratorState::Created(_) => 2, + GeneratorState::Finished => 3, + }; + match state_kind { + 0 => { + return Err(crate::error::runtime_error( + "cannot clear an executing frame", + )) + } + 1 => { + return Err(crate::error::runtime_error( + "cannot clear a suspended frame", + )) + } + 2 => { + // Tear down the never-started generator: its frame + // locals (bound arguments) die now, like CPython's + // refcount-driven dealloc. Drop the snapshot's own + // strong clones (mirror, materialised dict) *first* + // so the reap sees the true remaining refcount. + let caps = frame_finalizables(&g); + *g.state.borrow_mut() = GeneratorState::Finished; + if let Some(mirror) = py.locals_mirror.borrow().as_ref() { + mirror.borrow_mut().clear(); + } + py.invalidate_locals(); + for cap in caps { + self.prompt_reap_dropped(cap); + } + return Ok(Object::None); + } + _ => {} + } + } + if let Some(mirror) = py.locals_mirror.borrow().as_ref() { + mirror.borrow_mut().clear(); + } + py.invalidate_locals(); + Ok(Object::None) + } + /// Refresh the live-locals mirror on the current Python frame. /// Called between bytecode steps so `sys._getframe(...).f_locals` /// reflects the most recent `STORE_FAST` / `DELETE_FAST`. @@ -1391,6 +1652,11 @@ impl Interpreter { /// (the frame is gone); the suspended/created case covers the /// introspection the conformance suite performs. fn gen_code_object(&self, g: &Rc) -> Object { + // The generator holds its own strong reference (CPython + // `gi_code`), so the attribute survives exhaustion. + if !matches!(g.code, Object::None) { + return g.code.clone(); + } match &*g.state.borrow() { GeneratorState::Created(boxed) | GeneratorState::Suspended(boxed) => boxed .downcast_ref::() @@ -1412,6 +1678,9 @@ impl Interpreter { match boxed.downcast_mut::() { Some(frame) => { if let Some(py) = frame.py_frame.clone() { + if py.gen_owner.borrow().is_none() { + *py.gen_owner.borrow_mut() = Some(Rc::downgrade(g)); + } return Object::Frame(py); } // Not yet entered: build the frame snapshot now so @@ -1419,6 +1688,7 @@ impl Interpreter { // `next()`. `back` is None — a created/suspended // generator frame has no live caller. let py = self.build_py_frame(frame, None); + *py.gen_owner.borrow_mut() = Some(Rc::downgrade(g)); frame.py_frame = Some(py.clone()); Object::Frame(py) } @@ -1431,6 +1701,29 @@ impl Interpreter { } } + /// `gi_yieldfrom` / `cr_await` / `ag_await`: the sub-iterator a + /// suspended `yield from` / `await` is delegating to, or None when + /// not delegating. A frame parked in a delegation sits just past the + /// `SEND`/`YIELD_VALUE` pair with the delegate at top-of-stack. + fn gen_yieldfrom(&self, g: &Rc) -> Object { + let state = g.state.borrow(); + if let GeneratorState::Suspended(boxed) = &*state { + if let Some(frame) = boxed.downcast_ref::() { + let pc = frame.pc as usize; + if pc >= 2 { + if let Some(send_ins) = frame.code.instructions.get(pc - 2) { + if send_ins.op == OpCode::Send { + if let Some(delegate) = frame.stack.last() { + return delegate.clone(); + } + } + } + } + } + } + Object::None + } + // =========================================================== // RFC 0031 — VM observability hooks (settrace / setprofile / // sys.monitoring / sys.audit). @@ -1626,7 +1919,7 @@ impl Interpreter { // approximate with (type, value, None) — the instance // already carries `__traceback__`. let exc_type = match &exc.instance { - Object::Instance(inst) => Object::Type(inst.class.clone()), + Object::Instance(inst) => Object::Type(inst.cls()), _ => Object::None, }; let arg = Object::new_tuple(vec![exc_type, exc.instance.clone(), Object::None]); @@ -1864,7 +2157,8 @@ impl Interpreter { OpCode::DeleteFast => { let slot = ins.arg as usize; if slot < frame.locals.len() { - frame.locals[slot] = Object::None; + let old = std::mem::replace(&mut frame.locals[slot], Object::None); + self.prompt_reap_dropped(old); } } OpCode::StoreName => { @@ -1908,17 +2202,23 @@ impl Interpreter { return Ok(StepOutcome::Continue); } } - frame + let old = frame .globals .borrow_mut() .shift_remove(&DictKey(Object::from_str(name))); + if let Some(old) = old { + self.prompt_reap_dropped(old); + } } OpCode::DeleteGlobal => { let name = self.name_at(&frame.code, ins.arg)?; - frame + let old = frame .globals .borrow_mut() .shift_remove(&DictKey(Object::from_str(name))); + if let Some(old) = old { + self.prompt_reap_dropped(old); + } } OpCode::LoadDeref => { let cell = frame @@ -1948,7 +2248,8 @@ impl Interpreter { .get(ins.arg as usize) .cloned() .ok_or_else(|| RuntimeError::Internal("bad cell index".to_owned()))?; - *cell.borrow_mut() = Object::None; + let old = std::mem::replace(&mut *cell.borrow_mut(), Object::None); + self.prompt_reap_dropped(old); } OpCode::MakeCell => { let slot = ins.arg as usize; @@ -2305,11 +2606,26 @@ impl Interpreter { // adjusted for exhaustion. Continue dispatch. return Ok(StepOutcome::Continue); } - let it_obj = frame + let mut it_obj = frame .stack .last() .cloned() .ok_or_else(|| RuntimeError::Internal("FOR_ITER no iter".to_owned()))?; + // A suspended frame's `f_locals` write (PEP 667) can smuggle + // a non-iterator into the loop slot. CPython 3.13 tolerates + // this: iterate the object afresh and pin the new iterator + // in the slot (TypeError "'X' object is not iterable" for + // non-iterables, via iter()). + if !matches!( + &it_obj, + Object::Iter(_) | Object::Generator(_) | Object::Instance(_) + ) { + let fresh = self.make_iter(&it_obj, &frame.globals)?; + if let Some(slot) = frame.stack.last_mut() { + *slot = fresh.clone(); + } + it_obj = fresh; + } let next = match &it_obj { Object::Iter(it) => it.borrow_mut().next_value(), Object::Generator(g) => match self.generator_send(g, Object::None) { @@ -2789,12 +3105,15 @@ impl Interpreter { OpCode::RaiseVarargs => { let mut exc = match ins.arg { 0 => { - // Re-raise the currently-handled exception. - let top = frame + // Re-raise the currently-handled exception. A bare + // `raise` preserves the original traceback: the + // re-raise site is *not* recorded (CPython RERAISE). + let mut top = frame .exc_handlers .last() .map(|(_, pe)| pe.clone()) .ok_or_else(|| runtime_error("No active exception to re-raise"))?; + top.suppress_tb_once = true; top } 1 => { @@ -2835,7 +3154,7 @@ impl Interpreter { let exc = frame.pop()?; let is_group = matches!( &exc, - Object::Instance(i) if i.class.is_subclass_of( + Object::Instance(i) if i.cls().is_subclass_of( &builtin_types().base_exception_group ) ); @@ -2891,7 +3210,10 @@ impl Interpreter { .ok_or_else(|| runtime_error("No active exception to re-raise"))?; exc }; - let pe = Self::normalize_exception(exc, None)?; + let mut pe = Self::normalize_exception(exc, None)?; + // Re-raises keep the original traceback; the RERAISE + // site itself is not recorded (matches CPython). + pe.suppress_tb_once = true; Self::sync_exc_attrs(&pe); return Err(RuntimeError::PyException(pe)); } @@ -2906,8 +3228,8 @@ impl Interpreter { } OpCode::WithExceptStart => { // Stack on entry (top → bottom): [exc, exit] - // We call exit(type(exc), exc, None) and push the - // result, leaving exc and exit beneath. + // We call exit(type(exc), exc, exc.__traceback__) and + // push the result, leaving exc and exit beneath. let exc = frame .stack .last() @@ -2918,12 +3240,19 @@ impl Interpreter { .get(frame.stack.len().wrapping_sub(2)) .cloned() .ok_or_else(|| RuntimeError::Internal("WITH_EXCEPT_START".to_owned()))?; - let ty = match &exc { - Object::Instance(inst) => Object::Type(inst.class.clone()), - _ => Object::None, + let (ty, tb) = match &exc { + Object::Instance(inst) => { + let tb = inst + .dict + .borrow() + .get(&DictKey(Object::from_static("__traceback__"))) + .cloned() + .unwrap_or(Object::None); + (Object::Type(inst.cls()), tb) + } + _ => (Object::None, Object::None), }; - let result = - self.call(&exit_method, &[ty, exc, Object::None], &[], &frame.globals)?; + let result = self.call(&exit_method, &[ty, exc, tb], &[], &frame.globals)?; frame.push(result); } OpCode::ImportName => { @@ -2984,6 +3313,19 @@ impl Interpreter { let v = frame.pop()?; let it = match v { Object::Generator(_) => v, + // PEP 492: a coroutine may only be `yield from`-ed + // inside another coroutine or a `types.coroutine`- + // marked generator (CO_ITERABLE_COROUTINE). + Object::Coroutine(_) => { + if frame.code.is_coroutine || frame.code.is_iterable_coroutine { + v + } else { + return Err(type_error( + "cannot 'yield from' a coroutine object in a non-coroutine \ + generator", + )); + } + } other => self.make_iter(&other, &frame.globals)?, }; frame.push(it); @@ -3052,6 +3394,34 @@ impl Interpreter { None => Err(stop_iteration()), } } + // A custom awaitable's `__await__` may return any + // iterator object (e.g. asyncio Futures). CPython's + // `PyIter_Send`: use `send(value)` when available, + // else `__next__` for a None send. `StopIteration` + // raised by either carries the await's result and is + // handled below. + Object::Instance(_) => { + let globals = frame.globals.clone(); + if !matches!(value, Object::None) { + if let Some(m) = instance_method(&iter, "send") { + self.call(&m, std::slice::from_ref(&value), &[], &globals) + } else { + Err(type_error(format!( + "'{}' object has no attribute 'send'", + iter.type_name_owned() + ))) + } + } else if let Some(m) = instance_method(&iter, "__next__") { + self.call(&m, &[], &[], &globals) + } else if let Some(m) = instance_method(&iter, "send") { + self.call(&m, std::slice::from_ref(&value), &[], &globals) + } else { + Err(type_error(format!( + "'{}' object is not an iterator", + iter.type_name_owned() + ))) + } + } _ => Err(type_error("SEND expects an iterator or generator")), }; match result { @@ -3085,7 +3455,7 @@ impl Interpreter { } OpCode::GetAwaitable => { let v = frame.pop()?; - let it = self.get_awaitable(v)?; + let it = self.get_awaitable(v, ins.arg)?; frame.push(it); } OpCode::GetAiter => { @@ -3115,8 +3485,28 @@ impl Interpreter { OpCode::BeforeAsyncWith => { let cm = frame.pop()?; let globals = frame.globals.clone(); - let aexit = self.load_attr(&cm, "__aexit__")?; - let aenter = self.load_attr(&cm, "__aenter__")?; + // CPython looks up `__aenter__` first, then `__aexit__`, + // with protocol-specific TypeErrors (not AttributeError). + let aenter = self.load_attr(&cm, "__aenter__").map_err(|e| match e { + RuntimeError::PyException(exc) if exc.type_name() == "AttributeError" => { + type_error(format!( + "'{}' object does not support the asynchronous context manager \ + protocol", + cm.type_name_owned() + )) + } + other => other, + })?; + let aexit = self.load_attr(&cm, "__aexit__").map_err(|e| match e { + RuntimeError::PyException(exc) if exc.type_name() == "AttributeError" => { + type_error(format!( + "'{}' object does not support the asynchronous context manager \ + protocol (missed __aexit__ method)", + cm.type_name_owned() + )) + } + other => other, + })?; let aw = self.call(&aenter, &[], &[], &globals)?; frame.push(aexit); frame.push(aw); @@ -3213,8 +3603,14 @@ impl Interpreter { // Push a traceback entry for *this* frame regardless of // whether we end up handling here — CPython's `__traceback__` // includes the catching frame too. The chain grows - // outward-from-raise as the exception propagates. - self.append_traceback(&mut exc, frame, raise_pc, line); + // outward-from-raise as the exception propagates. A bare + // `raise`/`RERAISE` is the exception: the original traceback + // already records this frame, so the re-raise site is skipped. + if exc.suppress_tb_once { + exc.suppress_tb_once = false; + } else { + self.append_traceback(&mut exc, frame, raise_pc, line); + } if let Some(handler) = find_handler(&frame.code.exception_table, raise_pc) { // Drop entries above the recorded stack depth. while frame.stack.len() > handler.depth as usize { @@ -3280,6 +3676,7 @@ impl Interpreter { locals_provider: RefCell::new(None), locals_mirror: RefCell::new(None), trace: RefCell::new(Object::None), + gen_owner: RefCell::new(None), override_lineno: Cell::new(None), last_line: Cell::new(None), trace_lines: Cell::new(true), @@ -3476,7 +3873,7 @@ impl Interpreter { make_exception_with_class(t, "") } Object::Instance(inst) => { - if !inst.class.flags.is_exception && !inst.class.is_subclass_of(&bt.base_exception) + if !inst.cls().flags.is_exception && !inst.cls().is_subclass_of(&bt.base_exception) { return Err(type_error("exceptions must derive from BaseException")); } @@ -3642,28 +4039,31 @@ impl Interpreter { // The code object backing the generator. "code" => return Ok(self.gen_code_object(g)), // Currently executing (illegal to re-enter)? + // CPython exposes these as 0/1 ints. "running" => { - return Ok(Object::Bool(matches!( + return Ok(Object::Int(i64::from(matches!( &*g.state.borrow(), GeneratorState::Running - ))) + )))) } // The (stable) Python-visible frame, or None once // the generator has finished. "frame" => return Ok(self.gen_py_frame(g)), - // PEP 580-era `gi_suspended` (generators only). - "suspended" if prefix == "gi_" => { - return Ok(Object::Bool(matches!( + "suspended" => { + return Ok(Object::Int(i64::from(matches!( &*g.state.borrow(), GeneratorState::Suspended(_) - ))) + )))) } // The sub-iterator a `yield from` / `await` is - // currently delegating to. We don't track the - // active delegate yet, so report "not delegating". - "yieldfrom" if prefix == "gi_" => return Ok(Object::None), + // currently delegating to (CPython `gi_yieldfrom`: + // the object on top of the suspended frame's stack + // when paused inside a `yield from` / `await`). + "yieldfrom" if prefix == "gi_" => { + return Ok(self.gen_yieldfrom(g)) + } "await" if prefix == "cr_" || prefix == "ag_" => { - return Ok(Object::None) + return Ok(self.gen_yieldfrom(g)) } _ => {} } @@ -3990,6 +4390,22 @@ impl Interpreter { "f_trace" => Ok(fr.trace.borrow().clone()), "f_trace_lines" => Ok(Object::Bool(fr.trace_lines.get())), "f_trace_opcodes" => Ok(Object::Bool(fr.trace_opcodes.get())), + "clear" => { + let fr = fr.clone(); + Ok(Object::Builtin(Rc::new(BuiltinFn { + name: "frame.clear", + call: Box::new(move |_args| { + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| { + RuntimeError::Internal("no running interpreter".to_owned()) + })?; + // SAFETY: published by the enclosing VM frame. + let interp = unsafe { &mut *ptr }; + interp.frame_clear(&fr) + }), + call_kw: None, + }))) + } _ => Err(attribute_error(format!( "'frame' object has no attribute '{}'", name @@ -4015,7 +4431,9 @@ impl Interpreter { Ok(Object::from_static(builtin_display_name(b.name))) } "__module__" => Ok(Object::from_static("builtins")), - "__doc__" => Ok(Object::None), + "__doc__" => Ok(builtin_doc(b.name) + .map(Object::from_static) + .unwrap_or(Object::None)), "__self__" => Ok(Object::None), _ => Err(attribute_error(format!( "'builtin_function_or_method' object has no attribute '{}'", @@ -4030,7 +4448,15 @@ impl Interpreter { Object::Builtin(b) => Ok(Object::from_static(builtin_display_name(b.name))), _ => Ok(Object::from_static("?")), }, - "__doc__" => Ok(Object::None), + "__doc__" => match &bm.function { + Object::Function(f) => { + Ok(crate::builtins::code_docstring(&f.code).unwrap_or(Object::None)) + } + Object::Builtin(b) => Ok(builtin_doc(b.name) + .map(Object::from_static) + .unwrap_or(Object::None)), + _ => Ok(Object::None), + }, "__code__" => match &bm.function { Object::Function(f) => Ok(Object::Code(f.code.clone())), _ => Err(attribute_error(format!( @@ -4148,7 +4574,7 @@ impl Interpreter { instance_obj: &Object, name: &str, ) -> Result { - let result = if let Some(getattribute) = self.user_getattribute(&inst.class) { + let result = if let Some(getattribute) = self.user_getattribute(&inst.cls()) { let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: instance_obj.clone(), function: getattribute, @@ -4164,7 +4590,7 @@ impl Interpreter { }; match result { Err(e) if self.is_attribute_error(&e) => { - if let Some(getattr) = inst.class.lookup("__getattr__") { + if let Some(getattr) = inst.cls().lookup("__getattr__") { let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: instance_obj.clone(), function: getattr, @@ -4240,7 +4666,7 @@ impl Interpreter { .cloned(); if name != "__self__" { if let Some(receiver) = super_receiver { - if let Some(v) = inst.class.lookup(name) { + if let Some(v) = inst.cls().lookup(name) { // CPython passes `su->obj_type` (the MRO-walk start // class) as `owner`, and a NULL instance when // `su->obj == starttype` — the class-bound form @@ -4255,8 +4681,8 @@ impl Interpreter { let owner = match (&obj_type, &receiver) { (Some(Object::Type(t)), _) => Object::Type(t.clone()), (_, Object::Type(t)) => Object::Type(t.clone()), - (_, Object::Instance(i)) => Object::Type(i.class.clone()), - _ => Object::Type(inst.class.clone()), + (_, Object::Instance(i)) => Object::Type(i.cls()), + _ => Object::Type(inst.cls()), }; let instance_for_get = match (&receiver, &owner) { (Object::Type(r), Object::Type(o)) if Rc::ptr_eq(r, o) => { @@ -4290,8 +4716,8 @@ impl Interpreter { } } - let meta_attr = inst.class.lookup(name); - let owner = Object::Type(inst.class.clone()); + let meta_attr = inst.cls().lookup(name); + let owner = Object::Type(inst.cls()); // (1) Data descriptor on class wins over instance dict. if let Some(ref attr) = meta_attr { @@ -4316,7 +4742,7 @@ impl Interpreter { // `functools.cached_property`) reaches for them anyway. match name { "__dict__" => return Ok(Object::Dict(inst.dict.clone())), - "__class__" => return Ok(Object::Type(inst.class.clone())), + "__class__" => return Ok(Object::Type(inst.cls())), _ => {} } @@ -4379,7 +4805,7 @@ impl Interpreter { // and any `__getattribute__` override raise `AttributeError`. Err(attribute_error(format!( "'{}' object has no attribute '{}'", - inst.class.name, name + inst.cls().name, name ))) } @@ -4418,10 +4844,19 @@ impl Interpreter { // (2) Look up the name in `ty` itself (and its MRO). if let Some(attr) = ty.lookup(name) { - // Apply the descriptor protocol with no instance: classmethods - // bind to the class, plain functions stay as functions, - // staticmethods unwrap, properties remain themselves. - return self.descriptor_get(&attr, &Object::None, &owner); + // A `__name__`/`__qualname__` *getset* in a class dict + // describes instances (e.g. `coroutine.__name__`); for the + // class itself CPython's metaclass getset (`type.__name__`) + // takes precedence and reports the type's own name. Fall + // through to the synthetic below. + let meta_owned = matches!(name, "__name__" | "__qualname__") + && matches!(attr, Object::Property(_)); + if !meta_owned { + // Apply the descriptor protocol with no instance: classmethods + // bind to the class, plain functions stay as functions, + // staticmethods unwrap, properties remain themselves. + return self.descriptor_get(&attr, &Object::None, &owner); + } } // (3) Fall-through to (possibly non-data) metaclass attribute. @@ -4568,7 +5003,7 @@ impl Interpreter { match attr { Object::Property(_) | Object::SlotDescriptor(_) => true, Object::Instance(inst) => { - inst.class.lookup("__set__").is_some() || inst.class.lookup("__delete__").is_some() + inst.cls().lookup("__set__").is_some() || inst.cls().lookup("__delete__").is_some() } _ => false, } @@ -4611,7 +5046,7 @@ impl Interpreter { Some(v) => Ok(v.clone()), None => Err(attribute_error(format!( "'{}' object has no attribute '{}'", - inst.class.name, slot.name + inst.cls().name, slot.name ))), } } @@ -4651,7 +5086,7 @@ impl Interpreter { } // User-defined descriptor: invoke its `__get__` if // present, otherwise pass the descriptor through. - if let Some(get_method) = inner_inst.class.lookup("__get__") { + if let Some(get_method) = inner_inst.cls().lookup("__get__") { let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: attr.clone(), function: get_method, @@ -4677,7 +5112,7 @@ impl Interpreter { })), Object::ClassMethod(inner) => Object::BoundMethod(Rc::new(BoundMethod { receiver: match receiver { - Object::Instance(inst) => Object::Type(inst.class.clone()), + Object::Instance(inst) => Object::Type(inst.cls()), other => other.clone(), }, function: (**inner).clone(), @@ -5379,6 +5814,20 @@ impl Interpreter { .map(|_| ()) } + /// Emit a `RuntimeWarning` through the live `warnings` machinery so + /// filters / `catch_warnings` apply. Used for "coroutine … was + /// never awaited" at finalization. + fn emit_runtime_warning(&mut self, message: String) -> Result<(), RuntimeError> { + let Some(warn) = self.module_attr("warnings", "warn") else { + return Ok(()); + }; + let category = + Object::Type(crate::builtin_types::builtin_types().runtime_warning.clone()); + let globals = self.builtins.clone(); + self.call(&warn, &[Object::from_str(message), category], &[], &globals) + .map(|_| ()) + } + /// `pow(base, exp[, mod])` — dispatch the numeric dunders for class /// instances. The two-argument form routes through the normal /// `__pow__`/`__rpow__` binary-op machinery; the three-argument form @@ -5894,7 +6343,7 @@ impl Interpreter { /// `__getitem__`) before falling back to iter-of-pairs. We do the /// same for user-defined instances: if the instance exposes /// `keys()`, call it and pull each value via subscript. - fn try_dict_from_mapping( + pub(crate) fn try_dict_from_mapping( &mut self, v: &Object, globals: &Rc>, @@ -5926,7 +6375,7 @@ impl Interpreter { ))))); } // Prefer the instance's own `keys` (rare), then walk the MRO. - // `inst.class.lookup` already handles inheritance, which is + // `inst.cls().lookup` already handles inheritance, which is // how `_MappingMixin` subclasses (defaultdict, Counter, …) // get their mapping API. let keys_attr = inst @@ -5934,7 +6383,7 @@ impl Interpreter { .borrow() .get(&DictKey(Object::from_str("keys"))) .cloned() - .or_else(|| inst.class.lookup("keys")); + .or_else(|| inst.cls().lookup("keys")); let Some(keys_fn) = keys_attr else { return Ok(None); }; @@ -5956,7 +6405,7 @@ impl Interpreter { Ok(Some(Object::Dict(Rc::new(RefCell::new(d))))) } - fn collect_iterable( + pub(crate) fn collect_iterable( &mut self, v: &Object, globals: &Rc>, @@ -6326,7 +6775,7 @@ impl Interpreter { // as `typing` aliases (`typing.List`, `int | typing.List`) and ABC // shims implemented as ordinary instances. if let Object::Instance(inst) = classinfo { - if let Some(hook) = inst.class.lookup("__instancecheck__") { + if let Some(hook) = inst.cls().lookup("__instancecheck__") { let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: classinfo.clone(), function: hook, @@ -6441,15 +6890,17 @@ impl Interpreter { if matches!(classinfo, Object::Type(_) | Object::None) || crate::is_pep604_union(classinfo).is_some() { - return Ok(Object::Bool(builtins::class_matches_classinfo( - cls_inner, classinfo, + return Ok(Object::Bool(builtins::class_matches_classinfo_named( + cls_inner, + classinfo, + "issubclass", )?)); } } // PEP 3119: a `__subclasscheck__` on `type(classinfo)` overrides the // default (class-like instances such as `typing` aliases / unions). if let Object::Instance(inst) = classinfo { - if let Some(hook) = inst.class.lookup("__subclasscheck__") { + if let Some(hook) = inst.cls().lookup("__subclasscheck__") { let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: classinfo.clone(), function: hook, @@ -6548,11 +6999,11 @@ impl Interpreter { globals: &Rc>, ) -> Result { if let Object::Instance(inst) = obj { - match inst.class.lookup("__hash__") { + match inst.cls().lookup("__hash__") { Some(Object::None) => { return Err(type_error(format!( "unhashable type: '{}'", - inst.class.name + inst.cls().name ))); } Some(method @ (Object::Function(_) | Object::BoundMethod(_))) => { @@ -6587,7 +7038,7 @@ impl Interpreter { return None; }; if !matches!( - inst.class.lookup("__hash__"), + inst.cls().lookup("__hash__"), Some(Object::Function(_) | Object::BoundMethod(_)) ) { return None; @@ -6619,9 +7070,9 @@ impl Interpreter { args: &[Object], _globals: &Rc>, ) -> Result { - let name = match args.get(1) { - Some(Object::Str(s)) => s.to_string(), - _ => return Err(type_error("attribute name must be string")), + let name = match attr_name_arg(args.get(1)) { + Some(n) => n, + None => return Err(type_error("attribute name must be string")), }; match self.load_attr(&args[0], &name) { Ok(v) => Ok(v), @@ -6639,9 +7090,9 @@ impl Interpreter { args: &[Object], _globals: &Rc>, ) -> Result { - let name = match args.get(1) { - Some(Object::Str(s)) => s.to_string(), - _ => return Err(type_error("attribute name must be string")), + let name = match attr_name_arg(args.get(1)) { + Some(n) => n, + None => return Err(type_error("attribute name must be string")), }; match self.load_attr(&args[0], &name) { Ok(_) => Ok(Object::Bool(true)), @@ -6930,6 +7381,41 @@ impl Interpreter { /// Run `__str__` on instances, falling back to `__repr__` then /// the default. Built-in types use their existing `to_str`. + /// CPython `PyObject_Str`/`PyObject_Repr` result check: a user + /// `__str__`/`__repr__` must return a `str` (subclasses included). + fn require_str_result(r: Object, dunder: &str) -> Result { + match &r { + Object::Str(_) => Ok(r.to_str()), + Object::Instance(inst) if matches!(&inst.native, Some(Object::Str(_))) => { + Ok(r.to_str()) + } + other => Err(type_error(format!( + "{dunder} returned non-string (type {})", + other.type_name() + ))), + } + } + + /// Crate-visible attribute load for builtins that need full + /// dispatch (e.g. weakproxy forwarding). + pub(crate) fn load_attr_public( + &mut self, + obj: &Object, + name: &str, + ) -> Result { + self.load_attr(obj, name) + } + + /// Crate-visible `str()` for builtins that need full dispatch + /// (e.g. `BaseException.__str__` rendering a nested exception arg). + pub(crate) fn stringify_public( + &mut self, + v: &Object, + globals: &Rc>, + ) -> Result { + self.stringify(v, globals) + } + fn stringify( &mut self, v: &Object, @@ -6938,7 +7424,7 @@ impl Interpreter { if let Object::Instance(inst) = v { if let Some(method) = instance_method(v, "__str__") { let r = self.call(&method, &[], &[], globals)?; - return Ok(r.to_str()); + return Self::require_str_result(r, "__str__"); } // A subclass of a built-in (`class S(str)`, `class F(float)`, …) // with no custom `__str__` inherits the base type's `__str__`, @@ -6972,7 +7458,7 @@ impl Interpreter { if let Object::Instance(inst) = v { if let Some(method) = instance_method(v, "__repr__") { let r = self.call(&method, &[], &[], globals)?; - return Ok(r.to_str()); + return Self::require_str_result(r, "__repr__"); } // Built-in subclass with no custom `__repr__` uses the base // type's `__repr__` on its native payload (e.g. `repr(F(2.5))` @@ -7190,37 +7676,119 @@ impl Interpreter { } /// Drive an awaitable into its underlying iterator (PEP 492 / - /// RFC 0016). A coroutine is itself awaitable; an async generator - /// is not (it must be consumed via `async for`). Any other object - /// is consulted via `__await__()`. - fn get_awaitable(&mut self, value: Object) -> Result { + /// RFC 0016). A coroutine is itself awaitable; a generator only if + /// its code is marked `CO_ITERABLE_COROUTINE` (`types.coroutine`); + /// an async generator is not (it must be consumed via `async + /// for`). Any other object is consulted via `__await__()`, whose + /// result must be an iterator that is not itself a coroutine. + /// `ctx` selects the CPython error wording: 0 = plain `await`, + /// 1 = `async for`'s `__anext__` result, 2 = `__aenter__`, + /// 3 = `__aexit__`. + fn get_awaitable(&mut self, value: Object, ctx: u32) -> Result { + let not_awaitable = |v: &Object| -> RuntimeError { + let t = v.type_name_owned(); + match ctx { + 1 => type_error(format!( + "'async for' received an invalid object from __anext__: {t}" + )), + 2 => type_error(format!( + "'async with' received an object from __aenter__ that does not implement \ + __await__: {t}" + )), + 3 => type_error(format!( + "'async with' received an object from __aexit__ that does not implement \ + __await__: {t}" + )), + _ => type_error(format!("object {t} can't be used in 'await' expression")), + } + }; match &value { + Object::Coroutine(g) => { + // CPython GET_AWAITABLE: a coroutine that is suspended + // inside its own `await` (it has a yield-from + // sub-iterator) cannot gain a second awaiter. + let busy = matches!(&*g.state.borrow(), GeneratorState::Suspended(boxed) + if boxed + .downcast_ref::() + .is_some_and(|f| detect_yield_from_subiter(f).is_some())); + if busy { + return Err(crate::error::runtime_error( + "coroutine is being awaited already", + )); + } + Ok(value) + } + // A generator function marked with `types.coroutine` produces + // an awaitable generator (CO_ITERABLE_COROUTINE). + Object::Generator(g) => { + let marked = matches!(&g.code, Object::Code(c) if c.is_iterable_coroutine); + if marked { + Ok(value) + } else { + Err(not_awaitable(&value)) + } + } // An async generator that surfaced through `__anext__` is - // already drivable via SEND; treat it as its own - // awaitable so the surrounding await-dance can run. - Object::Coroutine(_) | Object::Generator(_) | Object::AsyncGenerator(_) => Ok(value), + // already drivable via SEND; `await agen()` itself is an error. + Object::AsyncGenerator(_) => { + if ctx == 1 { + Ok(value) + } else { + Err(not_awaitable(&value)) + } + } // The deferred `asend`/`athrow`/`aclose` awaitable is already a // drivable awaitable (SEND applies the op via `step_agen_await`). Object::AsyncGenAwait(_) => Ok(value), Object::Instance(_) => { - if let Some(method) = instance_method(&value, "__await__") { + let Some(method) = instance_method(&value, "__await__") else { + return Err(not_awaitable(&value)); + }; + let result = (|| { let it = self.call(&method, &[], &[], &fallback_globals())?; - return Ok(it); + // CPython `_PyCoro_GetAwaitableIter`: the result must be + // a plain iterator — a coroutine is rejected outright, + // and anything without `__next__` is a non-iterator. + match &it { + Object::Coroutine(_) => { + Err(type_error("__await__() returned a coroutine")) + } + Object::Iter(_) | Object::Generator(_) => Ok(it), + Object::Instance(_) if instance_method(&it, "__next__").is_some() => { + Ok(it) + } + other => Err(type_error(format!( + "__await__() returned non-iterator of type '{}'", + other.type_name_owned() + ))), + } + })(); + match result { + // GET_ANEXT folds *any* conversion failure (a raising + // `__await__`, a bad result, …) into the invalid-object + // TypeError with the original exception as its cause + // (CPython's `_PyErr_FormatFromCause`). + Err(RuntimeError::PyException(inner)) if ctx == 1 => { + let RuntimeError::PyException(mut outer) = not_awaitable(&value) else { + unreachable!("not_awaitable builds a PyException"); + }; + outer.traceback = inner.traceback.clone(); + outer.cause = Some(Box::new(inner.clone())); + outer.context = Some(Box::new(inner)); + Self::sync_exc_attrs(&outer); + Err(RuntimeError::PyException(outer)) + } + other => other, } - Err(type_error(format!( - "object {} can't be used in 'await' expression", - value.type_name_owned() - ))) } - _ => Err(type_error(format!( - "object {} can't be used in 'await' expression", - value.type_name_owned() - ))), + _ => Err(not_awaitable(&value)), } } - /// `__aiter__` dispatch — `aiter()`. Async generators are - /// directly iterable; other objects must implement `__aiter__`. + /// `__aiter__` dispatch — `aiter()` / `async for`. Async + /// generators are directly iterable; other objects must implement + /// `__aiter__`, and the object it returns must be an async + /// iterator (CPython validates `__anext__` right in GET_AITER). fn get_aiter( &mut self, value: Object, @@ -7228,17 +7796,24 @@ impl Interpreter { ) -> Result { match &value { Object::AsyncGenerator(_) => Ok(value), - Object::Instance(_) => { - if let Some(method) = instance_method(&value, "__aiter__") { - return self.call(&method, &[], &[], globals); + Object::Instance(_) if instance_method(&value, "__aiter__").is_some() => { + let method = instance_method(&value, "__aiter__").expect("checked"); + let it = self.call(&method, &[], &[], globals)?; + let is_async_iter = matches!(it, Object::AsyncGenerator(_)) + || (matches!(it, Object::Instance(_)) + && instance_method(&it, "__anext__").is_some()); + if is_async_iter { + Ok(it) + } else { + Err(type_error(format!( + "'async for' received an object from __aiter__ that does not implement \ + __anext__: {}", + it.type_name_owned() + ))) } - Err(type_error(format!( - "'{}' object is not async-iterable", - value.type_name_owned() - ))) } _ => Err(type_error(format!( - "'{}' object is not async-iterable", + "'async for' requires an object with __aiter__ method, got {}", value.type_name_owned() ))), } @@ -7260,17 +7835,13 @@ impl Interpreter { // `StopAsyncIteration` for async generators. Ok(aiter.clone()) } - Object::Instance(_) => { - if let Some(method) = instance_method(aiter, "__anext__") { - return self.call(&method, &[], &[], globals); - } - Err(type_error(format!( - "'{}' object is not an async iterator", - aiter.type_name_owned() - ))) + Object::Instance(_) if instance_method(aiter, "__anext__").is_some() => { + let method = instance_method(aiter, "__anext__").expect("checked"); + self.call(&method, &[], &[], globals) } _ => Err(type_error(format!( - "'{}' object is not an async iterator", + "'async for' received an object from __aiter__ that does not implement \ + __anext__: {}", aiter.type_name_owned() ))), } @@ -7607,20 +8178,88 @@ impl Interpreter { Object::AsyncGenerator(g) => (g.clone(), true), _ => return Err(type_error("throw() requires a generator/coroutine")), }; - let exc_obj = args - .first() - .cloned() - .ok_or_else(|| type_error("throw() requires an exception argument"))?; - let instance = match &exc_obj { - Object::Type(t) => crate::builtin_types::make_exception_with_class(t.clone(), ""), - inst @ Object::Instance(_) => inst.clone(), + if args.is_empty() { + return Err(type_error("throw expected at least 1 argument, got 0")); + } + if args.len() > 3 { + return Err(type_error(format!( + "throw expected at most 3 arguments, got {}", + args.len() + ))); + } + if args.len() > 1 { + self.emit_deprecation_warning( + "the (type, exc, tb) signature of throw() is deprecated, \ + use the single-arg signature instead." + .to_owned(), + )?; + } + let typ = args[0].clone(); + let val = args.get(1).cloned().unwrap_or(Object::None); + // Validate the traceback argument first, as CPython does. + let tb = match args.get(2) { + None | Some(Object::None) => None, + Some(Object::Traceback(t)) => Some(t.clone()), + Some(_) => { + return Err(type_error( + "throw() third argument must be a traceback object", + )) + } + }; + let bt = crate::builtin_types::builtin_types(); + let instance = match &typ { + Object::Type(t) if t.is_subclass_of(&bt.base_exception) => { + // PyErr_NormalizeException: an instance of the class is + // used as-is; anything else becomes the call arguments. + let is_match = matches!( + &val, + Object::Instance(i) if i.cls().is_subclass_of(t) + ); + if is_match { + val + } else { + // CPython normalizes lazily, at the resume point: a + // failure here (raising __new__, non-exception result) + // is delivered *into* the generator frame, exhausting + // it, rather than raised before it runs. + match self.normalize_thrown_class(&typ, t, &val) { + Ok(built) => built, + Err(RuntimeError::PyException(exc)) => { + return match self.generator_throw(&g, exc) { + Err(RuntimeError::PyException(e)) + if is_async_gen && e.type_name() == "StopIteration" => + { + Err(stop_async_iteration()) + } + other => other, + } + } + Err(other) => return Err(other), + } + } + } + inst @ Object::Instance(i) if i.cls().is_subclass_of(&bt.base_exception) => { + if !matches!(val, Object::None) { + return Err(type_error( + "instance exception may not have a separate value", + )); + } + inst.clone() + } other => { return Err(type_error(format!( - "throw() argument must be an exception, got '{}'", + "exceptions must be classes or instances deriving from \ + BaseException, not {}", other.type_name() ))) } }; + if let (Some(t), Object::Instance(inst)) = (&tb, &instance) { + inst.dict.borrow_mut().insert( + DictKey(Object::from_static("__traceback__")), + Object::Traceback(t.clone()), + ); + } match self.generator_throw(&g, PyException::new(instance)) { Err(RuntimeError::PyException(exc)) if is_async_gen && exc.type_name() == "StopIteration" => @@ -7651,8 +8290,8 @@ impl Interpreter { return RuntimeError::PyException(exc); }; let bt = crate::builtin_types::builtin_types(); - let is_stop_iter = inst.class.is_subclass_of(&bt.stop_iteration); - let is_stop_async = inst.class.is_subclass_of(&bt.stop_async_iteration); + let is_stop_iter = inst.cls().is_subclass_of(&bt.stop_iteration); + let is_stop_async = inst.cls().is_subclass_of(&bt.stop_async_iteration); let msg = match gen.kind { CoroutineKind::Generator if is_stop_iter => "generator raised StopIteration", CoroutineKind::Coroutine if is_stop_iter => "coroutine raised StopIteration", @@ -7699,10 +8338,23 @@ impl Interpreter { .map_err(|_| RuntimeError::Internal("generator frame downcast".to_owned()))?, GeneratorState::Finished => { *gen.state.borrow_mut() = GeneratorState::Finished; + // bpo-25887: throwing into an exhausted coroutine is a + // RuntimeError (matching `send`), except for the + // GeneratorExit that `close()` injects. + if matches!(gen.kind, crate::object::CoroutineKind::Coroutine) + && exc.type_name() != "GeneratorExit" + { + return Err(crate::error::runtime_error( + "cannot reuse already awaited coroutine", + )); + } return Err(RuntimeError::PyException(exc)); } GeneratorState::Running => { - return Err(value_error("generator already executing")); + return Err(value_error(format!( + "{} already executing", + gen.kind.word() + ))); } }; // PEP 3134: an exception thrown into a generator suspended inside @@ -7716,7 +8368,21 @@ impl Interpreter { // executed instruction was YIELD_VALUE, the one before that // was SEND, and the stack top is an iterator-like. if let Some(sub_iter) = detect_yield_from_subiter(&frame) { - match self.throw_into_subiter(&sub_iter, exc.clone()) { + // Make the delegating frame visible on the Python call + // stack for the duration of the inner throw: CPython + // re-enters the outer generator frame, so code in the + // inner generator sees `f_back` chain through it + // (`check_stack_names` expects ['f', 'g']). + let pushed_outer = frame.py_frame.clone().map(|py| { + *py.back.borrow_mut() = self.frame_stack.borrow().last().cloned(); + self.frame_stack.borrow_mut().push(py.clone()); + py + }); + let inner_result = self.throw_into_subiter(&sub_iter, exc.clone()); + if pushed_outer.is_some() { + self.pop_py_frame(); + } + match inner_result { Ok(v) => { // Inner yielded: re-suspend the outer at the // same point and surface the new value. @@ -7813,12 +8479,30 @@ impl Interpreter { /// `finally` blocks run; we mirror that by routing through /// `generator_throw` and absorbing the resulting StopIteration. fn gen_method_close(&mut self, receiver: &Object) -> Result { - // Closing a deferred `asend`/`athrow` awaitable (CPython - // `async_gen_asend_close`): a not-yet-started awaitable just flips - // to CLOSED; a started one must deliver `GeneratorExit` to the - // suspended agen, and the agen answering with another suspension - // means it ignored the exit. - if let Object::AsyncGenAwait(a) = receiver { + // Snapshot finalizable locals before the frame is torn down: + // CPython's refcounting runs their `__del__` the moment + // `close()` drops the frame (gh-142766), and tests assert on + // that promptness. + let frame_caps = match receiver { + Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { + frame_finalizables(g) + } + _ => Vec::new(), + }; + let result = self.gen_method_close_inner(receiver); + for cap in frame_caps { + self.prompt_reap_dropped(cap); + } + result + } + + fn gen_method_close_inner(&mut self, receiver: &Object) -> Result { + // Closing a deferred `asend`/`athrow` awaitable (CPython + // `async_gen_asend_close`): a not-yet-started awaitable just flips + // to CLOSED; a started one must deliver `GeneratorExit` to the + // suspended agen, and the agen answering with another suspension + // means it ignored the exit. + if let Object::AsyncGenAwait(a) = receiver { let a = a.clone(); if a.consumed.get() { return Ok(Object::None); @@ -7832,7 +8516,7 @@ impl Interpreter { crate::builtin_types::make_exception_with_class(bt.generator_exit.clone(), ""); return match self.gen_method_throw(&a.agen, &[exc_inst]) { Ok(_yielded) => Err(crate::error::runtime_error( - "coroutine ignored GeneratorExit", + "async generator ignored GeneratorExit", )), Err(RuntimeError::PyException(exc)) if matches!( @@ -7860,11 +8544,32 @@ impl Interpreter { Ok(_yielded) => { // PEP 342: generator ignored GeneratorExit (yielded // a new value instead of allowing the exit to - // propagate). CPython raises RuntimeError here. + // propagate). CPython raises RuntimeError here, with + // a traceback pointing at the generator's frame (the + // unraisable hook asserts `exc_traceback is not None`). + let frame_obj = self.gen_py_frame(&g); *g.state.borrow_mut() = GeneratorState::Finished; - Err(crate::error::runtime_error( - "generator ignored GeneratorExit", - )) + let err = crate::error::runtime_error(format!( + "{} ignored GeneratorExit", + g.kind.word() + )); + if let RuntimeError::PyException(pyexc) = &err { + if let (Object::Instance(inst), Object::Frame(pf)) = + (&pyexc.instance, &frame_obj) + { + let tb = Rc::new(PyTraceback { + frame: pf.clone(), + lineno: pf.last_line.get().unwrap_or(1), + lasti: pf.lasti.get(), + next: RefCell::new(None), + }); + inst.dict.borrow_mut().insert( + DictKey(Object::from_static("__traceback__")), + Object::Traceback(tb), + ); + } + } + Err(err) } Err(RuntimeError::PyException(exc)) if exc.type_name() == "GeneratorExit" @@ -7960,6 +8665,85 @@ impl Interpreter { /// `sent` is the value pushed onto the frame's stack as the /// result of the prior `YIELD_VALUE`; for `__next__()` callers /// it's `None`. + /// Instantiate the exception class for `gen.throw(cls, val)` — + /// CPython's `PyErr_NormalizeException` step. `val` supplies the + /// constructor arguments (tuple → splatted, None → no args). + fn normalize_thrown_class( + &mut self, + typ: &Object, + t: &Rc, + val: &Object, + ) -> Result { + let bt = crate::builtin_types::builtin_types(); + let call_args: Vec = match val { + Object::None => vec![], + Object::Tuple(items) => items.to_vec(), + other => vec![other.clone()], + }; + let globals = self.builtins.clone(); + let built = self.call(typ, &call_args, &[], &globals)?; + let ok = matches!( + &built, + Object::Instance(i) if i.cls().is_subclass_of(&bt.base_exception) + ); + if !ok { + let typ_repr = self + .repr_of(typ, &globals) + .unwrap_or_else(|_| t.name.clone()); + let built_cls = Object::Type(crate::builtins::class_of(&built)); + let built_repr = self + .repr_of(&built_cls, &globals) + .unwrap_or_else(|_| built.type_name().to_string()); + return Err(type_error(format!( + "calling {typ_repr} should have returned an instance of \ + BaseException, not {built_repr}", + ))); + } + Ok(built) + } + + /// PEP 667 write-through: reconcile mutations made via a suspended + /// generator frame's `f_locals` dict back into the frame's locals + /// (and cells) before resuming execution. + fn apply_py_frame_locals_writes(frame: &mut Frame) { + let Some(py) = frame.py_frame.as_ref() else { + return; + }; + let cached = py.locals_cache.borrow().clone(); + let Some(Object::Dict(d)) = cached else { + return; + }; + { + let dict = d.borrow(); + let cell_names: Vec<&String> = frame + .code + .cellvars + .iter() + .chain(frame.code.freevars.iter()) + .collect(); + for (i, name) in frame.code.varnames.iter().enumerate() { + let Some(v) = dict.get(&DictKey(Object::from_str(name.clone()))) else { + continue; + }; + if let Some(ci) = cell_names.iter().position(|c| *c == name) { + if let Some(cell) = frame.cells.get(ci) { + *cell.borrow_mut() = v.clone(); + continue; + } + } + if let Some(slot) = frame.locals.get_mut(i) { + *slot = v.clone(); + } + } + } + if let Some(mirror) = py.locals_mirror.borrow().as_ref() { + *mirror.borrow_mut() = frame.locals.clone(); + } + // Refresh *after* the mirror is current so the in-place + // f_locals refresh sees the applied writes. + py.invalidate_locals(); + } + fn generator_send( &mut self, gen: &Rc, @@ -7982,19 +8766,33 @@ impl Interpreter { ), GeneratorState::Finished => { *gen.state.borrow_mut() = GeneratorState::Finished; + // bpo-25887: re-awaiting a completed coroutine is an + // error, not a silent StopIteration. + if matches!(gen.kind, crate::object::CoroutineKind::Coroutine) { + return Err(crate::error::runtime_error( + "cannot reuse already awaited coroutine", + )); + } return Err(stop_iteration()); } GeneratorState::Running => { - return Err(value_error("generator already executing")); + return Err(value_error(format!( + "{} already executing", + gen.kind.word() + ))); } }; // On the first call, `sent` must be None (or omitted). if first_resume && !matches!(sent, Object::None) { *gen.state.borrow_mut() = GeneratorState::Suspended(Box::new(frame)); - return Err(type_error( - "can't send non-None value to a just-started generator", - )); + return Err(type_error(format!( + "can't send non-None value to a just-started {}", + gen.kind.word() + ))); } + // PEP 667: writes made through the suspended frame's `f_locals` + // take effect when the generator resumes. + Self::apply_py_frame_locals_writes(&mut frame); let sent_for_frame = if first_resume { None } else { Some(sent) }; match self.run_until_yield_or_return(&mut frame, sent_for_frame) { Ok(FrameOutcome::Yielded(v)) => { @@ -8042,7 +8840,7 @@ impl Interpreter { _ => return Err(type_error("called match pattern must be a type")), }; let is_inst = match subject { - Object::Instance(inst) => inst.class.is_subclass_of(&ty), + Object::Instance(inst) => inst.cls().is_subclass_of(&ty), _ => { // Built-in mapping: roughly match by type_name. let bt = builtin_types(); @@ -8321,6 +9119,52 @@ impl Interpreter { }) } + /// Resolve a rich-comparison dunder on `obj`'s class the way CPython's + /// `slot_tp_richcompare`/`lookup_maybe_method` does: honour the + /// descriptor protocol for non-function class attributes (a `property` + /// used as `__eq__` is read through its getter), and *clear* any error + /// the descriptor raises — a failing lookup means "comparison not + /// supported here" (`NotImplemented`), not an exception. + fn cmp_method( + &mut self, + obj: &Object, + name: &str, + globals: &Rc>, + ) -> Option { + let inst = match obj { + Object::Instance(i) => i.clone(), + _ => return None, + }; + let m = inst.cls().lookup(name)?; + match &m { + Object::Property(p) => { + let fget = p.fget.clone(); + if matches!(fget, Object::None) { + return None; + } + self.call(&fget, std::slice::from_ref(obj), &[], globals).ok() + } + Object::Instance(desc) if desc.cls().lookup("__get__").is_some() => { + let getter = desc.cls().lookup("__get__")?; + let bound = Object::BoundMethod(Rc::new(BoundMethod { + receiver: m.clone(), + function: getter, + })); + self.call( + &bound, + &[obj.clone(), Object::Type(inst.cls())], + &[], + globals, + ) + .ok() + } + _ => Some(Object::BoundMethod(Rc::new(BoundMethod { + receiver: Object::Instance(inst), + function: m, + }))), + } + } + fn dispatch_compare_op( &mut self, a: &Object, @@ -8336,13 +9180,13 @@ impl Interpreter { // and only if *both* decline fall through to the native default // (identity for ==/!=, `TypeError` for an ordering). let not_impl = crate::vm_singletons::not_implemented(); - if let Some(method) = instance_method(a, dunder) { + if let Some(method) = self.cmp_method(a, dunder, globals) { let r = self.call(&method, std::slice::from_ref(b), &[], globals)?; if !r.is_same(¬_impl) { return Ok(r.is_truthy()); } } - if let Some(method) = instance_method(b, swapped) { + if let Some(method) = self.cmp_method(b, swapped, globals) { let r = self.call(&method, std::slice::from_ref(a), &[], globals)?; if !r.is_same(¬_impl) { return Ok(r.is_truthy()); @@ -8353,13 +9197,13 @@ impl Interpreter { // operand supplied a usable `__ne__` above, derive the result from // `__eq__` (forward then reflected) before falling back to identity. if matches!(op, CompareKind::NotEq) { - if let Some(method) = instance_method(a, "__eq__") { + if let Some(method) = self.cmp_method(a, "__eq__", globals) { let r = self.call(&method, std::slice::from_ref(b), &[], globals)?; if !r.is_same(¬_impl) { return Ok(!r.is_truthy()); } } - if let Some(method) = instance_method(b, "__eq__") { + if let Some(method) = self.cmp_method(b, "__eq__", globals) { let r = self.call(&method, std::slice::from_ref(a), &[], globals)?; if !r.is_same(¬_impl) { return Ok(!r.is_truthy()); @@ -8758,7 +9602,7 @@ impl Interpreter { IC::LoadAttrInstance { type_id, key_idx } => { let receiver = frame.top()?.clone(); if let Object::Instance(inst) = &receiver { - if specialize::rc_id(&inst.class) == type_id { + if specialize::rc_id(&inst.cls()) == type_id { let dict = inst.dict.borrow(); if let Some((k, v)) = dict.get_index(key_idx as usize) { if self.cached_slot_name_matches(&frame.code, name_idx, k) { @@ -8794,8 +9638,9 @@ impl Interpreter { IC::LoadAttrType { type_id, key_idx } => { let receiver = frame.top()?.clone(); if let Object::Instance(inst) = &receiver { - if specialize::rc_id(&inst.class) == type_id { - let dict = inst.class.dict.borrow(); + let cls = inst.cls(); + if specialize::rc_id(&cls) == type_id { + let dict = cls.dict.borrow(); if let Some((k, v)) = dict.get_index(key_idx as usize) { if self.cached_slot_name_matches(&frame.code, name_idx, k) { let v = v.clone(); @@ -8897,7 +9742,7 @@ impl Interpreter { IC::StoreAttrInstance { type_id, key_idx } => { let receiver = frame.top()?.clone(); if let Object::Instance(inst) = &receiver { - if specialize::rc_id(&inst.class) == type_id { + if specialize::rc_id(&inst.cls()) == type_id { let dict_len = inst.dict.borrow().len(); if dict_len > key_idx as usize { frame.pop()?; @@ -9473,11 +10318,48 @@ impl Interpreter { "{name} must be set to a string object" ))), }, - _ => Err(attribute_error(format!( - "'{}' object has no attribute '{}'", - obj.type_name(), - name - ))), + _ => { + // Known getset/member attrs raise "not writable", + // anything else "has no attribute" (CPython getset + // without setter vs. missing tp_setattro entry). + let readonly: &[&str] = match obj { + Object::Generator(_) => &[ + "gi_running", + "gi_frame", + "gi_code", + "gi_yieldfrom", + "gi_suspended", + ], + Object::Coroutine(_) => &[ + "cr_running", + "cr_frame", + "cr_code", + "cr_await", + "cr_origin", + "cr_suspended", + ], + _ => &[ + "ag_running", + "ag_frame", + "ag_code", + "ag_await", + "ag_suspended", + ], + }; + if readonly.contains(&name) { + Err(attribute_error(format!( + "attribute '{}' of '{}' objects is not writable", + name, + obj.type_name() + ))) + } else { + Err(attribute_error(format!( + "'{}' object has no attribute '{}'", + obj.type_name(), + name + ))) + } + } } } _ => Err(type_error(format!( @@ -9502,9 +10384,9 @@ impl Interpreter { ) -> Result<(), RuntimeError> { // User-defined __setattr__ on the class overrides everything. // We only honour Python-level overrides; the builtin default - // (`object.__setattr__`) falls through to direct dict writes + // (`object.__setattr__`) falls through to the generic store // below to keep the fast path inlineable. - if let Some(setattr) = inst.class.lookup("__setattr__") { + if let Some(setattr) = inst.cls().lookup("__setattr__") { if matches!( setattr, Object::Function(_) | Object::BoundMethod(_) | Object::Instance(_) @@ -9518,12 +10400,60 @@ impl Interpreter { return Ok(()); } } + self.generic_setattr_instance(inst, obj, name, value) + } + + /// CPython `PyObject_GenericSetAttr` — the `object.__setattr__` + /// slot body: honours data descriptors, `__class__`/`__dict__` + /// special handling and `__slots__` enforcement, but does *not* + /// re-dispatch a user `__setattr__` (it *is* the default that + /// overrides chain up to via `super().__setattr__`). + pub(crate) fn generic_setattr_instance( + &mut self, + inst: &Rc, + obj: &Object, + name: &str, + value: Object, + ) -> Result<(), RuntimeError> { + // `obj.__class__ = C` (CPython `object_set_class`): re-point the + // instance at a layout-compatible class. Both classes must be + // heap (user) types with the same solid base, dict-ness and + // slot layout. + if name == "__class__" { + let Object::Type(new_cls) = &value else { + return Err(type_error(format!( + "__class__ must be set to a class, not '{}' object", + value.type_name() + ))); + }; + let old_cls = inst.cls(); + if Rc::ptr_eq(&old_cls, new_cls) { + return Ok(()); + } + if old_cls.flags.is_builtin || new_cls.flags.is_builtin { + return Err(type_error( + "__class__ assignment only supported for mutable types \ + or ModuleType subclasses", + )); + } + let compatible = old_cls.solid_base_name() == new_cls.solid_base_name() + && old_cls.forbids_dict == new_cls.forbids_dict + && *old_cls.slot_names.borrow() == *new_cls.slot_names.borrow(); + if !compatible { + return Err(type_error(format!( + "__class__ assignment: '{}' object layout differs from '{}'", + new_cls.name, old_cls.name + ))); + } + inst.set_cls(new_cls.clone()); + return Ok(()); + } // `obj.__dict__ = d` (CPython's `__dict__` getset descriptor): // replace the instance dict's contents wholesale. Inline-values // state is permanently cleared. Divergence: CPython aliases `d` // itself as the instance dict; our `Rc` field can't be swapped, // so we copy `d`'s contents instead. - if name == "__dict__" && !inst.class.forbids_dict { + if name == "__dict__" && !inst.cls().forbids_dict { let Object::Dict(src) = &value else { return Err(type_error(format!( "__dict__ must be set to a dictionary, not a '{}'", @@ -9535,13 +10465,13 @@ impl Interpreter { inst.inline_values.set(false); return Ok(()); } - if let Some(attr) = inst.class.lookup(name) { + if let Some(attr) = inst.cls().lookup(name) { match &attr { Object::Property(prop) => { if matches!(prop.fset, Object::None) { return Err(attribute_error(format!( "property '{}' of '{}' object has no setter", - name, inst.class.name + name, inst.cls().name ))); } let setter = prop.fset.clone(); @@ -9555,7 +10485,7 @@ impl Interpreter { return Ok(()); } Object::Instance(descriptor_inst) => { - if let Some(setter) = descriptor_inst.class.lookup("__set__") { + if let Some(setter) = descriptor_inst.cls().lookup("__set__") { let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: attr.clone(), function: setter, @@ -9567,12 +10497,23 @@ impl Interpreter { _ => {} } } - if inst.class.forbids_dict { - let slots = inst.class.slot_names.borrow(); + if inst.cls().forbids_dict { + let cls = inst.cls(); + let slots = cls.slot_names.borrow(); if !slots.iter().any(|s| s == name) { + // CPython `_PyObject_GenericSetAttrWithDict`: a name that + // resolves on the class (but isn't a data descriptor) is + // "read-only"; an unknown name notes the missing + // `__dict__`. + if inst.cls().lookup(name).is_some() { + return Err(attribute_error(format!( + "'{}' object attribute '{}' is read-only", + inst.cls().name, name + ))); + } return Err(attribute_error(format!( - "'{}' object has no attribute '{}'", - inst.class.name, name + "'{}' object has no attribute '{}' and no __dict__ for setting new attributes", + inst.cls().name, name ))); } } @@ -9585,7 +10526,7 @@ impl Interpreter { fn delete_attr(&mut self, obj: &Object, name: &str) -> Result<(), RuntimeError> { match obj { Object::Instance(inst) => { - if let Some(delattr) = inst.class.lookup("__delattr__") { + if let Some(delattr) = inst.cls().lookup("__delattr__") { if matches!( delattr, Object::Function(_) | Object::BoundMethod(_) | Object::Instance(_) @@ -9599,63 +10540,7 @@ impl Interpreter { return Ok(()); } } - // `del obj.__dict__` (CPython's `__dict__` getset - // descriptor): detach the instance dict. The values are - // dropped and the instance reverts to an empty dict; the - // inline-values state is permanently cleared. - if name == "__dict__" && !inst.class.forbids_dict { - inst.dict.borrow_mut().clear(); - inst.inline_values.set(false); - return Ok(()); - } - if let Some(attr) = inst.class.lookup(name) { - match &attr { - Object::Property(prop) => { - if matches!(prop.fdel, Object::None) { - return Err(attribute_error(format!( - "property '{}' of '{}' object has no deleter", - name, inst.class.name - ))); - } - let deleter = prop.fdel.clone(); - self.call( - &deleter, - std::slice::from_ref(obj), - &[], - &self.builtins.clone(), - )?; - return Ok(()); - } - Object::Instance(descriptor_inst) => { - if let Some(deleter) = descriptor_inst.class.lookup("__delete__") { - let bound = Object::BoundMethod(Rc::new(BoundMethod { - receiver: attr.clone(), - function: deleter, - })); - self.call( - &bound, - std::slice::from_ref(obj), - &[], - &self.builtins.clone(), - )?; - return Ok(()); - } - } - _ => {} - } - } - if inst - .dict - .borrow_mut() - .shift_remove(&DictKey(Object::from_str(name))) - .is_none() - { - return Err(attribute_error(format!( - "'{}' object has no attribute '{}'", - inst.class.name, name - ))); - } - Ok(()) + self.generic_delattr_instance(inst, obj, name) } Object::Type(ty) => { // A user metaclass `__delattr__` intercepts class-attribute @@ -9699,6 +10584,86 @@ impl Interpreter { } } + /// CPython `PyObject_GenericSetAttr`'s delete half — the + /// `object.__delattr__` slot body: data descriptors, `__dict__` + /// detach and `__slots__` messages, without re-dispatching a user + /// `__delattr__`. + pub(crate) fn generic_delattr_instance( + &mut self, + inst: &Rc, + obj: &Object, + name: &str, + ) -> Result<(), RuntimeError> { + // `del obj.__dict__` (CPython's `__dict__` getset + // descriptor): detach the instance dict. The values are + // dropped and the instance reverts to an empty dict; the + // inline-values state is permanently cleared. + if name == "__dict__" && !inst.cls().forbids_dict { + inst.dict.borrow_mut().clear(); + inst.inline_values.set(false); + return Ok(()); + } + if let Some(attr) = inst.cls().lookup(name) { + match &attr { + Object::Property(prop) => { + if matches!(prop.fdel, Object::None) { + return Err(attribute_error(format!( + "property '{}' of '{}' object has no deleter", + name, + inst.cls().name + ))); + } + let deleter = prop.fdel.clone(); + self.call( + &deleter, + std::slice::from_ref(obj), + &[], + &self.builtins.clone(), + )?; + return Ok(()); + } + Object::Instance(descriptor_inst) => { + if let Some(deleter) = descriptor_inst.cls().lookup("__delete__") { + let bound = Object::BoundMethod(Rc::new(BoundMethod { + receiver: attr.clone(), + function: deleter, + })); + self.call( + &bound, + std::slice::from_ref(obj), + &[], + &self.builtins.clone(), + )?; + return Ok(()); + } + } + _ => {} + } + } + if inst + .dict + .borrow_mut() + .shift_remove(&DictKey(Object::from_str(name))) + .is_none() + { + // Slots class: a name resolving on the class (non-data + // descriptor) is "read-only", like the store path. + if inst.cls().forbids_dict && inst.cls().lookup(name).is_some() { + return Err(attribute_error(format!( + "'{}' object attribute '{}' is read-only", + inst.cls().name, + name + ))); + } + return Err(attribute_error(format!( + "'{}' object has no attribute '{}'", + inst.cls().name, + name + ))); + } + Ok(()) + } + /// Coerce a path-like argument to `str`/`bytes` via `__fspath__`, for the /// Rust path builtins (`open`, `os.fspath`/`fsdecode`/`fsencode`) which /// can't call back into the interpreter themselves. `str`/`bytes` pass @@ -10065,12 +11030,18 @@ impl Interpreter { })?; let rest: &[Object] = if args.is_empty() { &[] } else { &args[1..] }; return match b.name { - ".u.gen_send" => self + ".u.gen_send" | ".u.cor_send" => self .gen_method_send(&receiver, rest.first().cloned().unwrap_or(Object::None)), - ".u.gen_throw" => self.gen_method_throw(&receiver, rest), - ".u.gen_close" => self.gen_method_close(&receiver), + ".u.gen_throw" | ".u.cor_throw" => self.gen_method_throw(&receiver, rest), + ".u.gen_close" | ".u.cor_close" => self.gen_method_close(&receiver), ".u.gen_next" => self.gen_method_send(&receiver, Object::None), - ".u.gen_iter" => Ok(receiver.clone()), + ".u.gen_iter" => { + if matches!(receiver, Object::Coroutine(_)) { + Ok(make_coroutine_wrapper(&receiver)) + } else { + Ok(receiver.clone()) + } + } ".u.agen_aiter" => Ok(receiver.clone()), ".u.agen_anext" => match &receiver { Object::AsyncGenerator(_) => Ok(make_agen_await( @@ -10366,10 +11337,12 @@ impl Interpreter { return self.do_compile_call(args, outer_globals); } if b.name == "__vm:exec" { - return self.do_exec_call(args, outer_globals); + let merged = Self::merge_exec_kwargs("exec", args, kwargs)?; + return self.do_exec_call(&merged, outer_globals); } if b.name == "__vm:eval" { - return self.do_eval_call(args, outer_globals); + let merged = Self::merge_exec_kwargs("eval", args, kwargs)?; + return self.do_eval_call(&merged, outer_globals); } // RFC 0024: `gc.collect()` is a Rust BuiltinFn that // queues `__del__` finalizers but can't run them @@ -10616,20 +11589,26 @@ impl Interpreter { "__subclasses__() requires a type receiver", )); } - ".gen_send" => { + ".gen_send" | ".cor_send" => { let value = args.first().cloned().unwrap_or(Object::None); return self.gen_method_send(&bm.receiver, value); } - ".gen_throw" => { + ".gen_throw" | ".cor_throw" => { return self.gen_method_throw(&bm.receiver, args); } - ".gen_close" => { + ".gen_close" | ".cor_close" => { return self.gen_method_close(&bm.receiver); } ".gen_next" => { return self.gen_method_send(&bm.receiver, Object::None); } ".gen_iter" => { + // `coro.__await__()` returns a distinct + // `coroutine_wrapper` iterator in CPython; + // `gen.__iter__()` is the generator itself. + if matches!(bm.receiver, Object::Coroutine(_)) { + return Ok(make_coroutine_wrapper(&bm.receiver)); + } return Ok(bm.receiver.clone()); } // --- async generator methods --------------------- @@ -10756,54 +11735,10 @@ impl Interpreter { self.call(&bm.function, &combined, kwargs, outer_globals) } Object::Type(ty) => { - // CPython routes `str(x)` / `repr(x)` through dunders; - // intercept the built-in classes here so that the - // user's `__str__` / `__repr__` wins over the default - // type constructor. - if ty.flags.is_builtin && args.len() == 1 && kwargs.is_empty() { - if ty.name == "str" { - return self.do_str_call(&args[0], outer_globals); - } - if ty.name == "repr" { - return self.do_repr_call(&args[0], outer_globals); - } - if ty.name == "memoryview" { - // `memoryview(x)` reaches here (the type is the - // callable). Native bytes-like inputs fall through to - // the normal constructor; anything else is taken - // through the PEP 688 buffer protocol (`__buffer__`) - // so `memoryview(array('b', …))` yields a real view - // over the array's exported buffer - // (test_struct.test_pack_into / test_unpack_with_buffer). - match &args[0] { - Object::Bytes(_) | Object::ByteArray(_) | Object::MemoryView(_) => {} - other => { - if let Some(method) = instance_method(other, "__buffer__") { - let view = self.call( - &method, - &[Object::Int(0)], - &[], - outer_globals, - )?; - if matches!(view, Object::MemoryView(_)) { - return Ok(view); - } - return builtins::b_memoryview(std::slice::from_ref(&view)); - } - } - } - } - } - // `type(name, bases, ns)` / `Meta(name, bases, ns)` build a - // new class dynamically — through the winner-metaclass - // delegation that CPython's `type_new` performs. - let bt = builtin_types(); - if ty.is_subclass_of(&bt.type_) && args.len() == 3 { - return self.winner_aware_dynamic_type_call(ty.clone(), args, kwargs); - } // If the class's *metaclass* overrides `__call__`, // dispatch through it so EnumMeta etc. can hook // calls like `Color(3)`. + let bt = builtin_types(); let meta = ty.metaclass_or_type(); if !Rc::ptr_eq(&meta, &bt.type_) { if let Some(call_method) = meta.lookup("__call__") { @@ -10814,11 +11749,25 @@ impl Interpreter { return self.call(&bound, args, kwargs, outer_globals); } } - self.instantiate(ty.clone(), args, kwargs) + self.type_call_default(ty, args, kwargs) } Object::Instance(inst) => { - // Honour __call__ if defined. - if let Some(m) = inst.class.lookup("__call__") { + // Honour __call__ if defined. The `__call__` chase is a + // native recursion that never enters a Python frame when + // the resolved target is itself a callable instance + // (`A.__call__ = A()`), so it must participate in the + // recursion limit the way CPython's `tp_call` does via + // `Py_EnterRecursiveCall` — otherwise the loop exhausts + // the native stack/heap instead of raising. + let _guard = match crate::recursion::enter() { + crate::recursion::Enter::Ok(g) => g, + crate::recursion::Enter::Overflow => { + return Err(recursion_error( + "maximum recursion depth exceeded while calling a Python object", + )) + } + }; + if let Some(m) = inst.cls().lookup("__call__") { let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: Object::Instance(inst.clone()), function: m, @@ -10827,7 +11776,7 @@ impl Interpreter { } else { Err(type_error(format!( "'{}' object is not callable", - inst.class.name + inst.cls().name ))) } } @@ -10838,6 +11787,169 @@ impl Interpreter { } } + /// CPython's `type.__call__` (`type_call`) — the *default* class-call + /// behaviour, reached after any metaclass `__call__` override has had + /// its chance (or invoked explicitly as `type.__call__(cls, …)`, which + /// must NOT re-dispatch through the metaclass — `type.__call__(cls)` + /// inside a metaclass `__call__` would otherwise recurse forever). + pub(crate) fn type_call_default( + &mut self, + ty: &Rc, + args: &[Object], + kwargs: &[(String, Object)], + ) -> Result { + // CPython routes `str(x)` / `repr(x)` through dunders; + // intercept the built-in classes here so that the + // user's `__str__` / `__repr__` wins over the default + // type constructor. + if ty.flags.is_builtin && args.len() == 1 && kwargs.is_empty() { + let globals = self.builtins.clone(); + if ty.name == "str" { + return self.do_str_call(&args[0], &globals); + } + if ty.name == "repr" { + return self.do_repr_call(&args[0], &globals); + } + if ty.name == "memoryview" { + // `memoryview(x)` reaches here (the type is the + // callable). Native bytes-like inputs fall through to + // the normal constructor; anything else is taken + // through the PEP 688 buffer protocol (`__buffer__`) + // so `memoryview(array('b', …))` yields a real view + // over the array's exported buffer + // (test_struct.test_pack_into / test_unpack_with_buffer). + match &args[0] { + Object::Bytes(_) | Object::ByteArray(_) | Object::MemoryView(_) => {} + other => { + if let Some(method) = instance_method(other, "__buffer__") { + let view = self.call(&method, &[Object::Int(0)], &[], &globals)?; + if matches!(view, Object::MemoryView(_)) { + return Ok(view); + } + return builtins::b_memoryview(std::slice::from_ref(&view)); + } + } + } + } + } + // `type(name, bases, ns)` / `Meta(name, bases, ns)` build a + // new class dynamically — through the winner-metaclass + // delegation that CPython's `type_new` performs. + let bt = builtin_types(); + if ty.is_subclass_of(&bt.type_) && args.len() == 3 { + return self.winner_aware_dynamic_type_call(ty.clone(), args, kwargs); + } + // `types.FunctionType(code, globals[, name[, argdefs[, closure]]])`. + if Rc::ptr_eq(ty, &bt.function_) { + return Self::function_type_call(args, kwargs); + } + self.instantiate(ty.clone(), args, kwargs) + } + + /// Construct a function object from a code object — CPython's + /// `func_new_impl`. Keyword forms mirror the positional ones. + fn function_type_call( + args: &[Object], + kwargs: &[(String, Object)], + ) -> Result { + let mut args = args.to_vec(); + for (k, v) in kwargs { + let pos = match k.as_str() { + "code" => 0, + "globals" => 1, + "name" => 2, + "argdefs" => 3, + "closure" => 4, + other => { + return Err(type_error(format!( + "function() got an unexpected keyword argument '{other}'" + ))) + } + }; + if args.len() > pos { + return Err(type_error(format!( + "function() got multiple values for argument '{k}'" + ))); + } + while args.len() < pos { + args.push(Object::None); + } + args.push(v.clone()); + } + let code = match args.first() { + Some(Object::Code(c)) => c.clone(), + Some(other) => { + return Err(type_error(format!( + "function() argument 'code' must be code, not {}", + other.type_name() + ))) + } + None => return Err(type_error("function() missing required argument 'code'")), + }; + let globals = match args.get(1) { + Some(Object::Dict(d)) => d.clone(), + Some(other) => { + return Err(type_error(format!( + "function() argument 'globals' must be dict, not {}", + other.type_name() + ))) + } + None => { + return Err(type_error( + "function() missing required argument 'globals'", + )) + } + }; + let name = match args.get(2) { + Some(Object::Str(s)) => s.to_string(), + Some(Object::None) | None => code.name.clone(), + Some(other) => { + return Err(type_error(format!( + "function() argument 'name' must be str or None, not {}", + other.type_name() + ))) + } + }; + let defaults = match args.get(3) { + Some(Object::Tuple(items)) => items.to_vec(), + Some(Object::None) | None => vec![], + Some(other) => { + return Err(type_error(format!( + "function() argument 'argdefs' must be tuple or None, not {}", + other.type_name() + ))) + } + }; + let closure = match args.get(4) { + Some(Object::Tuple(items)) => items.to_vec(), + Some(Object::None) | None => vec![], + Some(other) => { + return Err(type_error(format!( + "function() argument 'closure' must be tuple or None, not {}", + other.type_name() + ))) + } + }; + if closure.len() != code.freevars.len() { + return Err(type_error(format!( + "function() requires a code object with {} free vars, not {}", + closure.len(), + code.freevars.len(), + ))); + } + // Seed `__builtins__` so the new function's frames resolve + // builtins even with a bare `{}` globals dict. + Ok(Object::Function(Rc::new(crate::object::PyFunction { + name, + code, + globals, + defaults, + kw_defaults: vec![], + closure, + attrs: Rc::new(RefCell::new(DictData::new())), + }))) + } + /// PEP 3115: store `name = value` into a custom class namespace by /// dispatching through its `__setitem__` (so the mapping observes the /// binding — `enum._EnumDict` records members this way). Falls back to @@ -11010,6 +12122,76 @@ impl Interpreter { } } + // Pull `metaclass=` out of kwargs; the rest are passed to + // `__init_subclass__` (matching CPython's PEP 487 rules). The + // keyword may be *any* object — a non-type callable is simply + // called with `(name, bases, ns)` (CPython `__build_class__`). + let mut metaclass_obj: Option = None; + let mut subclass_kwargs: Vec<(String, Object)> = Vec::new(); + for (k, v) in kwargs { + if k == "metaclass" { + metaclass_obj = Some(v.clone()); + } else { + subclass_kwargs.push((k.clone(), v.clone())); + } + } + + // CPython computes the metaclass winner over `type(base)` of the + // *raw* bases — before any base validation, which belongs to + // `type.__new__`. A non-type base (say an int) contributes its + // class; an explicit metaclass that isn't a subclass of `type` + // (e.g. an `int` subclass, or a plain function) wins outright + // and is later called generically. + let explicit_meta_type = match &metaclass_obj { + Some(Object::Type(t)) => Some(t.clone()), + Some(other) => { + // Non-type metaclass: no winner computation, call it + // with the raw bases after the body runs. + return self.build_class_with_callable_meta( + other.clone(), + &body_fn, + &name, + &resolved_bases, + &subclass_kwargs, + ); + } + None => None, + }; + { + let bt = builtin_types(); + let mut winner: Rc = + explicit_meta_type.clone().unwrap_or_else(|| bt.type_.clone()); + for b in &resolved_bases { + let m = match b { + Object::Type(t) => t.metaclass_or_type(), + other => crate::builtins::class_of(other), + }; + if winner.is_subclass_of(&m) { + continue; + } + if m.is_subclass_of(&winner) { + winner = m; + continue; + } + return Err(type_error( + "metaclass conflict: the metaclass of a derived class must be a \ + (non-strict) subclass of the metaclasses of all its bases", + )); + } + // A winner that can't construct classes (not a `type` + // subclass) is called generically, like CPython's + // `meta(name, bases, ns, **kwds)` with a non-type meta. + if !winner.is_subclass_of(&bt.type_) { + return self.build_class_with_callable_meta( + Object::Type(winner), + &body_fn, + &name, + &resolved_bases, + &subclass_kwargs, + ); + } + } + let mut bases: Vec> = Vec::new(); for b in &resolved_bases { match b { @@ -11034,26 +12216,10 @@ impl Interpreter { bases.push(builtin_types().object_.clone()); } - // Pull `metaclass=` out of kwargs; the rest are passed to - // `__init_subclass__` (matching CPython's PEP 487 rules). - let mut metaclass_arg: Option> = None; - let mut subclass_kwargs: Vec<(String, Object)> = Vec::new(); - for (k, v) in kwargs { - if k == "metaclass" { - if let Object::Type(t) = v { - metaclass_arg = Some(t.clone()); - } else { - return Err(type_error("metaclass= must be a type")); - } - } else { - subclass_kwargs.push((k.clone(), v.clone())); - } - } - // Determine the effective metaclass: explicit `metaclass=` // beats anything inherited; otherwise pick the most-derived // metaclass of any base. - let metaclass = resolve_metaclass(metaclass_arg, &bases)?; + let metaclass = resolve_metaclass(explicit_meta_type, &bases)?; // PEP 3115: a metaclass may supply a custom namespace mapping via // `__prepare__`. When it does, the class body's name bindings flow @@ -11320,6 +12486,69 @@ impl Interpreter { Ok(Object::Type(ty)) } + /// `__build_class__` tail for a metaclass that is not a subclass of + /// `type` (a non-type callable, or a type like `class Meta(int)`). + /// CPython skips all base validation and class construction + /// machinery in this case: the body runs in a plain namespace and + /// the metaclass is simply called with `(name, bases, ns, **kwds)`; + /// whatever it returns *is* the "class". + fn build_class_with_callable_meta( + &mut self, + meta: Object, + body_fn: &Rc, + name: &str, + bases: &[Object], + subclass_kwargs: &[(String, Object)], + ) -> Result { + let class_ns = Rc::new(RefCell::new(DictData::new())); + { + let mut ns = class_ns.borrow_mut(); + ns.insert( + DictKey(Object::from_static("__name__")), + Object::from_str(name), + ); + ns.insert( + DictKey(Object::from_static("__qualname__")), + Object::from_str(name), + ); + if let Some(m) = body_fn + .globals + .borrow() + .get(&DictKey(Object::from_static("__name__"))) + .cloned() + { + ns.insert(DictKey(Object::from_static("__module__")), m); + } + } + let code = body_fn.code.clone(); + let mut frame = self.make_frame( + code, + Vec::new(), + body_fn.closure.clone(), + body_fn.globals.clone(), + false, + ); + frame.class_namespace = Some(class_ns.clone()); + let _ = self.run_frame(&mut frame)?; + + let call_args = vec![ + Object::from_str(name), + Object::new_tuple(bases.to_vec()), + Object::Dict(class_ns), + ]; + let result = self.call(&meta, &call_args, subclass_kwargs, &body_fn.globals)?; + // PEP 3135: point any `__class__` cell at whatever the meta + // returned, so zero-arg `super()` in methods doesn't dangle. + for (i, cell_name) in body_fn.code.cellvars.iter().enumerate() { + if cell_name == "__class__" { + if let Some(cell) = frame.cells.get(i) { + *cell.borrow_mut() = result.clone(); + } + } + } + Ok(result) + } + /// Dynamic 3-arg class construction (`type(name, bases, ns)` or /// `Meta(name, bases, ns)`) with CPython `type_new`'s winner rule: /// the build is owned by the most-derived metaclass among the seed @@ -11633,7 +12862,7 @@ impl Interpreter { .collect(); for (attr_name, value) in entries { if let Object::Instance(inst) = &value { - if let Some(hook) = inst.class.lookup("__set_name__") { + if let Some(hook) = inst.cls().lookup("__set_name__") { let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: value.clone(), function: hook, @@ -11651,7 +12880,7 @@ impl Interpreter { if let Err(RuntimeError::PyException(pe)) = &res { pe.add_note(format!( "Error calling __set_name__ on '{}' instance '{}' in '{}'", - inst.class.name, attr_name, ty.name + inst.cls().name, attr_name, ty.name )); } res?; @@ -11812,14 +13041,33 @@ impl Interpreter { // classmethod) — route to dedicated constructors. match cls.name.as_str() { "property" => { + // Bind CPython's keyword form (`fget=`, `fset=`, + // `fdel=`, `doc=`) onto the positional layout. + let mut bound: Vec = args.to_vec(); if !kwargs.is_empty() { - // CPython accepts `fget=`, `fset=`, etc., - // but we keep the keyword form simple here. - return Err(type_error( - "property() takes positional arguments only here", - )); + bound.resize(4, Object::None); + for (k, v) in kwargs { + let idx = match k.as_str() { + "fget" => 0, + "fset" => 1, + "fdel" => 2, + "doc" => 3, + other => { + return Err(type_error(format!( + "property() got an unexpected keyword argument '{other}'" + ))) + } + }; + if idx < args.len() { + return Err(type_error(format!( + "argument for property() given by name ('{k}') and position ({})", + idx + 1 + ))); + } + bound[idx] = v.clone(); + } } - return builtins::construct_property(args); + return builtins::construct_property(&bound); } "staticmethod" => { return builtins::construct_staticmethod(args); @@ -11968,6 +13216,11 @@ impl Interpreter { return (builtin.call)(args); } if cls.flags.is_exception { + // PEP 654: `BaseExceptionGroup(msg, excs)` lowers to + // `ExceptionGroup` when every leaf is an `Exception`; + // nesting a BaseException inside an `ExceptionGroup` + // (subclass) is a TypeError. + let cls = crate::builtin_types::resolve_exception_group_class(cls.clone(), args)?; let instance = self.build_exception_instance(cls.clone(), args); // If a class anywhere between `cls` and `BaseException` // (exclusive) defines its own `__init__`, run it so @@ -12101,6 +13354,16 @@ impl Interpreter { // payload so the inherited protocols keep working. self.native_for_value_subclass(&cls, args, kwargs)? }; + // CPython `object_new` arity (bpo-31506): the default + // allocator rejects excess arguments unless `__init__` + // is overridden (which then owns the signature). + if (!args.is_empty() || !kwargs.is_empty()) + && native_desc.is_none() + && !cls.flags.is_builtin + && !crate::builtin_types::overrides_dunder_init(&cls) + { + return Err(type_error(format!("{}() takes no arguments", cls.name))); + } let inst = match native_desc { Some(desc) => { Object::Instance(Rc::new(PyInstance::with_native(cls.clone(), desc))) @@ -12121,7 +13384,7 @@ impl Interpreter { // alone (this is how `int.__new__` etc. work for immutable // subclasses). let init_eligible = match &instance { - Object::Instance(inst) => Rc::ptr_eq(&inst.class, &cls), + Object::Instance(inst) => Rc::ptr_eq(&inst.cls(), &cls), // Built-in `__new__` returns may not be Instance; in that // case don't run __init__ — the caller meant to bypass. _ => false, @@ -12137,6 +13400,39 @@ impl Interpreter { if init_owner_is_object && !is_object_new { return Ok(instance); } + // `__init__` stored as a data descriptor (e.g. + // `__init__ = property(...)`) resolves through + // `__get__`, propagating any error it raises — + // CPython's `slot_tp_init` does the lookup via + // `lookup_maybe_method` and surfaces the failure. + let init = match init { + Object::Property(p) => { + let fget = p.fget.clone(); + if matches!(fget, Object::None) { + return Err(attribute_error("unreadable attribute __init__")); + } + let resolved = self.call( + &fget, + std::slice::from_ref(&instance), + &[], + &Rc::new(RefCell::new(DictData::new())), + )?; + let result = self.call( + &resolved, + args, + kwargs, + &Rc::new(RefCell::new(DictData::new())), + )?; + if !matches!(result, Object::None) { + return Err(type_error(format!( + "__init__() should return None, not '{}'", + result.type_name() + ))); + } + return Ok(instance); + } + other => other, + }; let bound = Object::BoundMethod(Rc::new(BoundMethod { receiver: instance.clone(), function: init, @@ -12481,19 +13777,26 @@ impl Interpreter { let gen_name = attr_str("__name__").unwrap_or_else(|| f.name.clone()); let gen_qualname = attr_str("__qualname__").unwrap_or_else(|| code.qualname.clone()); + let gen_code = Object::Code(frame.code.clone()); let gen = Rc::new(PyGenerator::new( gen_name, gen_qualname, kind, + gen_code, Box::new(frame), )); - if code.is_coroutine { - Ok(Object::Coroutine(gen)) + let obj = if code.is_coroutine { + Object::Coroutine(gen) } else if code.is_async_generator { - Ok(Object::AsyncGenerator(gen)) + Object::AsyncGenerator(gen) } else { - Ok(Object::Generator(gen)) - } + Object::Generator(gen) + }; + // RFC 0024: generator frames can participate in + // reference cycles (a local that holds the generator + // itself), so track them like instances. + gc_trace::track(obj.clone()); + Ok(obj) } FrameOutcome::Returned(_) | FrameOutcome::Yielded(_) => { Err(RuntimeError::Internal( @@ -12827,7 +14130,7 @@ impl Interpreter { if let Err(e) = self.call(&warn_explicit, &args, &[], &globals) { if let RuntimeError::PyException(pe) = &e { if let Object::Instance(inst) = &pe.instance { - if inst.class.is_subclass_of(&syntax_warning_ty) { + if inst.cls().is_subclass_of(&syntax_warning_ty) { return Err(crate::error::syntax_error_located( w.message.clone(), Some(filename), @@ -13068,6 +14371,40 @@ impl Interpreter { } } + /// Python 3.13 made `globals`/`locals` passable by keyword on + /// `exec`/`eval`. Fold keyword arguments into positional slots. + fn merge_exec_kwargs( + which: &str, + args: &[Object], + kwargs: &[(String, Object)], + ) -> Result, RuntimeError> { + if kwargs.is_empty() { + return Ok(args.to_vec()); + } + let mut merged = args.to_vec(); + for (k, v) in kwargs { + let pos = match k.as_str() { + "globals" => 1, + "locals" => 2, + other => { + return Err(type_error(format!( + "{which}() got an unexpected keyword argument '{other}'" + ))) + } + }; + if merged.len() > pos { + return Err(type_error(format!( + "{which}() got multiple values for argument '{k}'" + ))); + } + while merged.len() < pos { + merged.push(Object::None); + } + merged.push(v.clone()); + } + Ok(merged) + } + /// `exec(source, globals=None, locals=None)`. Accepts either a /// `Code` object (the typical CPython use case) or a Python source /// string we compile on the fly. The body runs with `globals` @@ -13330,6 +14667,10 @@ impl Interpreter { return Ok(obj); } if let Some(frozen) = self.cache.frozen_source(full) { + // Distinct per-module pseudo-filenames (``) + // let tracebacks resolve source lines through `linecache`'s + // frozen-source hook (backed by `_imp.find_frozen`). + let display = format!(""); // RFC 0021 — frozen modules pay a parse + compile cost // on every fresh `Interpreter::new()` (tests, the REPL, // and the bench harness all spin up many). A @@ -13338,9 +14679,9 @@ impl Interpreter { // stages and go straight from `&'static str` source to // a fully-compiled `CodeObject`. if let Some(code) = frozen_code_cache::get(full) { - return self.run_frozen_compiled(full, code, frozen.is_package, ""); + return self.run_frozen_compiled(full, code, frozen.is_package, &display); } - return self.load_from_source(full, frozen.source, frozen.is_package, ""); + return self.load_from_source(full, frozen.source, frozen.is_package, &display); } // RFC 0022 — try the C-extension loader before the source // loader. We invoke it through a hook to keep the @@ -13458,7 +14799,7 @@ impl Interpreter { // We cache only the compiled code, never the running module // — module *state* is interpreter-local (different // `sys.modules`, different `__name__`). - if filename == "" { + if filename.starts_with(" Option { Object::Instance(i) => i.clone(), _ => return None, }; - let m = inst.class.lookup(name)?; + let m = inst.cls().lookup(name)?; Some(Object::BoundMethod(Rc::new(BoundMethod { receiver: Object::Instance(inst), function: m, @@ -14202,12 +15543,30 @@ fn fallback_globals() -> Rc> { Rc::new(RefCell::new(DictData::new())) } +/// Extract an attribute name from a `getattr`/`hasattr`/`setattr`-style +/// argument, accepting `str` and `str` subclasses (whose payload rides +/// in the instance's native slot), as CPython does. +pub(crate) fn attr_name_of(o: &Object) -> Option { + match o { + Object::Str(s) => Some(s.to_string()), + Object::Instance(inst) => match &inst.native { + Some(Object::Str(s)) => Some(s.to_string()), + _ => None, + }, + _ => None, + } +} + +fn attr_name_arg(o: Option<&Object>) -> Option { + o.and_then(attr_name_of) +} + /// `True` when `o` is a `StopAsyncIteration` instance (or one of /// its subclasses). fn is_stop_async_iteration_obj(o: &Object) -> bool { if let Object::Instance(inst) = o { let target = builtin_types().stop_async_iteration.clone(); - return inst.class.is_subclass_of(&target); + return inst.cls().is_subclass_of(&target); } false } @@ -14271,7 +15630,14 @@ fn make_gen_method(name: &str, receiver: &Object) -> Object { "generator method must be dispatched via Interpreter::call".to_owned(), )) } + // Coroutine methods carry their own sentinels so docstrings and + // display names can say "coroutine" (CPython has distinct method + // tables for `PyGen_Type` and `PyCoro_Type`). + let is_coro = matches!(receiver, Object::Coroutine(_)); let internal_name: &'static str = match name { + "send" if is_coro => ".cor_send", + "throw" if is_coro => ".cor_throw", + "close" if is_coro => ".cor_close", "send" => ".gen_send", "throw" => ".gen_throw", "close" => ".gen_close", @@ -14295,6 +15661,110 @@ fn make_gen_method(name: &str, receiver: &Object) -> Object { })) } +thread_local! { + /// Lazily-created `coroutine_wrapper` type (CPython's + /// `_PyCoroWrapper_Type`) — the iterator `coro.__await__()` returns. + static COROUTINE_WRAPPER_TYPE: RefCell>> = const { RefCell::new(None) }; +} + +/// Build the `coroutine_wrapper` iterator for `coro.__await__()`. +/// CPython returns a distinct object (not the coroutine itself) that +/// forwards `__next__`/`send`/`throw`/`close` to the wrapped coroutine +/// and is its own `__iter__`. The forwarding builtins reach the live +/// interpreter through `current_interpreter_ptr`, like weakref proxies. +fn make_coroutine_wrapper(coro: &Object) -> Object { + fn wrapped_coro(args: &[Object]) -> Result { + let Some(Object::Instance(inst)) = args.first() else { + return Err(type_error("expected coroutine_wrapper instance")); + }; + inst.dict + .borrow() + .get(&DictKey(Object::from_static("__wrapped_coro__"))) + .cloned() + .ok_or_else(|| type_error("coroutine_wrapper lost its coroutine")) + } + fn with_interp( + f: impl FnOnce(&mut Interpreter) -> Result, + ) -> Result { + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| type_error("no running interpreter"))?; + // SAFETY: published by an enclosing VM frame on this thread. + let interp = unsafe { &mut *ptr }; + f(interp) + } + fn cw_next(args: &[Object]) -> Result { + let coro = wrapped_coro(args)?; + with_interp(|i| i.gen_method_send(&coro, Object::None)) + } + fn cw_send(args: &[Object]) -> Result { + let coro = wrapped_coro(args)?; + let v = args.get(1).cloned().unwrap_or(Object::None); + with_interp(|i| i.gen_method_send(&coro, v)) + } + fn cw_throw(args: &[Object]) -> Result { + let coro = wrapped_coro(args)?; + let rest: &[Object] = if args.len() > 1 { &args[1..] } else { &[] }; + with_interp(|i| i.gen_method_throw(&coro, rest)) + } + fn cw_close(args: &[Object]) -> Result { + let coro = wrapped_coro(args)?; + with_interp(|i| i.gen_method_close(&coro)) + } + fn cw_iter(args: &[Object]) -> Result { + args.first() + .cloned() + .ok_or_else(|| type_error("expected coroutine_wrapper instance")) + } + fn cw_reduce(_args: &[Object]) -> Result { + Err(type_error("cannot pickle 'coroutine_wrapper' object")) + } + let ty = COROUTINE_WRAPPER_TYPE.with(|cell| { + if let Some(t) = cell.borrow().clone() { + return t; + } + let mut td = DictData::new(); + for (name, f) in [ + ( + "__next__", + cw_next as fn(&[Object]) -> Result, + ), + ("send", cw_send), + ("throw", cw_throw), + ("close", cw_close), + ("__iter__", cw_iter), + ("__reduce__", cw_reduce), + ("__reduce_ex__", cw_reduce), + ] { + td.insert( + DictKey(Object::from_static(name)), + Object::Builtin(Rc::new(BuiltinFn { + name, + call: Box::new(f), + call_kw: None, + })), + ); + } + let t = TypeObject::new_with_flags( + "coroutine_wrapper", + vec![crate::builtin_types::builtin_types().object_.clone()], + td, + crate::types::TypeFlags { + is_exception: false, + is_builtin: true, + }, + ) + .expect("coroutine_wrapper type"); + *cell.borrow_mut() = Some(t.clone()); + t + }); + let inst = Rc::new(crate::types::PyInstance::new(ty)); + inst.dict.borrow_mut().insert( + DictKey(Object::from_static("__wrapped_coro__")), + coro.clone(), + ); + Object::Instance(inst) +} + /// Map a `type().` access to the sentinel name for /// its *unbound* form — the function that takes the instance as `args[0]`, /// e.g. `type(agen).__anext__(agen)`. CPython exposes generator/coroutine/ @@ -14319,9 +15789,9 @@ fn unbound_gen_method_sentinel(ty: &Rc, name: &str) -> Option<&'stat } if Rc::ptr_eq(ty, &bt.coroutine_) { return match name { - "send" => Some(".u.gen_send"), - "throw" => Some(".u.gen_throw"), - "close" => Some(".u.gen_close"), + "send" => Some(".u.cor_send"), + "throw" => Some(".u.cor_throw"), + "close" => Some(".u.cor_close"), "__await__" => Some(".u.gen_iter"), _ => None, }; @@ -14339,6 +15809,20 @@ fn unbound_gen_method_sentinel(ty: &Rc, name: &str) -> Option<&'stat None } +/// True when `instance` is a `PyInstance` whose dict holds a non-None +/// value for `name`. Used to recognise exceptions that already carry an +/// explicit `__context__` (set by an earlier raise/sync) so the fresh- +/// exception context chaining doesn't overwrite it. +fn instance_has_nonnull_attr(instance: &Object, name: &str) -> bool { + match instance { + Object::Instance(i) => matches!( + i.dict.borrow().get(&DictKey(Object::from_str(name))), + Some(v) if !matches!(v, Object::None) + ), + _ => false, + } +} + /// Look up the `value` attribute on a `StopIteration` instance. Falls /// back to `None` if absent. fn exception_value(instance: &Object) -> Object { @@ -14832,7 +16316,7 @@ fn apply_trailer(value: Object, trailer: &str) -> Result { .borrow() .get(&DictKey(Object::from_str(attr))) .cloned() - .or_else(|| inst.class.lookup(attr)) + .or_else(|| inst.cls().lookup(attr)) .ok_or_else(|| attribute_error(format!("has no attribute '{attr}'"))), _ => Err(attribute_error(format!( "'{}' has no attribute '{}'", @@ -15392,12 +16876,52 @@ fn format_missing_arguments(func_name: &str, kind: &str, names: &[&str]) -> Stri /// final dotted component — matching CPython, where `gc.collect.__name__` /// is `'collect'` and `str.format.__name__` is `'format'`. fn builtin_display_name(name: &'static str) -> &'static str { + // Generator-family sentinels carry internal names; surface the + // Python-visible method name. + match name.strip_prefix(".u").unwrap_or(name) { + ".gen_send" | ".cor_send" => return "send", + ".gen_throw" | ".cor_throw" => return "throw", + ".gen_close" | ".cor_close" => return "close", + ".gen_next" => return "__next__", + ".gen_iter" => return "__iter__", + ".agen_aiter" => return "__aiter__", + ".agen_anext" => return "__anext__", + ".agen_send" => return "asend", + ".agen_throw" => return "athrow", + ".agen_close" => return "aclose", + _ => {} + } match name.strip_prefix('.') { Some(rest) => rest.rsplit('.').next().unwrap_or(rest), None => name, } } +/// Docstrings for builtin methods, matching CPython's C-level method +/// tables where tests inspect them (`gen.__next__.__doc__` etc.). +fn builtin_doc(name: &str) -> Option<&'static str> { + // Bound methods use ".gen_send"; unbound descriptors ".u.gen_send". + match name.strip_prefix(".u").unwrap_or(name) { + ".gen_next" => Some("Implement next(self)."), + ".gen_iter" => Some("Implement iter(self)."), + ".gen_send" => Some( + "send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.", + ), + ".gen_throw" => Some( + "throw(value)\nthrow(type[,value[,tb]])\n\nRaise exception in generator,\nreturn next yielded value or raise StopIteration.", + ), + ".gen_close" => Some("close() -> raise GeneratorExit inside generator."), + ".cor_send" => Some( + "send(arg) -> send 'arg' into coroutine,\nreturn next iterated value or raise StopIteration.", + ), + ".cor_throw" => Some( + "throw(typ[,val[,tb]]) -> raise exception in coroutine,\nreturn next iterated value or raise StopIteration.", + ), + ".cor_close" => Some("close() -> raise GeneratorExit inside coroutine."), + _ => None, + } +} + /// Apply a CPython-style format spec to a value. We implement the /// subset needed by f-strings: fill/align, sign, width, precision, /// type. Anything we don't yet handle falls back to the plain string. @@ -16320,7 +17844,7 @@ fn is_index_error(e: &RuntimeError) -> bool { if let RuntimeError::PyException(pe) = e { if let Object::Instance(inst) = &pe.instance { return inst - .class + .cls() .is_subclass_of(&crate::builtin_types::builtin_types().index_error); } } @@ -16331,7 +17855,7 @@ fn is_type_error(e: &RuntimeError) -> bool { if let RuntimeError::PyException(pe) = e { if let Object::Instance(inst) = &pe.instance { return inst - .class + .cls() .is_subclass_of(&crate::builtin_types::builtin_types().type_error); } } @@ -16502,7 +18026,7 @@ fn constant_to_object(c: Constant) -> Object { Constant::Bytes(b) => Object::new_bytes(b), Constant::Tuple(xs) => Object::new_tuple(xs.into_iter().map(constant_to_object).collect()), Constant::Code(c) => Object::Code(Rc::from(*c)), - Constant::Ellipsis => Object::None, + Constant::Ellipsis => crate::vm_singletons::ellipsis(), } } @@ -16735,7 +18259,9 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result bool { - matches!(obj, Object::Type(_) | Object::None) || is_pep604_union(obj).is_some() + matches!(obj, Object::Type(_) | Object::None) + || is_pep604_union(obj).is_some() + || is_generic_alias(obj) } /// A PEP 604 union can only be *initiated* by an operand that carries @@ -16744,7 +18270,7 @@ fn is_union_eligible(obj: &Object) -> bool { /// but cannot start one, so `None | None` raises `TypeError` like /// CPython. fn is_union_initiator(obj: &Object) -> bool { - matches!(obj, Object::Type(_)) || is_pep604_union(obj).is_some() + matches!(obj, Object::Type(_)) || is_pep604_union(obj).is_some() || is_generic_alias(obj) } /// Detect whether `obj` is a PEP 604 union. Returns the flattened diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index ff13016..6bd7694 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -172,7 +172,7 @@ impl fmt::Debug for Object { Object::Iter(_) => write!(f, ""), Object::Slice(s) => write!(f, "slice({:?}, {:?}, {:?})", s.start, s.stop, s.step), Object::Type(t) => write!(f, "", t.name), - Object::Instance(i) => write!(f, "<{} object>", i.class.name), + Object::Instance(i) => write!(f, "<{} object>", i.cls().name), Object::Module(m) => write!(f, "", m.name), Object::Generator(g) => write!(f, "", g.name.borrow()), Object::Coroutine(g) => write!(f, "", g.name.borrow()), @@ -248,6 +248,12 @@ pub struct PyFrame { /// subsequent `'line'` / `'return'` / `'exception'` events on /// the frame. `Object::None` disables tracing for the frame. pub trace: RefCell, + /// Backlink to the generator/coroutine that owns this frame + /// (weak — the frame must not keep its generator alive). Set by + /// the VM when the snapshot is cached on a generator frame; lets + /// `frame.clear()` tear down the suspended generator like + /// CPython's `frame_clear`. + pub gen_owner: RefCell>>, /// Per-frame `f_lineno` override. CPython lets debuggers set /// `f_lineno` to jump to a different line; we keep storage so /// reads round-trip, even though writes don't actually move the @@ -292,8 +298,11 @@ impl PyFrame { /// calls return the same dict object so `id(frame.f_locals)` is /// stable. pub fn locals(&self) -> Object { - if let Some(v) = self.locals_cache.borrow().as_ref() { - return v.clone(); + if self.locals_cache.borrow().is_some() { + self.refresh_locals(); + if let Some(v) = self.locals_cache.borrow().as_ref() { + return v.clone(); + } } let provider = self.locals_provider.borrow().clone(); let dict = provider @@ -303,11 +312,53 @@ impl PyFrame { dict } - /// Force re-materialisation on the next `locals()` call. Used by - /// the VM after the frame has executed enough to make the cached - /// snapshot stale (function entry, generator resume). + /// Refresh the materialised `f_locals` dict *in place*, keeping + /// its identity stable (PEP 667: a handle obtained earlier + /// observes later execution of the frame). Frame names are + /// rewritten from the live state; user-added extra keys are + /// preserved. + pub fn refresh_locals(&self) { + let cached = self.locals_cache.borrow().clone(); + let Some(Object::Dict(cached_rc)) = cached else { + return; + }; + let provider = self.locals_provider.borrow().clone(); + let Some(provider) = provider else { return }; + let Object::Dict(fresh_rc) = provider() else { + return; + }; + // Module/class scopes hand back the namespace dict itself — + // already live, nothing to merge. + if Rc::ptr_eq(&cached_rc, &fresh_rc) { + return; + } + let mut out = fresh_rc.borrow().clone(); + { + let old = cached_rc.borrow(); + for (k, v) in old.iter() { + let is_frame_name = match &k.0 { + Object::Str(s) => { + let name = s.as_ref(); + self.code.varnames.iter().any(|n| n == name) + || self.code.cellvars.iter().any(|n| n == name) + || self.code.freevars.iter().any(|n| n == name) + } + _ => false, + }; + if !is_frame_name && !out.contains_key(k) { + out.insert(k.clone(), v.clone()); + } + } + } + *cached_rc.borrow_mut() = out; + } + + /// Bring the materialised snapshot (if any) up to date with the + /// frame's live state. Used by the VM after the frame has executed + /// enough to make the cached contents stale (function entry, + /// generator resume). The dict's identity is preserved. pub fn invalidate_locals(&self) { - self.locals_cache.borrow_mut().take(); + self.refresh_locals(); } } @@ -566,7 +617,7 @@ fn instance_has_custom_dunder(obj: &Object, name: &str) -> bool { obj, Object::Instance(inst) if matches!( - inst.class.lookup(name), + inst.cls().lookup(name), Some(Object::Function(_) | Object::BoundMethod(_)) ) ) @@ -737,6 +788,10 @@ pub struct PyGenerator { /// PEP 479 (a `StopIteration` escaping the *body* becomes a /// `RuntimeError`) with the right wording per flavour. pub kind: CoroutineKind, + /// `gi_code` — held on the generator itself (CPython keeps a + /// strong reference) so it stays readable after the generator + /// finishes and the frame is dropped. + pub code: Object, pub state: RefCell, } @@ -745,12 +800,14 @@ impl PyGenerator { name: impl Into, qualname: impl Into, kind: CoroutineKind, + code: Object, frame: Box, ) -> Self { Self { name: RefCell::new(name.into()), qualname: RefCell::new(qualname.into()), kind, + code, state: RefCell::new(GeneratorState::Created(frame)), } } @@ -766,6 +823,39 @@ impl fmt::Debug for PyGenerator { } } +impl Drop for PyGenerator { + fn drop(&mut self) { + // CPython finalizes a generator the moment its refcount dies: + // a *suspended* frame gets `GeneratorExit` thrown in so + // `finally:`/`with` cleanup runs. We can't run Python from + // `Drop`, so resurrect the live frame into the VM's + // pending-finalizer queue; `gc.collect()` and the module-exit + // path drain it. Created-but-never-started frames have run no + // user code, so (like CPython's `gen_close`) they are simply + // marked completed. + let Ok(mut state) = self.state.try_borrow_mut() else { + return; + }; + let prev = std::mem::replace(&mut *state, GeneratorState::Finished); + drop(state); + if let GeneratorState::Suspended(frame) = prev { + let resurrected = Rc::new(PyGenerator { + name: RefCell::new(self.name.borrow().clone()), + qualname: RefCell::new(self.qualname.borrow().clone()), + kind: self.kind, + code: self.code.clone(), + state: RefCell::new(GeneratorState::Suspended(frame)), + }); + let obj = match self.kind { + CoroutineKind::Generator => Object::Generator(resurrected), + CoroutineKind::Coroutine => Object::Coroutine(resurrected), + CoroutineKind::AsyncGenerator => Object::AsyncGenerator(resurrected), + }; + crate::vm_singletons::try_push_pending_finalizer(obj); + } + } +} + /// Flavour of a `PyGenerator`. Stored alongside the suspended frame /// so the same suspension machinery serves all three async-shaped /// objects. @@ -776,6 +866,19 @@ pub enum CoroutineKind { AsyncGenerator, } +impl CoroutineKind { + /// The flavour word CPython uses in error messages + /// ("generator already executing", "coroutine ignored + /// GeneratorExit", "async generator ..."). + pub fn word(self) -> &'static str { + match self { + Self::Generator => "generator", + Self::Coroutine => "coroutine", + Self::AsyncGenerator => "async generator", + } + } +} + /// State machine for an active or exhausted generator. The frame is /// stored as `Box` because `PyGenerator` lives in the /// `object` module but `Frame` lives in `vm::lib`. @@ -1513,20 +1616,20 @@ impl Object { Object::Instance(inst) => { // int/str/… subclass instances are truthy per their // wrapped value unless the class overrides __bool__/__len__. - if inst.class.lookup("__bool__").is_none() && inst.class.lookup("__len__").is_none() + if inst.cls().lookup("__bool__").is_none() && inst.cls().lookup("__len__").is_none() { if let Some(native) = &inst.native { return native.is_truthy(); } } // Honour __bool__ then __len__ before defaulting to True. - if let Some(m) = inst.class.lookup("__bool__") { + if let Some(m) = inst.cls().lookup("__bool__") { // Caller dispatches; we cannot run Python here. // Default to True; the dispatch site handles the // dunder dispatch when it has interpreter access. let _ = m; true - } else if let Some(m) = inst.class.lookup("__len__") { + } else if let Some(m) = inst.cls().lookup("__len__") { let _ = m; true } else { @@ -2024,7 +2127,7 @@ impl Object { /// `Object::Instance` instead of the static placeholder. pub fn type_name_owned(&self) -> String { match self { - Object::Instance(inst) => inst.class.name.clone(), + Object::Instance(inst) => inst.cls().name.clone(), Object::Type(t) => format!("type[{}]", t.name), other => other.type_name().to_owned(), } @@ -2190,7 +2293,7 @@ impl Object { // `` instead of ``. let key = DictKey(Object::from_static("__repr__")); let has_user_repr = inst - .class + .cls() .mro .borrow() .iter() @@ -2209,11 +2312,11 @@ impl Object { } } } - format!("<{} object>", inst.class.name) + format!("<{} object>", inst.cls().name) } else { format!( "<{} object at 0x{:x}>", - inst.class.name, + inst.cls().name, Rc::as_ptr(inst) as usize ) } diff --git a/crates/weavepy-vm/src/specialize.rs b/crates/weavepy-vm/src/specialize.rs index ae94a7b..77b5fc6 100644 --- a/crates/weavepy-vm/src/specialize.rs +++ b/crates/weavepy-vm/src/specialize.rs @@ -131,7 +131,7 @@ pub fn attempt_specialize_load_attr(obj: &Object, name: &str) -> InlineCache { // Only cache when the type doesn't customize lookup. // If the class has __getattr__ / __getattribute__ / // descriptors, the slow path is mandatory. - if type_has_attr_override(&inst.class) { + if type_has_attr_override(&inst.cls()) { return InlineCache::Cooldown(COOLDOWN); } // First check the instance dict — that's the @@ -139,17 +139,18 @@ pub fn attempt_specialize_load_attr(obj: &Object, name: &str) -> InlineCache { let dict = inst.dict.borrow(); if let Some(idx) = dict.index_of_key_str(name) { return InlineCache::LoadAttrInstance { - type_id: rc_id(&inst.class), + type_id: rc_id(&inst.cls()), key_idx: idx, }; } drop(dict); // Otherwise look in the type's dict — the // `LoadAttrType` shape (descriptor or class attribute). - let class_dict = inst.class.dict.borrow(); + let cls = inst.cls(); + let class_dict = cls.dict.borrow(); if let Some(idx) = class_dict.index_of_key_str(name) { return InlineCache::LoadAttrType { - type_id: rc_id(&inst.class), + type_id: rc_id(&cls), key_idx: idx, }; } @@ -210,13 +211,13 @@ pub fn attempt_specialize_load_global( pub fn attempt_specialize_store_attr(obj: &Object, name: &str) -> InlineCache { match obj { Object::Instance(inst) => { - if type_has_attr_override(&inst.class) { + if type_has_attr_override(&inst.cls()) { return InlineCache::Cooldown(COOLDOWN); } let dict = inst.dict.borrow(); if let Some(idx) = dict.index_of_key_str(name) { return InlineCache::StoreAttrInstance { - type_id: rc_id(&inst.class), + type_id: rc_id(&inst.cls()), key_idx: idx, }; } diff --git a/crates/weavepy-vm/src/stdlib/abc_mod.rs b/crates/weavepy-vm/src/stdlib/abc_mod.rs index eba4e97..e161fad 100644 --- a/crates/weavepy-vm/src/stdlib/abc_mod.rs +++ b/crates/weavepy-vm/src/stdlib/abc_mod.rs @@ -113,7 +113,7 @@ fn abc_instancecheck(args: &[Object]) -> Result { let cls = args.first().cloned().unwrap_or(Object::None); let inst = args.get(1).cloned().unwrap_or(Object::None); if let (Object::Type(t), Object::Instance(i)) = (&cls, &inst) { - if i.class.is_subclass_of(t) { + if i.cls().is_subclass_of(t) { return Ok(Object::Bool(true)); } if let Some(Object::Set(reg)) = t @@ -124,7 +124,7 @@ fn abc_instancecheck(args: &[Object]) -> Result { { for entry in reg.borrow().iter() { if let Object::Type(et) = &entry.0 { - if i.class.is_subclass_of(et) { + if i.cls().is_subclass_of(et) { return Ok(Object::Bool(true)); } } diff --git a/crates/weavepy-vm/src/stdlib/ast_mod.rs b/crates/weavepy-vm/src/stdlib/ast_mod.rs index 7924b3f..05688fb 100644 --- a/crates/weavepy-vm/src/stdlib/ast_mod.rs +++ b/crates/weavepy-vm/src/stdlib/ast_mod.rs @@ -903,8 +903,6 @@ fn constant(c: &past::Constant) -> Object { C::Str(s) => Object::from_str(s.clone()), C::Bytes(b) => Object::new_bytes(b.clone()), C::Tuple(items) => Object::new_tuple(items.iter().map(constant).collect()), - // WeavePy models the `...` singleton as `None` (parity with the - // compiler's `Constant::Ellipsis` lowering). - C::Ellipsis => Object::None, + C::Ellipsis => crate::vm_singletons::ellipsis(), } } diff --git a/crates/weavepy-vm/src/stdlib/marshal_mod.rs b/crates/weavepy-vm/src/stdlib/marshal_mod.rs index 4237ecc..9fb86ea 100644 --- a/crates/weavepy-vm/src/stdlib/marshal_mod.rs +++ b/crates/weavepy-vm/src/stdlib/marshal_mod.rs @@ -34,6 +34,7 @@ const CO_VARARGS: u32 = 0x0004; const CO_VARKEYWORDS: u32 = 0x0008; const CO_GENERATOR: u32 = 0x0020; const CO_COROUTINE: u32 = 0x0080; +const CO_ITERABLE_COROUTINE: u32 = 0x0100; const CO_ASYNC_GENERATOR: u32 = 0x0200; #[allow(dead_code)] @@ -369,6 +370,9 @@ fn code_flags(co: &CodeObject) -> u32 { if co.is_coroutine { f |= CO_COROUTINE; } + if co.is_iterable_coroutine { + f |= CO_ITERABLE_COROUTINE; + } if co.is_async_generator { f |= CO_ASYNC_GENERATOR; } @@ -477,7 +481,7 @@ impl<'a> MarshalReader<'a> { TYPE_NONE => Ok(Object::None), TYPE_TRUE => Ok(Object::Bool(true)), TYPE_FALSE => Ok(Object::Bool(false)), - TYPE_ELLIPSIS => Ok(Object::None), // Ellipsis singleton not modelled separately yet. + TYPE_ELLIPSIS => Ok(crate::vm_singletons::ellipsis()), TYPE_INT => { let v = self.read_int()?; Ok(Object::Int(i64::from(v))) @@ -675,6 +679,7 @@ impl<'a> MarshalReader<'a> { is_generator: flags & CO_GENERATOR != 0, is_coroutine: flags & CO_COROUTINE != 0, is_async_generator: flags & CO_ASYNC_GENERATOR != 0, + is_iterable_coroutine: flags & CO_ITERABLE_COROUTINE != 0, }; Ok(Object::Code(Rc::new(co))) } diff --git a/crates/weavepy-vm/src/stdlib/mod.rs b/crates/weavepy-vm/src/stdlib/mod.rs index 179f413..c8350f5 100644 --- a/crates/weavepy-vm/src/stdlib/mod.rs +++ b/crates/weavepy-vm/src/stdlib/mod.rs @@ -23,6 +23,7 @@ pub mod codecs_mod; pub mod csv_mod; pub mod datetime_mod; pub mod errno_mod; +pub mod testinternalcapi_mod; pub mod fcntl_mod; pub mod fnmatch_mod; pub mod gc_mod; @@ -38,7 +39,6 @@ pub mod lzma_mod; pub mod marshal_mod; pub mod math; pub mod os; -pub mod random; pub mod resource_mod; pub mod secrets_mod; pub mod select_mod; @@ -90,7 +90,6 @@ pub fn register_all(cache: &ModuleCache) { cache.register_builtin("os.path", os::build_path); cache.register_builtin("io", io::build); cache.register_builtin("json", json::build); - cache.register_builtin("random", random::build); cache.register_builtin("time", time::build); cache.register_builtin("_thread", thread_real::build); cache.register_builtin("errno", errno_mod::build); @@ -180,6 +179,14 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/keyword.py"), is_package: false, }, + // `random` — verbatim CPython distribution layer over the + // Rust `_random` MT19937 core (RFC 0037: `random.Random(42)` + // is stream-identical to CPython). + FrozenSource { + name: "random", + source: include_str!("python/random_mod.py"), + is_package: false, + }, // Internal: `_SeqIter`, the lazy legacy-`__getitem__` iterator // `iter(obj)` returns when *obj* has no `__iter__` (CPython's // built-in `iterator`/seqiterobject). Kept out of `builtins` to diff --git a/crates/weavepy-vm/src/stdlib/python/_testlimitedcapi.py b/crates/weavepy-vm/src/stdlib/python/_testlimitedcapi.py index 9bad119..009d81f 100644 --- a/crates/weavepy-vm/src/stdlib/python/_testlimitedcapi.py +++ b/crates/weavepy-vm/src/stdlib/python/_testlimitedcapi.py @@ -22,3 +22,11 @@ def sequence_setitem(obj, i, value): def sequence_delitem(obj, i): # PySequence_DelItem(obj, i) del obj[i] + + +def object_hasattrstring(obj, name): + # PyObject_HasAttrString(obj, name) — `name` arrives as bytes + # (a C `char*`); returns 1/0. + if isinstance(name, (bytes, bytearray)): + name = name.decode("utf-8") + return 1 if hasattr(obj, name) else 0 diff --git a/crates/weavepy-vm/src/stdlib/python/asyncio.py b/crates/weavepy-vm/src/stdlib/python/asyncio.py index 5078397..4272261 100644 --- a/crates/weavepy-vm/src/stdlib/python/asyncio.py +++ b/crates/weavepy-vm/src/stdlib/python/asyncio.py @@ -198,6 +198,14 @@ def _step(self, value, exc=None): self._schedule_callbacks() return except BaseException as e: + # CPython's C `_asyncio` Task drives the coroutine without + # a Python frame, so the Task machinery never appears in + # the exception's traceback. Strip our own `_step` frame + # (the head entry, where the exception was caught) to give + # the same observable traceback. + tb = e.__traceback__ + if tb is not None and tb.tb_frame.f_code.co_name == "_step": + e.__traceback__ = tb.tb_next self.set_exception(e) return diff --git a/crates/weavepy-vm/src/stdlib/python/collections.py b/crates/weavepy-vm/src/stdlib/python/collections.py index a32dc60..bcf5b82 100644 --- a/crates/weavepy-vm/src/stdlib/python/collections.py +++ b/crates/weavepy-vm/src/stdlib/python/collections.py @@ -294,9 +294,21 @@ def __repr__(self): class defaultdict(_MappingMixin): """Dict that creates missing values via a ``default_factory``.""" - def __init__(self, default_factory=None): + def __init__(self, default_factory=None, *args, **kwargs): + if default_factory is not None and not callable(default_factory): + raise TypeError("first argument must be callable or None") _MappingMixin.__init__(self) self.default_factory = default_factory + if args: + src = args[0] + if hasattr(src, "keys"): + for k in src.keys(): + self[k] = src[k] + else: + for k, v in src: + self[k] = v + for k, v in kwargs.items(): + self[k] = v def __getitem__(self, key): if key in self._data: diff --git a/crates/weavepy-vm/src/stdlib/python/inspect.py b/crates/weavepy-vm/src/stdlib/python/inspect.py index cb431cb..ed2cb33 100644 --- a/crates/weavepy-vm/src/stdlib/python/inspect.py +++ b/crates/weavepy-vm/src/stdlib/python/inspect.py @@ -78,7 +78,8 @@ ] -# Code-object flags. Keep in sync with weavepy-compiler/src/code.rs. +# Code-object flags — CPython's values (keep in sync with +# `code_flags` in weavepy-vm/src/builtins.rs). CO_OPTIMIZED = 0x0001 CO_NEWLOCALS = 0x0002 CO_VARARGS = 0x0004 @@ -86,9 +87,9 @@ CO_NESTED = 0x0010 CO_GENERATOR = 0x0020 CO_NOFREE = 0x0040 -CO_COROUTINE = 0x0100 -CO_ITERABLE_COROUTINE = 0x0200 -CO_ASYNC_GENERATOR = 0x0400 +CO_COROUTINE = 0x0080 +CO_ITERABLE_COROUTINE = 0x0100 +CO_ASYNC_GENERATOR = 0x0200 def _safe_type_name(t): @@ -306,6 +307,86 @@ def istraceback(obj): return type(obj).__name__ == "traceback" +GEN_CREATED = 'GEN_CREATED' +GEN_RUNNING = 'GEN_RUNNING' +GEN_SUSPENDED = 'GEN_SUSPENDED' +GEN_CLOSED = 'GEN_CLOSED' + + +def getgeneratorstate(generator): + """Get current state of a generator-iterator.""" + if generator.gi_running: + return GEN_RUNNING + if generator.gi_suspended: + return GEN_SUSPENDED + if generator.gi_frame is None: + return GEN_CLOSED + return GEN_CREATED + + +CORO_CREATED = 'CORO_CREATED' +CORO_RUNNING = 'CORO_RUNNING' +CORO_SUSPENDED = 'CORO_SUSPENDED' +CORO_CLOSED = 'CORO_CLOSED' + + +def getcoroutinestate(coroutine): + """Get current state of a coroutine.""" + if coroutine.cr_running: + return CORO_RUNNING + if coroutine.cr_suspended: + return CORO_SUSPENDED + if coroutine.cr_frame is None: + return CORO_CLOSED + return CORO_CREATED + + +AGEN_CREATED = 'AGEN_CREATED' +AGEN_RUNNING = 'AGEN_RUNNING' +AGEN_SUSPENDED = 'AGEN_SUSPENDED' +AGEN_CLOSED = 'AGEN_CLOSED' + + +def getasyncgenstate(agen): + """Get current state of an asynchronous generator.""" + if agen.ag_running: + return AGEN_RUNNING + if agen.ag_suspended: + return AGEN_SUSPENDED + if agen.ag_frame is None: + return AGEN_CLOSED + return AGEN_CREATED + + +def getgeneratorlocals(generator): + """Get the mapping of generator local variables to their current values.""" + if not isgenerator(generator): + raise TypeError("{!r} is not a Python generator".format(generator)) + frame = getattr(generator, "gi_frame", None) + if frame is not None: + return generator.gi_frame.f_locals + return {} + + +def getcoroutinelocals(coroutine): + """Get the mapping of coroutine local variables to their current values.""" + frame = getattr(coroutine, "cr_frame", None) + if frame is not None: + return frame.f_locals + return {} + + +def getasyncgenlocals(agen): + """Get the mapping of asynchronous generator local variables to their + current values.""" + if not isasyncgen(agen): + raise TypeError(f"{agen!r} is not a Python async generator") + frame = getattr(agen, "ag_frame", None) + if frame is not None: + return agen.ag_frame.f_locals + return {} + + def isframe(obj): return type(obj).__name__ == "frame" diff --git a/crates/weavepy-vm/src/stdlib/python/linecache.py b/crates/weavepy-vm/src/stdlib/python/linecache.py index f744678..6ba85c2 100644 --- a/crates/weavepy-vm/src/stdlib/python/linecache.py +++ b/crates/weavepy-vm/src/stdlib/python/linecache.py @@ -87,6 +87,22 @@ def updatecache(filename, module_globals=None): lines = lines.splitlines(keepends=True) _cache[filename] = (len(lines), None, lines, filename) return lines + # WeavePy frozen stdlib modules carry `` filenames; + # their source is recoverable through `_imp.find_frozen`. + if filename.startswith(""): + modname = filename[8:-1] + try: + import _imp + found = _imp.find_frozen(modname) + except Exception: + found = None + if found is not None: + src = found[0] + if isinstance(src, bytes): + src = src.decode("utf-8", "replace") + lines = src.splitlines(keepends=True) + _cache[filename] = (len(lines), None, lines, filename) + return lines name = filename # Try direct file system access. try: diff --git a/crates/weavepy-vm/src/stdlib/python/pickle.py b/crates/weavepy-vm/src/stdlib/python/pickle.py index a00553f..8afc822 100644 --- a/crates/weavepy-vm/src/stdlib/python/pickle.py +++ b/crates/weavepy-vm/src/stdlib/python/pickle.py @@ -243,7 +243,10 @@ def _save(self, obj): except Exception: is_callable_like = False try: - is_type = type(obj).__name__ == "type" + # Name-based (not `isinstance`) for the same threading reason + # as above; walk the metaclass MRO so classes with a custom + # metaclass (`EnumType`, `ABCMeta`, …) count as types too. + is_type = any(t.__name__ == "type" for t in type(obj).__mro__) except Exception: is_type = False if is_callable_like or is_type: @@ -261,6 +264,17 @@ def _save(self, obj): if qualname and _resolves_to_self(module, qualname, obj): self._save_global(module, qualname) return + # Classes and plain/builtin functions are *only* picklable by + # reference. CPython's `save_global` raises PicklingError for + # anything that doesn't resolve (e.g. a class defined inside a + # function: `` in its qualname); falling through to + # `__reduce_ex__` here would mis-pickle the class as an + # instance of its metaclass. + if is_type or tname in ("function", "builtin_function_or_method"): + raise PicklingError( + "Can't pickle %r: it's not found as %s.%s" + % (obj, module, qualname) + ) # Arbitrary instances — try __reduce_ex__ / __reduce__ (the # canonical CPython pickle protocol). Falls back to the # PicklingError below if neither is provided. diff --git a/crates/weavepy-vm/src/stdlib/python/random_mod.py b/crates/weavepy-vm/src/stdlib/python/random_mod.py new file mode 100644 index 0000000..1abcae7 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/random_mod.py @@ -0,0 +1,1070 @@ +"""Random variable generators. + + bytes + ----- + uniform bytes (values between 0 and 255) + + integers + -------- + uniform within range + + sequences + --------- + pick random element + pick random sample + pick weighted random sample + generate random permutation + + distributions on the real line: + ------------------------------ + uniform + triangular + normal (Gaussian) + lognormal + negative exponential + gamma + beta + pareto + Weibull + + distributions on the circle (angles 0 to 2pi) + --------------------------------------------- + circular uniform + von Mises + + discrete distributions + ---------------------- + binomial + + +General notes on the underlying Mersenne Twister core generator: + +* The period is 2**19937-1. +* It is one of the most extensively tested generators in existence. +* The random() method is implemented in C, executes in a single Python step, + and is, therefore, threadsafe. + +""" + +# Translated by Guido van Rossum from C source provided by +# Adrian Baddeley. Adapted by Raymond Hettinger for use with +# the Mersenne Twister and os.urandom() core generators. + +from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil +from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin +from math import tau as TWOPI, floor as _floor, isfinite as _isfinite +from math import lgamma as _lgamma, fabs as _fabs, log2 as _log2 +from os import urandom as _urandom +from _collections_abc import Sequence as _Sequence +from operator import index as _index +from itertools import accumulate as _accumulate, repeat as _repeat +from bisect import bisect as _bisect +import os as _os +import _random + +__all__ = [ + "Random", + "SystemRandom", + "betavariate", + "binomialvariate", + "choice", + "choices", + "expovariate", + "gammavariate", + "gauss", + "getrandbits", + "getstate", + "lognormvariate", + "normalvariate", + "paretovariate", + "randbytes", + "randint", + "random", + "randrange", + "sample", + "seed", + "setstate", + "shuffle", + "triangular", + "uniform", + "vonmisesvariate", + "weibullvariate", +] + +NV_MAGICCONST = 4 * _exp(-0.5) / _sqrt(2.0) +LOG4 = _log(4.0) +SG_MAGICCONST = 1.0 + _log(4.5) +BPF = 53 # Number of bits in a float +RECIP_BPF = 2 ** -BPF +_ONE = 1 +_sha512 = None + + +class Random(_random.Random): + """Random number generator base class used by bound module functions. + + Used to instantiate instances of Random to get generators that don't + share state. + + Class Random can also be subclassed if you want to use a different basic + generator of your own devising: in that case, override the following + methods: random(), seed(), getstate(), and setstate(). + Optionally, implement a getrandbits() method so that randrange() + can cover arbitrarily large ranges. + + """ + + VERSION = 3 # used by getstate/setstate + + def __init__(self, x=None): + """Initialize an instance. + + Optional argument x controls seeding, as for Random.seed(). + """ + + self.seed(x) + self.gauss_next = None + + def seed(self, a=None, version=2): + """Initialize internal state from a seed. + + The only supported seed types are None, int, float, + str, bytes, and bytearray. + + None or no argument seeds from current time or from an operating + system specific randomness source if available. + + If *a* is an int, all bits are used. + + For version 2 (the default), all of the bits are used if *a* is a str, + bytes, or bytearray. For version 1 (provided for reproducing random + sequences from older versions of Python), the algorithm for str and + bytes generates a narrower range of seeds. + + """ + + if version == 1 and isinstance(a, (str, bytes)): + a = a.decode('latin-1') if isinstance(a, bytes) else a + x = ord(a[0]) << 7 if a else 0 + for c in map(ord, a): + x = ((1000003 * x) ^ c) & 0xFFFFFFFFFFFFFFFF + x ^= len(a) + a = -2 if x == -1 else x + + elif version == 2 and isinstance(a, (str, bytes, bytearray)): + global _sha512 + if _sha512 is None: + try: + # hashlib is pretty heavy to load, try lean internal + # module first + from _sha2 import sha512 as _sha512 + except ImportError: + # fallback to official implementation + from hashlib import sha512 as _sha512 + + if isinstance(a, str): + a = a.encode() + a = int.from_bytes(a + _sha512(a).digest()) + + elif not isinstance(a, (type(None), int, float, str, bytes, bytearray)): + raise TypeError('The only supported seed types are:\n' + 'None, int, float, str, bytes, and bytearray.') + + super().seed(a) + self.gauss_next = None + + def getstate(self): + """Return internal state; can be passed to setstate() later.""" + return self.VERSION, super().getstate(), self.gauss_next + + def setstate(self, state): + """Restore internal state from object returned by getstate().""" + version = state[0] + if version == 3: + version, internalstate, self.gauss_next = state + super().setstate(internalstate) + elif version == 2: + version, internalstate, self.gauss_next = state + # In version 2, the state was saved as signed ints, which causes + # inconsistencies between 32/64-bit systems. The state is + # really unsigned 32-bit ints, so we convert negative ints from + # version 2 to positive longs for version 3. + try: + internalstate = tuple(x % (2 ** 32) for x in internalstate) + except ValueError as e: + raise TypeError from e + super().setstate(internalstate) + else: + raise ValueError("state with version %s passed to " + "Random.setstate() of version %s" % + (version, self.VERSION)) + + + ## ------------------------------------------------------- + ## ---- Methods below this point do not need to be overridden or extended + ## ---- when subclassing for the purpose of using a different core generator. + + + ## -------------------- pickle support ------------------- + + # Issue 17489: Since __reduce__ was defined to fix #759889 this is no + # longer called; we leave it here because it has been here since random was + # rewritten back in 2001 and why risk breaking something. + def __getstate__(self): # for pickle + return self.getstate() + + def __setstate__(self, state): # for pickle + self.setstate(state) + + def __reduce__(self): + return self.__class__, (), self.getstate() + + + ## ---- internal support method for evenly distributed integers ---- + + def __init_subclass__(cls, /, **kwargs): + """Control how subclasses generate random integers. + + The algorithm a subclass can use depends on the random() and/or + getrandbits() implementation available to it and determines + whether it can generate random integers from arbitrarily large + ranges. + """ + + for c in cls.__mro__: + if '_randbelow' in c.__dict__: + # just inherit it + break + if 'getrandbits' in c.__dict__: + cls._randbelow = cls._randbelow_with_getrandbits + break + if 'random' in c.__dict__: + cls._randbelow = cls._randbelow_without_getrandbits + break + + def _randbelow_with_getrandbits(self, n): + "Return a random int in the range [0,n). Defined for n > 0." + + getrandbits = self.getrandbits + k = n.bit_length() + r = getrandbits(k) # 0 <= r < 2**k + while r >= n: + r = getrandbits(k) + return r + + def _randbelow_without_getrandbits(self, n, maxsize=1< 0. + + The implementation does not use getrandbits, but only random. + """ + + random = self.random + if n >= maxsize: + from warnings import warn + warn("Underlying random() generator does not supply \n" + "enough bits to choose from a population range this large.\n" + "To remove the range limitation, add a getrandbits() method.") + return _floor(random() * n) + rem = maxsize % n + limit = (maxsize - rem) / maxsize # int(limit * maxsize) % n == 0 + r = random() + while r >= limit: + r = random() + return _floor(r * maxsize) % n + + _randbelow = _randbelow_with_getrandbits + + + ## -------------------------------------------------------- + ## ---- Methods below this point generate custom distributions + ## ---- based on the methods defined above. They do not + ## ---- directly touch the underlying generator and only + ## ---- access randomness through the methods: random(), + ## ---- getrandbits(), or _randbelow(). + + + ## -------------------- bytes methods --------------------- + + def randbytes(self, n): + """Generate n random bytes.""" + return self.getrandbits(n * 8).to_bytes(n, 'little') + + + ## -------------------- integer methods ------------------- + + def randrange(self, start, stop=None, step=_ONE): + """Choose a random item from range(stop) or range(start, stop[, step]). + + Roughly equivalent to ``choice(range(start, stop, step))`` but + supports arbitrarily large ranges and is optimized for common cases. + + """ + + # This code is a bit messy to make it fast for the + # common case while still doing adequate error checking. + istart = _index(start) + if stop is None: + # We don't check for "step != 1" because it hasn't been + # type checked and converted to an integer yet. + if step is not _ONE: + raise TypeError("Missing a non-None stop argument") + if istart > 0: + return self._randbelow(istart) + raise ValueError("empty range for randrange()") + + # Stop argument supplied. + istop = _index(stop) + width = istop - istart + istep = _index(step) + # Fast path. + if istep == 1: + if width > 0: + return istart + self._randbelow(width) + raise ValueError(f"empty range in randrange({start}, {stop})") + + # Non-unit step argument supplied. + if istep > 0: + n = (width + istep - 1) // istep + elif istep < 0: + n = (width + istep + 1) // istep + else: + raise ValueError("zero step for randrange()") + if n <= 0: + raise ValueError(f"empty range in randrange({start}, {stop}, {step})") + return istart + istep * self._randbelow(n) + + def randint(self, a, b): + """Return random integer in range [a, b], including both end points. + """ + + return self.randrange(a, b+1) + + + ## -------------------- sequence methods ------------------- + + def choice(self, seq): + """Choose a random element from a non-empty sequence.""" + + # As an accommodation for NumPy, we don't use "if not seq" + # because bool(numpy.array()) raises a ValueError. + if not len(seq): + raise IndexError('Cannot choose from an empty sequence') + return seq[self._randbelow(len(seq))] + + def shuffle(self, x): + """Shuffle list x in place, and return None.""" + + randbelow = self._randbelow + for i in reversed(range(1, len(x))): + # pick an element in x[:i+1] with which to exchange x[i] + j = randbelow(i + 1) + x[i], x[j] = x[j], x[i] + + def sample(self, population, k, *, counts=None): + """Chooses k unique random elements from a population sequence. + + Returns a new list containing elements from the population while + leaving the original population unchanged. The resulting list is + in selection order so that all sub-slices will also be valid random + samples. This allows raffle winners (the sample) to be partitioned + into grand prize and second place winners (the subslices). + + Members of the population need not be hashable or unique. If the + population contains repeats, then each occurrence is a possible + selection in the sample. + + Repeated elements can be specified one at a time or with the optional + counts parameter. For example: + + sample(['red', 'blue'], counts=[4, 2], k=5) + + is equivalent to: + + sample(['red', 'red', 'red', 'red', 'blue', 'blue'], k=5) + + To choose a sample from a range of integers, use range() for the + population argument. This is especially fast and space efficient + for sampling from a large population: + + sample(range(10000000), 60) + + """ + + # Sampling without replacement entails tracking either potential + # selections (the pool) in a list or previous selections in a set. + + # When the number of selections is small compared to the + # population, then tracking selections is efficient, requiring + # only a small set and an occasional reselection. For + # a larger number of selections, the pool tracking method is + # preferred since the list takes less space than the + # set and it doesn't suffer from frequent reselections. + + # The number of calls to _randbelow() is kept at or near k, the + # theoretical minimum. This is important because running time + # is dominated by _randbelow() and because it extracts the + # least entropy from the underlying random number generators. + + # Memory requirements are kept to the smaller of a k-length + # set or an n-length list. + + # There are other sampling algorithms that do not require + # auxiliary memory, but they were rejected because they made + # too many calls to _randbelow(), making them slower and + # causing them to eat more entropy than necessary. + + if not isinstance(population, _Sequence): + raise TypeError("Population must be a sequence. " + "For dicts or sets, use sorted(d).") + n = len(population) + if counts is not None: + cum_counts = list(_accumulate(counts)) + if len(cum_counts) != n: + raise ValueError('The number of counts does not match the population') + total = cum_counts.pop() if cum_counts else 0 + if not isinstance(total, int): + raise TypeError('Counts must be integers') + if total < 0: + raise ValueError('Counts must be non-negative') + selections = self.sample(range(total), k=k) + bisect = _bisect + return [population[bisect(cum_counts, s)] for s in selections] + randbelow = self._randbelow + if not 0 <= k <= n: + raise ValueError("Sample larger than population or is negative") + result = [None] * k + setsize = 21 # size of a small set minus size of an empty list + if k > 5: + setsize += 4 ** _ceil(_log(k * 3, 4)) # table size for big sets + if n <= setsize: + # An n-length list is smaller than a k-length set. + # Invariant: non-selected at pool[0 : n-i] + pool = list(population) + for i in range(k): + j = randbelow(n - i) + result[i] = pool[j] + pool[j] = pool[n - i - 1] # move non-selected item into vacancy + else: + selected = set() + selected_add = selected.add + for i in range(k): + j = randbelow(n) + while j in selected: + j = randbelow(n) + selected_add(j) + result[i] = population[j] + return result + + def choices(self, population, weights=None, *, cum_weights=None, k=1): + """Return a k sized list of population elements chosen with replacement. + + If the relative weights or cumulative weights are not specified, + the selections are made with equal probability. + + """ + random = self.random + n = len(population) + if cum_weights is None: + if weights is None: + floor = _floor + n += 0.0 # convert to float for a small speed improvement + return [population[floor(random() * n)] for i in _repeat(None, k)] + try: + cum_weights = list(_accumulate(weights)) + except TypeError: + if not isinstance(weights, int): + raise + k = weights + raise TypeError( + f'The number of choices must be a keyword argument: {k=}' + ) from None + elif weights is not None: + raise TypeError('Cannot specify both weights and cumulative weights') + if len(cum_weights) != n: + raise ValueError('The number of weights does not match the population') + total = cum_weights[-1] + 0.0 # convert to float + if total <= 0.0: + raise ValueError('Total of weights must be greater than zero') + if not _isfinite(total): + raise ValueError('Total of weights must be finite') + bisect = _bisect + hi = n - 1 + return [population[bisect(cum_weights, random() * total, 0, hi)] + for i in _repeat(None, k)] + + + ## -------------------- real-valued distributions ------------------- + + def uniform(self, a, b): + """Get a random number in the range [a, b) or [a, b] depending on rounding. + + The mean (expected value) and variance of the random variable are: + + E[X] = (a + b) / 2 + Var[X] = (b - a) ** 2 / 12 + + """ + return a + (b - a) * self.random() + + def triangular(self, low=0.0, high=1.0, mode=None): + """Triangular distribution. + + Continuous distribution bounded by given lower and upper limits, + and having a given mode value in-between. + + http://en.wikipedia.org/wiki/Triangular_distribution + + The mean (expected value) and variance of the random variable are: + + E[X] = (low + high + mode) / 3 + Var[X] = (low**2 + high**2 + mode**2 - low*high - low*mode - high*mode) / 18 + + """ + u = self.random() + try: + c = 0.5 if mode is None else (mode - low) / (high - low) + except ZeroDivisionError: + return low + if u > c: + u = 1.0 - u + c = 1.0 - c + low, high = high, low + return low + (high - low) * _sqrt(u * c) + + def normalvariate(self, mu=0.0, sigma=1.0): + """Normal distribution. + + mu is the mean, and sigma is the standard deviation. + + """ + # Uses Kinderman and Monahan method. Reference: Kinderman, + # A.J. and Monahan, J.F., "Computer generation of random + # variables using the ratio of uniform deviates", ACM Trans + # Math Software, 3, (1977), pp257-260. + + random = self.random + while True: + u1 = random() + u2 = 1.0 - random() + z = NV_MAGICCONST * (u1 - 0.5) / u2 + zz = z * z / 4.0 + if zz <= -_log(u2): + break + return mu + z * sigma + + def gauss(self, mu=0.0, sigma=1.0): + """Gaussian distribution. + + mu is the mean, and sigma is the standard deviation. This is + slightly faster than the normalvariate() function. + + Not thread-safe without a lock around calls. + + """ + # When x and y are two variables from [0, 1), uniformly + # distributed, then + # + # cos(2*pi*x)*sqrt(-2*log(1-y)) + # sin(2*pi*x)*sqrt(-2*log(1-y)) + # + # are two *independent* variables with normal distribution + # (mu = 0, sigma = 1). + # (Lambert Meertens) + # (corrected version; bug discovered by Mike Miller, fixed by LM) + + # Multithreading note: When two threads call this function + # simultaneously, it is possible that they will receive the + # same return value. The window is very small though. To + # avoid this, you have to use a lock around all calls. (I + # didn't want to slow this down in the serial case by using a + # lock here.) + + random = self.random + z = self.gauss_next + self.gauss_next = None + if z is None: + x2pi = random() * TWOPI + g2rad = _sqrt(-2.0 * _log(1.0 - random())) + z = _cos(x2pi) * g2rad + self.gauss_next = _sin(x2pi) * g2rad + + return mu + z * sigma + + def lognormvariate(self, mu, sigma): + """Log normal distribution. + + If you take the natural logarithm of this distribution, you'll get a + normal distribution with mean mu and standard deviation sigma. + mu can have any value, and sigma must be greater than zero. + + """ + return _exp(self.normalvariate(mu, sigma)) + + def expovariate(self, lambd=1.0): + """Exponential distribution. + + lambd is 1.0 divided by the desired mean. It should be + nonzero. (The parameter would be called "lambda", but that is + a reserved word in Python.) Returned values range from 0 to + positive infinity if lambd is positive, and from negative + infinity to 0 if lambd is negative. + + The mean (expected value) and variance of the random variable are: + + E[X] = 1 / lambd + Var[X] = 1 / lambd ** 2 + + """ + # we use 1-random() instead of random() to preclude the + # possibility of taking the log of zero. + + return -_log(1.0 - self.random()) / lambd + + def vonmisesvariate(self, mu, kappa): + """Circular data distribution. + + mu is the mean angle, expressed in radians between 0 and 2*pi, and + kappa is the concentration parameter, which must be greater than or + equal to zero. If kappa is equal to zero, this distribution reduces + to a uniform random angle over the range 0 to 2*pi. + + """ + # Based upon an algorithm published in: Fisher, N.I., + # "Statistical Analysis of Circular Data", Cambridge + # University Press, 1993. + + # Thanks to Magnus Kessler for a correction to the + # implementation of step 4. + + random = self.random + if kappa <= 1e-6: + return TWOPI * random() + + s = 0.5 / kappa + r = s + _sqrt(1.0 + s * s) + + while True: + u1 = random() + z = _cos(_pi * u1) + + d = z / (r + z) + u2 = random() + if u2 < 1.0 - d * d or u2 <= (1.0 - d) * _exp(d): + break + + q = 1.0 / r + f = (q + z) / (1.0 + q * z) + u3 = random() + if u3 > 0.5: + theta = (mu + _acos(f)) % TWOPI + else: + theta = (mu - _acos(f)) % TWOPI + + return theta + + def gammavariate(self, alpha, beta): + """Gamma distribution. Not the gamma function! + + Conditions on the parameters are alpha > 0 and beta > 0. + + The probability distribution function is: + + x ** (alpha - 1) * math.exp(-x / beta) + pdf(x) = -------------------------------------- + math.gamma(alpha) * beta ** alpha + + The mean (expected value) and variance of the random variable are: + + E[X] = alpha * beta + Var[X] = alpha * beta ** 2 + + """ + + # Warning: a few older sources define the gamma distribution in terms + # of alpha > -1.0 + if alpha <= 0.0 or beta <= 0.0: + raise ValueError('gammavariate: alpha and beta must be > 0.0') + + random = self.random + if alpha > 1.0: + + # Uses R.C.H. Cheng, "The generation of Gamma + # variables with non-integral shape parameters", + # Applied Statistics, (1977), 26, No. 1, p71-74 + + ainv = _sqrt(2.0 * alpha - 1.0) + bbb = alpha - LOG4 + ccc = alpha + ainv + + while True: + u1 = random() + if not 1e-7 < u1 < 0.9999999: + continue + u2 = 1.0 - random() + v = _log(u1 / (1.0 - u1)) / ainv + x = alpha * _exp(v) + z = u1 * u1 * u2 + r = bbb + ccc * v - x + if r + SG_MAGICCONST - 4.5 * z >= 0.0 or r >= _log(z): + return x * beta + + elif alpha == 1.0: + # expovariate(1/beta) + return -_log(1.0 - random()) * beta + + else: + # alpha is between 0 and 1 (exclusive) + # Uses ALGORITHM GS of Statistical Computing - Kennedy & Gentle + while True: + u = random() + b = (_e + alpha) / _e + p = b * u + if p <= 1.0: + x = p ** (1.0 / alpha) + else: + x = -_log((b - p) / alpha) + u1 = random() + if p > 1.0: + if u1 <= x ** (alpha - 1.0): + break + elif u1 <= _exp(-x): + break + return x * beta + + def betavariate(self, alpha, beta): + """Beta distribution. + + Conditions on the parameters are alpha > 0 and beta > 0. + Returned values range between 0 and 1. + + The mean (expected value) and variance of the random variable are: + + E[X] = alpha / (alpha + beta) + Var[X] = alpha * beta / ((alpha + beta)**2 * (alpha + beta + 1)) + + """ + ## See + ## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html + ## for Ivan Frohne's insightful analysis of why the original implementation: + ## + ## def betavariate(self, alpha, beta): + ## # Discrete Event Simulation in C, pp 87-88. + ## + ## y = self.expovariate(alpha) + ## z = self.expovariate(1.0/beta) + ## return z/(y+z) + ## + ## was dead wrong, and how it probably got that way. + + # This version due to Janne Sinkkonen, and matches all the std + # texts (e.g., Knuth Vol 2 Ed 3 pg 134 "the beta distribution"). + y = self.gammavariate(alpha, 1.0) + if y: + return y / (y + self.gammavariate(beta, 1.0)) + return 0.0 + + def paretovariate(self, alpha): + """Pareto distribution. alpha is the shape parameter.""" + # Jain, pg. 495 + + u = 1.0 - self.random() + return u ** (-1.0 / alpha) + + def weibullvariate(self, alpha, beta): + """Weibull distribution. + + alpha is the scale parameter and beta is the shape parameter. + + """ + # Jain, pg. 499; bug fix courtesy Bill Arms + + u = 1.0 - self.random() + return alpha * (-_log(u)) ** (1.0 / beta) + + + ## -------------------- discrete distributions --------------------- + + def binomialvariate(self, n=1, p=0.5): + """Binomial random variable. + + Gives the number of successes for *n* independent trials + with the probability of success in each trial being *p*: + + sum(random() < p for i in range(n)) + + Returns an integer in the range: 0 <= X <= n + + The mean (expected value) and variance of the random variable are: + + E[X] = n * p + Var[x] = n * p * (1 - p) + + """ + # Error check inputs and handle edge cases + if n < 0: + raise ValueError("n must be non-negative") + if p <= 0.0 or p >= 1.0: + if p == 0.0: + return 0 + if p == 1.0: + return n + raise ValueError("p must be in the range 0.0 <= p <= 1.0") + + random = self.random + + # Fast path for a common case + if n == 1: + return _index(random() < p) + + # Exploit symmetry to establish: p <= 0.5 + if p > 0.5: + return n - self.binomialvariate(n, 1.0 - p) + + if n * p < 10.0: + # BG: Geometric method by Devroye with running time of O(np). + # https://dl.acm.org/doi/pdf/10.1145/42372.42381 + x = y = 0 + c = _log2(1.0 - p) + if not c: + return x + while True: + y += _floor(_log2(random()) / c) + 1 + if y > n: + return x + x += 1 + + # BTRS: Transformed rejection with squeeze method by Wolfgang Hörmann + # https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.47.8407&rep=rep1&type=pdf + assert n*p >= 10.0 and p <= 0.5 + setup_complete = False + + spq = _sqrt(n * p * (1.0 - p)) # Standard deviation of the distribution + b = 1.15 + 2.53 * spq + a = -0.0873 + 0.0248 * b + 0.01 * p + c = n * p + 0.5 + vr = 0.92 - 4.2 / b + + while True: + + u = random() + u -= 0.5 + us = 0.5 - _fabs(u) + k = _floor((2.0 * a / us + b) * u + c) + if k < 0 or k > n: + continue + + # The early-out "squeeze" test substantially reduces + # the number of acceptance condition evaluations. + v = random() + if us >= 0.07 and v <= vr: + return k + + # Acceptance-rejection test. + # Note, the original paper erroneously omits the call to log(v) + # when comparing to the log of the rescaled binomial distribution. + if not setup_complete: + alpha = (2.83 + 5.1 / b) * spq + lpq = _log(p / (1.0 - p)) + m = _floor((n + 1) * p) # Mode of the distribution + h = _lgamma(m + 1) + _lgamma(n - m + 1) + setup_complete = True # Only needs to be done once + v *= alpha / (a / (us * us) + b) + if _log(v) <= h - _lgamma(k + 1) - _lgamma(n - k + 1) + (k - m) * lpq: + return k + + +## ------------------------------------------------------------------ +## --------------- Operating System Random Source ------------------ + + +class SystemRandom(Random): + """Alternate random number generator using sources provided + by the operating system (such as /dev/urandom on Unix or + CryptGenRandom on Windows). + + Not available on all systems (see os.urandom() for details). + + """ + + def random(self): + """Get the next random number in the range 0.0 <= X < 1.0.""" + return (int.from_bytes(_urandom(7)) >> 3) * RECIP_BPF + + def getrandbits(self, k): + """getrandbits(k) -> x. Generates an int with k random bits.""" + if k < 0: + raise ValueError('number of bits must be non-negative') + numbytes = (k + 7) // 8 # bits / 8 and rounded up + x = int.from_bytes(_urandom(numbytes)) + return x >> (numbytes * 8 - k) # trim excess bits + + def randbytes(self, n): + """Generate n random bytes.""" + # os.urandom(n) fails with ValueError for n < 0 + # and returns an empty bytes string for n == 0. + return _urandom(n) + + def seed(self, *args, **kwds): + "Stub method. Not used for a system random number generator." + return None + + def _notimplemented(self, *args, **kwds): + "Method should not be called for a system random number generator." + raise NotImplementedError('System entropy source does not have state.') + getstate = setstate = _notimplemented + + +# ---------------------------------------------------------------------- +# Create one instance, seeded from current time, and export its methods +# as module-level functions. The functions share state across all uses +# (both in the user's code and in the Python libraries), but that's fine +# for most programs and is easier for the casual user than making them +# instantiate their own Random() instance. + +_inst = Random() +seed = _inst.seed +random = _inst.random +uniform = _inst.uniform +triangular = _inst.triangular +randint = _inst.randint +choice = _inst.choice +randrange = _inst.randrange +sample = _inst.sample +shuffle = _inst.shuffle +choices = _inst.choices +normalvariate = _inst.normalvariate +lognormvariate = _inst.lognormvariate +expovariate = _inst.expovariate +vonmisesvariate = _inst.vonmisesvariate +gammavariate = _inst.gammavariate +gauss = _inst.gauss +betavariate = _inst.betavariate +binomialvariate = _inst.binomialvariate +paretovariate = _inst.paretovariate +weibullvariate = _inst.weibullvariate +getstate = _inst.getstate +setstate = _inst.setstate +getrandbits = _inst.getrandbits +randbytes = _inst.randbytes + + +## ------------------------------------------------------ +## ----------------- test program ----------------------- + +def _test_generator(n, func, args): + from statistics import stdev, fmean as mean + from time import perf_counter + + t0 = perf_counter() + data = [func(*args) for i in _repeat(None, n)] + t1 = perf_counter() + + xbar = mean(data) + sigma = stdev(data, xbar) + low = min(data) + high = max(data) + + print(f'{t1 - t0:.3f} sec, {n} times {func.__name__}{args!r}') + print('avg %g, stddev %g, min %g, max %g\n' % (xbar, sigma, low, high)) + + +def _test(N=10_000): + _test_generator(N, random, ()) + _test_generator(N, normalvariate, (0.0, 1.0)) + _test_generator(N, lognormvariate, (0.0, 1.0)) + _test_generator(N, vonmisesvariate, (0.0, 1.0)) + _test_generator(N, binomialvariate, (15, 0.60)) + _test_generator(N, binomialvariate, (100, 0.75)) + _test_generator(N, gammavariate, (0.01, 1.0)) + _test_generator(N, gammavariate, (0.1, 1.0)) + _test_generator(N, gammavariate, (0.1, 2.0)) + _test_generator(N, gammavariate, (0.5, 1.0)) + _test_generator(N, gammavariate, (0.9, 1.0)) + _test_generator(N, gammavariate, (1.0, 1.0)) + _test_generator(N, gammavariate, (2.0, 1.0)) + _test_generator(N, gammavariate, (20.0, 1.0)) + _test_generator(N, gammavariate, (200.0, 1.0)) + _test_generator(N, gauss, (0.0, 1.0)) + _test_generator(N, betavariate, (3.0, 3.0)) + _test_generator(N, triangular, (0.0, 1.0, 1.0 / 3.0)) + + +## ------------------------------------------------------ +## ------------------ fork support --------------------- + +if hasattr(_os, "fork"): + _os.register_at_fork(after_in_child=_inst.seed) + + +# ------------------------------------------------------ +# -------------- command-line interface ---------------- + + +def _parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-c", "--choice", nargs="+", + help="print a random choice") + group.add_argument( + "-i", "--integer", type=int, metavar="N", + help="print a random integer between 1 and N inclusive") + group.add_argument( + "-f", "--float", type=float, metavar="N", + help="print a random floating-point number between 0 and N inclusive") + group.add_argument( + "--test", type=int, const=10_000, nargs="?", + help=argparse.SUPPRESS) + parser.add_argument("input", nargs="*", + help="""\ +if no options given, output depends on the input + string or multiple: same as --choice + integer: same as --integer + float: same as --float""") + args = parser.parse_args(arg_list) + return args, parser.format_help() + + +def main(arg_list: list[str] | None = None) -> int | str: + args, help_text = _parse_args(arg_list) + + # Explicit arguments + if args.choice: + return choice(args.choice) + + if args.integer is not None: + return randint(1, args.integer) + + if args.float is not None: + return uniform(0, args.float) + + if args.test: + _test(args.test) + return "" + + # No explicit argument, select based on input + if len(args.input) == 1: + val = args.input[0] + try: + # Is it an integer? + val = int(val) + return randint(1, val) + except ValueError: + try: + # Is it a float? + val = float(val) + return uniform(0, val) + except ValueError: + # Split in case of space-separated string: "a b c" + return choice(val.split()) + + if len(args.input) >= 2: + return choice(args.input) + + return help_text + + +if __name__ == '__main__': + print(main()) diff --git a/crates/weavepy-vm/src/stdlib/python/traceback.py b/crates/weavepy-vm/src/stdlib/python/traceback.py index fbfc8cb..7bd0d35 100644 --- a/crates/weavepy-vm/src/stdlib/python/traceback.py +++ b/crates/weavepy-vm/src/stdlib/python/traceback.py @@ -155,18 +155,58 @@ def __init__(self, frames=None): def append(self, frame): self._frames.append(frame) + def extend(self, frames): + self._frames.extend(frames) + + def insert(self, index, frame): + self._frames.insert(index, frame) + + def pop(self, index=-1): + return self._frames.pop(index) + + def remove(self, frame): + self._frames.remove(frame) + + def reverse(self): + self._frames.reverse() + + def count(self, frame): + return self._frames.count(frame) + + def index(self, frame, *args): + return self._frames.index(frame, *args) + def __len__(self): return len(self._frames) def __iter__(self): return iter(self._frames) + def __reversed__(self): + return reversed(self._frames) + + def __contains__(self, frame): + return frame in self._frames + def __getitem__(self, index): return self._frames[index] + def __setitem__(self, index, value): + self._frames[index] = value + + def __delitem__(self, index): + del self._frames[index] + def __bool__(self): return bool(self._frames) + def __eq__(self, other): + if isinstance(other, StackSummary): + return self._frames == other._frames + if isinstance(other, list): + return self._frames == other + return NotImplemented + @classmethod def extract(cls, frame_gen, *, limit=None, lookup_lines=True, capture_locals=False): # `frame_gen` yields plain (frame, lineno) pairs (no column info). diff --git a/crates/weavepy-vm/src/stdlib/python/types_mod.py b/crates/weavepy-vm/src/stdlib/python/types_mod.py index fa378c3..53b6404 100644 --- a/crates/weavepy-vm/src/stdlib/python/types_mod.py +++ b/crates/weavepy-vm/src/stdlib/python/types_mod.py @@ -277,11 +277,88 @@ def deleter(self, fdel): def coroutine(func): """Mark a generator function so it can be used with `await`. - CPython's full implementation rewires the function's flags; here we - simply return the function unchanged. Most generator-based - coroutines in modern code use ``async def`` directly. + Mirrors CPython's implementation: a generator function gets + `CO_ITERABLE_COROUTINE` set on its code (done natively); other + callables are wrapped so generator results are awaitable. """ - return func + if not callable(func): + raise TypeError('types.coroutine() expects a callable') + + co = getattr(func, '__code__', None) + flags = getattr(co, 'co_flags', None) + if flags is not None: + # Already a coroutine function or already marked: no-op. + if flags & 0x180: # CO_COROUTINE | CO_ITERABLE_COROUTINE + return func + if flags & 0x20: # CO_GENERATOR + return _weavepy_mark_iterable_coroutine(func) + + import functools as _functools + + @_functools.wraps(func) + def wrapped(*args, **kwargs): + coro = func(*args, **kwargs) + cls_name = type(coro).__name__ + if cls_name == 'coroutine' or ( + getattr(coro, 'gi_code', None) is not None + and coro.gi_code.co_flags & 0x180 + ): + return coro + if cls_name == 'generator': + return _GeneratorWrapper(coro) + return coro + + return wrapped + + +class _GeneratorWrapper: + """Adapt a plain generator into an awaitable (CPython types.py).""" + + def __init__(self, gen): + self.__wrapped = gen + self.__isgen = type(gen).__name__ == 'generator' + self.__name__ = getattr(gen, '__name__', None) + self.__qualname__ = getattr(gen, '__qualname__', None) + + def send(self, val): + return self.__wrapped.send(val) + + def throw(self, tp, *rest): + return self.__wrapped.throw(tp, *rest) + + def close(self): + return self.__wrapped.close() + + @property + def gi_code(self): + return self.__wrapped.gi_code + + @property + def gi_frame(self): + return self.__wrapped.gi_frame + + @property + def gi_running(self): + return self.__wrapped.gi_running + + @property + def gi_yieldfrom(self): + return self.__wrapped.gi_yieldfrom + + cr_code = gi_code + cr_frame = gi_frame + cr_running = gi_running + cr_await = gi_yieldfrom + + def __next__(self): + return next(self.__wrapped) + + def __iter__(self): + if self.__isgen: + return self.__wrapped + return self + + __await__ = __iter__ def resolve_bases(bases): diff --git a/crates/weavepy-vm/src/stdlib/random.rs b/crates/weavepy-vm/src/stdlib/random.rs deleted file mode 100644 index b8c1dc3..0000000 --- a/crates/weavepy-vm/src/stdlib/random.rs +++ /dev/null @@ -1,407 +0,0 @@ -//! The `random` built-in module. -//! -//! Implements the most-used functions on top of a small splitmix64 -//! generator. The interface matches CPython, but the underlying -//! algorithm differs — `random.seed(0); random.random()` therefore -//! produces different bits than CPython. The contract we make is -//! *statistical*: outputs are uniformly distributed. - -use crate::sync::Rc; -use crate::sync::RefCell; - -use crate::error::{type_error, value_error, RuntimeError}; -use crate::import::ModuleCache; -use crate::object::{BuiltinFn, DictData, DictKey, Object, PyModule}; - -/// Splitmix64 — small, fast, good enough for everyday Python code. -struct Rng { - state: u64, -} - -impl Rng { - fn new(seed: u64) -> Self { - Self { state: seed } - } - - fn next_u64(&mut self) -> u64 { - self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = self.state; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z ^ (z >> 31) - } - - fn next_double(&mut self) -> f64 { - let bits = self.next_u64() >> 11; - (bits as f64) / ((1u64 << 53) as f64) - } -} - -thread_local! { - static RNG: RefCell = RefCell::new(Rng::new(default_seed())); -} - -fn default_seed() -> u64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos() as u64) - .unwrap_or(0xDEAD_BEEF_DEAD_BEEF) - .max(1) -} - -pub fn build(_cache: &ModuleCache) -> Rc { - let dict = Rc::new(RefCell::new(DictData::new())); - { - let mut d = dict.borrow_mut(); - d.insert( - DictKey(Object::from_static("__name__")), - Object::from_static("random"), - ); - d.insert( - DictKey(Object::from_static("__doc__")), - Object::from_static("Pseudo-random number generators."), - ); - d.insert(DictKey(Object::from_static("seed")), b("seed", random_seed)); - d.insert( - DictKey(Object::from_static("random")), - b("random", random_random), - ); - d.insert( - DictKey(Object::from_static("uniform")), - b("uniform", random_uniform), - ); - d.insert( - DictKey(Object::from_static("randint")), - b("randint", random_randint), - ); - d.insert( - DictKey(Object::from_static("randrange")), - b("randrange", random_randrange), - ); - d.insert( - DictKey(Object::from_static("choice")), - b("choice", random_choice), - ); - d.insert( - DictKey(Object::from_static("choices")), - b("choices", random_choices), - ); - d.insert( - DictKey(Object::from_static("shuffle")), - b("shuffle", random_shuffle), - ); - d.insert( - DictKey(Object::from_static("sample")), - b("sample", random_sample), - ); - d.insert( - DictKey(Object::from_static("gauss")), - b("gauss", random_gauss), - ); - d.insert( - DictKey(Object::from_static("getrandbits")), - b("getrandbits", random_getrandbits), - ); - } - Rc::new(PyModule { - name: "random".to_owned(), - filename: None, - dict, - }) -} - -fn b(name: &'static str, body: fn(&[Object]) -> Result) -> Object { - Object::Builtin(Rc::new(BuiltinFn { - name, - call: Box::new(body), - call_kw: None, - })) -} - -fn random_seed(args: &[Object]) -> Result { - let seed = match args.first() { - Some(Object::Int(n)) => *n as u64, - Some(Object::None) | None => default_seed(), - _ => return Err(type_error("seed must be int or None")), - }; - RNG.with(|r| { - *r.borrow_mut() = Rng::new(seed.max(1)); - }); - Ok(Object::None) -} - -fn random_random(_args: &[Object]) -> Result { - let v = RNG.with(|r| r.borrow_mut().next_double()); - Ok(Object::Float(v)) -} - -/// Module-level `random.getrandbits(k)` — a non-negative int with `k` -/// random bits (`0 <= result < 2**k`), drawn from the module RNG. -fn random_getrandbits(args: &[Object]) -> Result { - use num_bigint::{BigInt, Sign}; - let k = match args.first() { - Some(Object::Bool(b)) => u64::from(*b), - Some(Object::Int(n)) if *n >= 0 => *n as u64, - Some(Object::Int(_)) => { - return Err(value_error("number of bits must be non-negative")) - } - _ => return Err(type_error("getrandbits() requires an integer argument")), - }; - if k == 0 { - return Ok(Object::Int(0)); - } - let nbytes = ((k + 7) / 8) as usize; - let excess = (nbytes as u64) * 8 - k; - let mut buf = vec![0u8; nbytes]; - RNG.with(|r| { - let mut rng = r.borrow_mut(); - let mut i = 0; - while i < nbytes { - let w = rng.next_u64().to_le_bytes(); - let take = (nbytes - i).min(8); - buf[i..i + take].copy_from_slice(&w[..take]); - i += take; - } - }); - if excess > 0 { - buf[nbytes - 1] &= 0xFFu8 >> excess; - } - Ok(Object::int_from_bigint(BigInt::from_bytes_le(Sign::Plus, &buf))) -} - -fn random_uniform(args: &[Object]) -> Result { - let a = to_f64(args.first())?; - let b = to_f64(args.get(1))?; - let r = RNG.with(|r| r.borrow_mut().next_double()); - Ok(Object::Float(a + (b - a) * r)) -} - -fn random_randint(args: &[Object]) -> Result { - let a = to_i64(args.first())?; - let b = to_i64(args.get(1))?; - if a > b { - return Err(value_error("randint: a must be <= b")); - } - let span = (b - a + 1) as u64; - let raw = RNG.with(|r| r.borrow_mut().next_u64()); - Ok(Object::Int(a + (raw % span) as i64)) -} - -/// Coerce a `randrange` bound to a `BigInt`, accepting any integer -/// (incl. arbitrary-precision) — CPython's `randrange` has no upper -/// bound on the magnitude of its arguments. -fn to_bigint(arg: Option<&Object>) -> Result { - use num_bigint::BigInt; - match arg { - Some(Object::Int(i)) => Ok(BigInt::from(*i)), - Some(Object::Bool(b)) => Ok(BigInt::from(i64::from(*b))), - Some(Object::Long(b)) => Ok((**b).clone()), - _ => Err(type_error("expected int")), - } -} - -/// Uniform random `BigInt` in `[0, n)` via rejection sampling on a -/// bit-masked candidate (`n` must be positive). Mirrors the shape of -/// CPython's `Random._randbelow` without depending on i64 width. -fn rand_below_bigint(n: &num_bigint::BigInt) -> num_bigint::BigInt { - use num_bigint::{BigInt, Sign}; - let bits = n.bits(); - if bits == 0 { - return BigInt::from(0); - } - let nbytes = ((bits + 7) / 8) as usize; - let excess = (nbytes as u64) * 8 - bits; - loop { - let mut buf = vec![0u8; nbytes]; - RNG.with(|r| { - let mut rng = r.borrow_mut(); - let mut i = 0; - while i < nbytes { - let w = rng.next_u64().to_le_bytes(); - let take = (nbytes - i).min(8); - buf[i..i + take].copy_from_slice(&w[..take]); - i += take; - } - }); - if excess > 0 { - buf[nbytes - 1] &= 0xFFu8 >> excess; - } - let cand = BigInt::from_bytes_le(Sign::Plus, &buf); - if &cand < n { - return cand; - } - } -} - -fn random_randrange(args: &[Object]) -> Result { - use num_bigint::BigInt; - use num_integer::Integer; - let zero = BigInt::from(0); - match args.len() { - 1 => { - let stop = to_bigint(args.first())?; - if stop <= zero { - return Err(value_error("empty range for randrange()")); - } - Ok(Object::int_from_bigint(rand_below_bigint(&stop))) - } - 2 => { - let start = to_bigint(args.first())?; - let stop = to_bigint(args.get(1))?; - let width = &stop - &start; - if width <= zero { - return Err(value_error("empty range for randrange()")); - } - Ok(Object::int_from_bigint(start + rand_below_bigint(&width))) - } - 3 => { - let start = to_bigint(args.first())?; - let stop = to_bigint(args.get(1))?; - let step = to_bigint(args.get(2))?; - if step == zero { - return Err(value_error("zero step for randrange()")); - } - let width = &stop - &start; - let one = BigInt::from(1); - // Count of reachable values: ceil(width/step), via floor div - // on the CPython-adjusted numerator (matches `range` length). - let n = if step > zero { - (&width + &step - &one).div_floor(&step) - } else { - (&width + &step + &one).div_floor(&step) - }; - if n <= zero { - return Err(value_error("empty range for randrange()")); - } - Ok(Object::int_from_bigint(start + step * rand_below_bigint(&n))) - } - _ => Err(type_error("randrange expects 1-3 args")), - } -} - -fn random_choice(args: &[Object]) -> Result { - let seq = args - .first() - .ok_or_else(|| type_error("choice expects a sequence"))?; - let items = sequence_items(seq)?; - if items.is_empty() { - return Err(value_error("choice from empty sequence")); - } - let raw = RNG.with(|r| r.borrow_mut().next_u64()); - let idx = (raw as usize) % items.len(); - Ok(items[idx].clone()) -} - -fn random_choices(args: &[Object]) -> Result { - let seq = args - .first() - .ok_or_else(|| type_error("choices expects a sequence"))?; - let items = sequence_items(seq)?; - if items.is_empty() { - return Err(value_error("choices from empty sequence")); - } - let k = match args.get(1) { - Some(Object::Int(n)) => *n as usize, - None => 1, - _ => return Err(type_error("k must be int")), - }; - let mut out = Vec::with_capacity(k); - for _ in 0..k { - let raw = RNG.with(|r| r.borrow_mut().next_u64()); - out.push(items[(raw as usize) % items.len()].clone()); - } - Ok(Object::new_list(out)) -} - -fn random_shuffle(args: &[Object]) -> Result { - let list = match args.first() { - Some(Object::List(l)) => l.clone(), - _ => return Err(type_error("shuffle expects a list")), - }; - let mut data = list.borrow_mut(); - let n = data.len(); - if n > 1 { - for i in (1..n).rev() { - let raw = RNG.with(|r| r.borrow_mut().next_u64()); - let j = (raw as usize) % (i + 1); - data.swap(i, j); - } - } - Ok(Object::None) -} - -fn random_sample(args: &[Object]) -> Result { - let seq = args - .first() - .ok_or_else(|| type_error("sample expects a sequence"))?; - let k = match args.get(1) { - Some(Object::Int(n)) => *n as usize, - _ => return Err(type_error("sample k must be int")), - }; - let mut items = sequence_items(seq)?; - if k > items.len() { - return Err(value_error("sample larger than population")); - } - let mut out = Vec::with_capacity(k); - for _ in 0..k { - let raw = RNG.with(|r| r.borrow_mut().next_u64()); - let idx = (raw as usize) % items.len(); - out.push(items.swap_remove(idx)); - } - Ok(Object::new_list(out)) -} - -fn random_gauss(args: &[Object]) -> Result { - let mu = to_f64(args.first())?; - let sigma = to_f64(args.get(1))?; - let (u1, u2) = RNG.with(|r| { - let mut r = r.borrow_mut(); - (r.next_double().max(f64::MIN_POSITIVE), r.next_double()) - }); - let mag = sigma * (-2.0 * u1.ln()).sqrt(); - let z = mag * (2.0 * std::f64::consts::PI * u2).cos(); - Ok(Object::Float(mu + z)) -} - -fn to_i64(arg: Option<&Object>) -> Result { - match arg { - Some(Object::Int(i)) => Ok(*i), - Some(Object::Bool(b)) => Ok(i64::from(*b)), - _ => Err(type_error("expected int")), - } -} - -fn to_f64(arg: Option<&Object>) -> Result { - match arg { - Some(Object::Int(i)) => Ok(*i as f64), - Some(Object::Float(f)) => Ok(*f), - Some(Object::Bool(b)) => Ok(i64::from(*b) as f64), - _ => Err(type_error("expected number")), - } -} - -fn sequence_items(obj: &Object) -> Result, RuntimeError> { - match obj { - Object::List(l) => Ok(l.borrow().clone()), - Object::Tuple(t) => Ok(t.to_vec()), - Object::Str(s) => Ok(s.chars().map(|c| Object::from_str(c.to_string())).collect()), - Object::Range(r) => { - let mut out = Vec::new(); - let mut i = r.start; - if r.step > 0 { - while i < r.stop { - out.push(Object::Int(i)); - i += r.step; - } - } else if r.step < 0 { - while i > r.stop { - out.push(Object::Int(i)); - i += r.step; - } - } - Ok(out) - } - _ => Err(type_error("expected a sequence")), - } -} diff --git a/crates/weavepy-vm/src/stdlib/random_core.rs b/crates/weavepy-vm/src/stdlib/random_core.rs index d9c11ad..09bd8ce 100644 --- a/crates/weavepy-vm/src/stdlib/random_core.rs +++ b/crates/weavepy-vm/src/stdlib/random_core.rs @@ -1,18 +1,25 @@ -//! The `_random` accelerator module — RFC 0023. +//! The `_random` accelerator module — RFC 0023 / RFC 0037. //! -//! Provides the Mersenne Twister state machine that the Python -//! `random` module wraps. We expose a `Random` class with the -//! methods that `random.py` uses internally: -//! `seed`, `random`, `getstate`, `setstate`, `getrandbits`. +//! A faithful port of CPython's `_randommodule.c`: the genuine +//! MT19937 Mersenne Twister, seeded with `init_by_array` exactly like +//! CPython, so `random.Random(42)` produces bit-identical streams. +//! The frozen pure-Python `random.py` (verbatim CPython) wraps this +//! class with the user-facing distribution API. use crate::sync::Rc; use crate::sync::RefCell; -use crate::error::{type_error, RuntimeError}; +use crate::error::{type_error, value_error, RuntimeError}; use crate::import::ModuleCache; use crate::object::{BuiltinFn, DictData, DictKey, Object, PyModule}; use crate::types::{PyInstance, TypeFlags, TypeObject}; +const N: usize = 624; +const M: usize = 397; +const MATRIX_A: u32 = 0x9908_b0df; +const UPPER_MASK: u32 = 0x8000_0000; +const LOWER_MASK: u32 = 0x7fff_ffff; + pub fn build(_cache: &ModuleCache) -> Rc { let dict = Rc::new(RefCell::new(DictData::new())); { @@ -45,6 +52,7 @@ fn random_type() -> Rc { ("seed", random_seed), ("random", random_random), ("getrandbits", random_getrandbits), + ("randbytes", random_randbytes), ("getstate", random_getstate), ("setstate", random_setstate), ] { @@ -69,71 +77,236 @@ fn random_type() -> Rc { .expect("Random type") } -/// Linear-congruential PRNG state. We use a 64-bit splitmix engine -/// because the full Mersenne Twister state (624 × 32-bit words) is -/// heavy to thread through `__dict__`. The distribution is uniform -/// over [0, 1) and good enough for non-cryptographic use — which -/// matches CPython's `random` module's contract. -fn current_state(inst: &Rc) -> u64 { +// =================================================================== +// MT19937 core (identical to CPython's `_randommodule.c`) +// =================================================================== + +struct Mt { + key: [u32; N], + pos: usize, +} + +impl Mt { + /// `init_genrand` — seed the state vector from a single u32. + fn init_genrand(s: u32) -> Self { + let mut key = [0u32; N]; + key[0] = s; + for i in 1..N { + key[i] = (1_812_433_253u32) + .wrapping_mul(key[i - 1] ^ (key[i - 1] >> 30)) + .wrapping_add(i as u32); + } + Mt { key, pos: N } + } + + /// `init_by_array` — seed from an arbitrary-length u32 key. + fn init_by_array(init_key: &[u32]) -> Self { + let mut mt = Self::init_genrand(19_650_218); + let key_length = init_key.len(); + let mut i: usize = 1; + let mut j: usize = 0; + let mut k = N.max(key_length); + while k > 0 { + mt.key[i] = (mt.key[i] + ^ (mt.key[i - 1] ^ (mt.key[i - 1] >> 30)).wrapping_mul(1_664_525)) + .wrapping_add(init_key[j]) + .wrapping_add(j as u32); + i += 1; + j += 1; + if i >= N { + mt.key[0] = mt.key[N - 1]; + i = 1; + } + if j >= key_length { + j = 0; + } + k -= 1; + } + k = N - 1; + while k > 0 { + mt.key[i] = (mt.key[i] + ^ (mt.key[i - 1] ^ (mt.key[i - 1] >> 30)).wrapping_mul(1_566_083_941)) + .wrapping_sub(i as u32); + i += 1; + if i >= N { + mt.key[0] = mt.key[N - 1]; + i = 1; + } + k -= 1; + } + mt.key[0] = 0x8000_0000; + mt + } + + /// `genrand_uint32` — the raw 32-bit output stream. + fn genrand_u32(&mut self) -> u32 { + if self.pos >= N { + // Regenerate the whole block. + for kk in 0..(N - M) { + let y = (self.key[kk] & UPPER_MASK) | (self.key[kk + 1] & LOWER_MASK); + self.key[kk] = + self.key[kk + M] ^ (y >> 1) ^ if y & 1 != 0 { MATRIX_A } else { 0 }; + } + for kk in (N - M)..(N - 1) { + let y = (self.key[kk] & UPPER_MASK) | (self.key[kk + 1] & LOWER_MASK); + self.key[kk] = self.key[kk + M - N] ^ (y >> 1) + ^ if y & 1 != 0 { MATRIX_A } else { 0 }; + } + let y = (self.key[N - 1] & UPPER_MASK) | (self.key[0] & LOWER_MASK); + self.key[N - 1] = + self.key[M - 1] ^ (y >> 1) ^ if y & 1 != 0 { MATRIX_A } else { 0 }; + self.pos = 0; + } + let mut y = self.key[self.pos]; + self.pos += 1; + y ^= y >> 11; + y ^= (y << 7) & 0x9d2c_5680; + y ^= (y << 15) & 0xefc6_0000; + y ^ (y >> 18) + } +} + +// =================================================================== +// Instance-state plumbing. The 624-word state lives in a bytearray in +// the instance dict (so Python-level subclasses share it), the cursor +// in an int. +// =================================================================== + +const STATE_KEY: &str = "_mt_state"; +const POS_KEY: &str = "_mt_pos"; + +fn self_instance(args: &[Object], what: &str) -> Result, RuntimeError> { + match args.first() { + Some(Object::Instance(i)) => Ok(i.clone()), + _ => Err(type_error(format!("{what} requires a _random.Random self"))), + } +} + +fn load_mt(inst: &Rc) -> Result { let dict = inst.dict.borrow(); - match dict.get(&DictKey(Object::from_static("_state"))) { - Some(Object::Int(i)) => *i as u64, - Some(Object::Long(b)) => { - use num_traits::ToPrimitive; - b.to_u64().unwrap_or(0xDEAD_BEEF) + let bytes = match dict.get(&DictKey(Object::from_static(STATE_KEY))) { + Some(Object::ByteArray(b)) => b.clone(), + _ => { + drop(dict); + // Unseeded use (e.g. subclass skipping __init__): seed from + // system entropy, as CPython does at allocation time. + let mt = seed_from_entropy(); + store_mt(inst, &mt); + return Ok(mt); } - _ => 0xDEAD_BEEF, + }; + let pos = match dict.get(&DictKey(Object::from_static(POS_KEY))) { + Some(Object::Int(i)) => *i as usize, + _ => N, + }; + let buf = bytes.borrow(); + let mut key = [0u32; N]; + for (i, chunk) in buf.chunks_exact(4).enumerate().take(N) { + key[i] = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); } + Ok(Mt { key, pos }) } -fn set_state(inst: &Rc, state: u64) { - inst.dict.borrow_mut().insert( - DictKey(Object::from_static("_state")), - Object::Int(state as i64), +fn store_mt(inst: &Rc, mt: &Mt) { + let mut buf = Vec::with_capacity(N * 4); + for w in &mt.key { + buf.extend_from_slice(&w.to_le_bytes()); + } + let mut dict = inst.dict.borrow_mut(); + dict.insert( + DictKey(Object::from_static(STATE_KEY)), + Object::ByteArray(Rc::new(RefCell::new(buf))), + ); + dict.insert( + DictKey(Object::from_static(POS_KEY)), + Object::Int(mt.pos as i64), ); } -fn splitmix64(state: &mut u64) -> u64 { - *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = *state; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z ^ (z >> 31) +/// Mutate-in-place fast path: run `f` against the deserialized state, +/// then persist the (changed) words back into the bytearray buffer. +fn with_mt( + inst: &Rc, + f: impl FnOnce(&mut Mt) -> R, +) -> Result { + let mut mt = load_mt(inst)?; + let r = f(&mut mt); + store_mt(inst, &mt); + Ok(r) } -fn random_init(args: &[Object]) -> Result { - let inst = match args.first() { - Some(Object::Instance(i)) => i.clone(), - _ => return Err(type_error("Random.__init__: missing self")), - }; - let seed = args.get(1).cloned().unwrap_or(Object::None); - let initial = match &seed { - Object::None => { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos() as u64) - .unwrap_or(0x1234_5678) - } - Object::Int(i) => *i as u64, - Object::Long(b) => { - use num_traits::ToPrimitive; - b.to_u64().unwrap_or(0xDEAD) - } - Object::Float(f) => f.to_bits(), +fn seed_from_entropy() -> Mt { + // CPython pulls 624 words from the OS urandom pool (falling back + // to time+pid). getrandom/urandom equivalents without new deps: + // hash system time and a process-unique counter through splitmix64. + use std::time::{SystemTime, UNIX_EPOCH}; + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0xDEAD_BEEF); + let pid = u64::from(std::process::id()); + let mut s = nanos ^ (pid << 32) ^ 0x9E37_79B9_7F4A_7C15; + let mut words = [0u32; N]; + for w in &mut words { + s = s.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = s; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + *w = (z ^ (z >> 31)) as u32; + } + Mt::init_by_array(&words) +} + +/// CPython `random_seed`: None → entropy; int → `init_by_array` over +/// the absolute value's 32-bit little-endian digits; floats and other +/// hashables are reduced like CPython reduces them (via hash) by the +/// pure-Python layer before they get here. +fn seed_from_object(arg: &Object) -> Result { + use num_bigint::BigInt; + use num_traits::Signed; + let n: BigInt = match arg { + Object::None => return Ok(seed_from_entropy()), + Object::Int(i) => BigInt::from(*i), + Object::Long(b) => (**b).clone(), + Object::Bool(b) => BigInt::from(i64::from(*b)), + Object::Float(f) => BigInt::from(f.to_bits()), Object::Str(s) => { - let mut h = 0u64; - for b in s.bytes() { - h = h.wrapping_mul(31).wrapping_add(u64::from(b)); - } - h + // Defensive: random.py normally converts str seeds to int + // first (sha512). Fall back to a stable byte fold. + BigInt::from_bytes_le(num_bigint::Sign::Plus, s.as_bytes()) + } + other => { + return Err(type_error(format!( + "cannot seed from '{}'", + other.type_name() + ))) } - other => other - .repr() - .bytes() - .fold(0u64, |h, b| h.wrapping_mul(31).wrapping_add(u64::from(b))), }; - set_state(&inst, initial); + let n = n.abs(); + let (_, bytes) = n.to_bytes_le(); + let mut words: Vec = bytes + .chunks(4) + .map(|c| { + let mut b = [0u8; 4]; + b[..c.len()].copy_from_slice(c); + u32::from_le_bytes(b) + }) + .collect(); + if words.is_empty() { + words.push(0); + } + Ok(Mt::init_by_array(&words)) +} + +// =================================================================== +// Methods +// =================================================================== + +fn random_init(args: &[Object]) -> Result { + let inst = self_instance(args, "Random.__init__")?; + let seed = args.get(1).cloned().unwrap_or(Object::None); + let mt = seed_from_object(&seed)?; + store_mt(&inst, &mt); Ok(Object::None) } @@ -141,84 +314,132 @@ fn random_seed(args: &[Object]) -> Result { random_init(args) } +/// `genrand_res53`: 53-bit resolution double in [0, 1). fn random_random(args: &[Object]) -> Result { - let inst = match args.first() { - Some(Object::Instance(i)) => i.clone(), - _ => return Err(type_error("random.random() requires self")), - }; - let mut s = current_state(&inst); - let v = splitmix64(&mut s); - set_state(&inst, s); - // Mantissa-only 53-bit fraction in [0, 1). - let frac = (v >> 11) as f64 / (1u64 << 53) as f64; - Ok(Object::Float(frac)) + let inst = self_instance(args, "random()")?; + let v = with_mt(&inst, |mt| { + let a = mt.genrand_u32() >> 5; + let b = mt.genrand_u32() >> 6; + (f64::from(a) * 67_108_864.0 + f64::from(b)) * (1.0 / 9_007_199_254_740_992.0) + })?; + Ok(Object::Float(v)) } +/// `getrandbits(k)`: k random bits as a non-negative int, assembled +/// from 32-bit words little-endian with the *last* word truncated — +/// CPython's exact layout, which `random.py`'s `_randbelow` depends on. fn random_getrandbits(args: &[Object]) -> Result { - let inst = match args.first() { - Some(Object::Instance(i)) => i.clone(), - _ => return Err(type_error("getrandbits() requires self")), - }; + use num_bigint::{BigUint, Sign}; + let inst = self_instance(args, "getrandbits()")?; let k = match args.get(1) { - Some(Object::Int(k)) if *k >= 0 => *k as u32, - _ => { - return Err(type_error( - "getrandbits() argument must be non-negative int", - )) + Some(Object::Bool(b)) => i64::from(*b), + Some(Object::Int(i)) => *i, + Some(Object::Long(b)) => { + use num_traits::ToPrimitive; + b.to_i64().ok_or_else(|| { + value_error("number of bits is too large") + })? } + _ => return Err(type_error("getrandbits() requires an integer argument")), }; + if k < 0 { + return Err(value_error("number of bits must be non-negative")); + } if k == 0 { return Ok(Object::Int(0)); } - let mut state = current_state(&inst); - let mut remaining = k; - let mut result_lo: u128 = 0; - let mut shift = 0u32; - while remaining > 0 { - let take = remaining.min(64); - let v = splitmix64(&mut state); - let mask: u64 = if take == 64 { !0 } else { (1u64 << take) - 1 }; - result_lo |= u128::from(v & mask) << shift; - shift += take; - remaining -= take; - if shift >= 128 { - break; - } + let k = k as u64; + if k <= 32 { + let v = with_mt(&inst, |mt| mt.genrand_u32())? >> (32 - k as u32); + return Ok(Object::Int(i64::from(v))); } - set_state(&inst, state); - if let Ok(small) = i64::try_from(result_lo) { - Ok(Object::Int(small)) - } else { - Ok(Object::int_from_bigint(num_bigint::BigInt::from(result_lo))) + let words = ((k - 1) / 32 + 1) as usize; + let digits = with_mt(&inst, |mt| { + let mut out = Vec::with_capacity(words); + let mut remaining = k; + for _ in 0..words { + let mut r = mt.genrand_u32(); + if remaining < 32 { + r >>= 32 - remaining as u32; + } + out.push(r); + remaining = remaining.saturating_sub(32); + } + out + })?; + let mut bytes = Vec::with_capacity(words * 4); + for d in &digits { + bytes.extend_from_slice(&d.to_le_bytes()); } + let big = BigUint::from_bytes_le(&bytes); + Ok(Object::int_from_bigint(num_bigint::BigInt::from_biguint( + Sign::Plus, + big, + ))) } -fn random_getstate(args: &[Object]) -> Result { - let inst = match args.first() { - Some(Object::Instance(i)) => i.clone(), - _ => return Err(type_error("getstate() requires self")), +/// `randbytes(n)` — CPython implements this on the C class. +fn random_randbytes(args: &[Object]) -> Result { + let inst = self_instance(args, "randbytes()")?; + let n = match args.get(1) { + Some(Object::Int(i)) if *i >= 0 => *i as usize, + Some(Object::Int(_)) => return Err(value_error("negative argument not allowed")), + _ => return Err(type_error("randbytes() requires a non-negative int")), }; - Ok(Object::new_tuple(vec![Object::Int( - current_state(&inst) as i64 - )])) + let out = with_mt(&inst, |mt| { + let mut buf = Vec::with_capacity(n); + while buf.len() < n { + let w = mt.genrand_u32().to_le_bytes(); + let take = (n - buf.len()).min(4); + buf.extend_from_slice(&w[..take]); + } + buf + })?; + Ok(Object::new_bytes(out)) +} + +/// `getstate()` → 625-tuple: the 624 state words plus the cursor. +fn random_getstate(args: &[Object]) -> Result { + let inst = self_instance(args, "getstate()")?; + let mt = load_mt(&inst)?; + let mut items: Vec = mt + .key + .iter() + .map(|w| Object::Int(i64::from(*w))) + .collect(); + items.push(Object::Int(mt.pos as i64)); + Ok(Object::new_tuple(items)) } fn random_setstate(args: &[Object]) -> Result { - let inst = match args.first() { - Some(Object::Instance(i)) => i.clone(), - _ => return Err(type_error("setstate() requires self")), + let inst = self_instance(args, "setstate()")?; + let items = match args.get(1) { + Some(Object::Tuple(t)) => t.clone(), + _ => return Err(type_error("state vector must be a tuple")), }; - let state = match args.get(1) { - Some(Object::Tuple(items)) if !items.is_empty() => match &items[0] { - Object::Int(i) => *i as u64, + if items.len() != N + 1 { + return Err(value_error(format!( + "state vector is the wrong size; expected {}, got {}", + N + 1, + items.len() + ))); + } + let mut key = [0u32; N]; + for (i, slot) in key.iter_mut().enumerate() { + *slot = match &items[i] { + Object::Int(v) => *v as u32, Object::Long(b) => { use num_traits::ToPrimitive; - b.to_u64().unwrap_or(0) + b.to_u64().unwrap_or(0) as u32 } - _ => return Err(type_error("setstate(): invalid state tuple")), - }, - _ => return Err(type_error("setstate(): state must be a tuple")), + _ => return Err(type_error("state vector items must be ints")), + }; + } + let pos = match &items[N] { + Object::Int(v) if (0..=N as i64).contains(v) => *v as usize, + Object::Int(_) => return Err(value_error("invalid state")), + _ => return Err(type_error("state vector items must be ints")), }; - set_state(&inst, state); + store_mt(&inst, &Mt { key, pos }); Ok(Object::None) } diff --git a/crates/weavepy-vm/src/stdlib/socket_mod.rs b/crates/weavepy-vm/src/stdlib/socket_mod.rs index f0ecbe9..ba64516 100644 --- a/crates/weavepy-vm/src/stdlib/socket_mod.rs +++ b/crates/weavepy-vm/src/stdlib/socket_mod.rs @@ -420,7 +420,7 @@ fn socket_methods() -> Vec<(&'static str, Object)> { fn extract_self(args: &[Object]) -> Result, RuntimeError> { match args.first() { - Some(Object::Instance(inst)) if inst.class.name == "socket" => Ok(inst.clone()), + Some(Object::Instance(inst)) if inst.cls().name == "socket" => Ok(inst.clone()), _ => Err(type_error("socket method requires socket self")), } } diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index 69d01ad..a212205 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -789,7 +789,7 @@ fn sys_exc_info( if let Some(top) = stack.last() { let inst = top.instance.clone(); let type_obj = match &inst { - Object::Instance(i) => Object::Type(i.class.clone()), + Object::Instance(i) => Object::Type(i.cls()), _ => Object::None, }; let tb = match &inst { @@ -820,7 +820,7 @@ fn sys_default_excepthook(args: &[Object]) -> Result { // from inside `except:` and the user hook is `None`. let value = args.get(1).cloned().unwrap_or(Object::None); let kind = match &value { - Object::Instance(i) => i.class.name.clone(), + Object::Instance(i) => i.cls().name.clone(), _ => "Exception".to_owned(), }; let msg = crate::builtin_types::exception_message(&value).unwrap_or_default(); @@ -1359,17 +1359,24 @@ fn stdlib_module_names_value() -> Object { Object::FrozenSet(Rc::new(set)) } -/// `sys.getrefcount(obj)` — best-effort. Always returns a -/// non-zero value to satisfy `assert sys.getrefcount(x) > 0`- -/// style sanity checks. The exact number is implementation- -/// specific even in CPython. +/// `sys.getrefcount(obj)` — best-effort, derived from the real +/// `Rc::strong_count` of the payload. Infrastructure references +/// (the cycle-GC registry's handle, weakref slots' strong clones) +/// are discounted so the number tracks *program-visible* bindings; +/// `+1` accounts for the argument reference, like CPython. The +/// exact number is implementation-specific even in CPython. fn sys_getrefcount(args: &[Object]) -> Result { - if args.is_empty() { + let Some(obj) = args.first() else { return Err(type_error("getrefcount() takes exactly 1 argument")); - } - // Two is what CPython returns for a freshly-bound name: the - // local + the argument. - Ok(Object::Int(2)) + }; + let strong = crate::gc_trace::strong_count_for(obj); + let id = crate::weakref_registry::id_of(obj); + let registry = usize::from(crate::gc_trace::is_tracked(id)); + let weak_clones = crate::weakref_registry::strong_clone_count(id); + // The clone in our `args` slice plays the role of CPython's + // "+1 for the argument reference" — no extra increment needed. + let visible = strong.saturating_sub(registry).saturating_sub(weak_clones); + Ok(Object::Int(visible.max(1) as i64)) } /// Default `sys.displayhook`: if the value is None do nothing, diff --git a/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs b/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs index 24bedf0..174e559 100644 --- a/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs +++ b/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs @@ -28,8 +28,8 @@ const INLINE_CAPACITY: usize = 30; fn has_inline_values(args: &[Object]) -> Result { let inline = match args.first() { Some(Object::Instance(inst)) => { - inst.class.has_managed_dict() - && !inst.class.has_var_sized_base() + inst.cls().has_managed_dict() + && !inst.cls().has_var_sized_base() && inst.inline_values.get() && inst.dict.borrow().len() <= INLINE_CAPACITY } diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs index 02be674..b8fcbbf 100644 --- a/crates/weavepy-vm/src/stdlib/thread_real.rs +++ b/crates/weavepy-vm/src/stdlib/thread_real.rs @@ -275,7 +275,7 @@ fn make_lock_object(lock: Arc) -> Object { ); } let inst = Rc::new(PyInstance { - class: lock_type(), + class: crate::sync::RefCell::new(lock_type()), dict, native: None, inline_values: crate::sync::Cell::new(true), @@ -376,7 +376,7 @@ fn make_rlock_object(rlock: Arc) -> Object { ); } let inst = Rc::new(PyInstance { - class: rlock_type(), + class: crate::sync::RefCell::new(rlock_type()), dict, native: None, inline_values: crate::sync::Cell::new(true), @@ -577,7 +577,7 @@ fn is_system_exit(err: &RuntimeError) -> bool { let RuntimeError::PyException(exc) = err else { return false; }; - matches!(&exc.instance, Object::Instance(inst) if inst.class.name == "SystemExit") + matches!(&exc.instance, Object::Instance(inst) if inst.cls().name == "SystemExit") } /// Run `threading.excepthook` (if installed) with the worker's @@ -614,7 +614,7 @@ fn invoke_threading_excepthook( // `threading.py` accepts a simple tuple-with-attribute shim, // which we materialise here as a `SimpleNamespace`. let exc_type = match &exc.instance { - Object::Instance(inst) => Object::Type(inst.class.clone()), + Object::Instance(inst) => Object::Type(inst.cls()), _ => Object::None, }; let mut ns = DictData::new(); @@ -663,7 +663,7 @@ mod tests { let l = allocate_lock(&[]).unwrap(); match l { Object::Instance(inst) => { - assert_eq!(inst.class.name, "lock"); + assert_eq!(inst.cls().name, "lock"); } _ => panic!("expected Object::Instance"), } diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs index 7453865..05515cb 100644 --- a/crates/weavepy-vm/src/stdlib/weakref_real.rs +++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs @@ -165,15 +165,107 @@ fn ref_type() -> Rc { }) } +/// Dereference a proxy instance, raising `ReferenceError` once the +/// referent has been collected — CPython's `proxy_checkref`. +fn proxy_target(me: &Object) -> Result { + if let Object::Instance(inst) = me { + let getter = inst + .dict + .borrow() + .get(&DictKey(Object::from_static("__weakref_get__"))) + .cloned(); + if let Some(Object::Builtin(b)) = getter { + let t = (b.call)(&[])?; + if !matches!(t, Object::None) { + return Ok(t); + } + let bt = crate::builtin_types::builtin_types(); + let inst = crate::builtin_types::make_exception_with_class( + bt.reference_error.clone(), + "weakly-referenced object no longer exists", + ); + return Err(RuntimeError::PyException(crate::error::PyException::new( + inst, + ))); + } + } + Err(type_error("expected a weak proxy")) +} + +/// Forward an operation to the referent by calling the named builtin +/// (`iter`, `next`, `len`, …) on it through the live interpreter. +fn proxy_forward_via_builtin( + builtin: &'static str, + target: &Object, +) -> Result { + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| type_error("no running interpreter"))?; + // SAFETY: published by an enclosing VM frame on this thread. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + let f = globals + .borrow() + .get(&DictKey(Object::from_static(builtin))) + .cloned() + .ok_or_else(|| type_error(format!("builtin {builtin} unavailable")))?; + interp.call_object_with_globals(&f, std::slice::from_ref(target), &[], &globals) +} + +/// The shared forwarding dunders for both proxy flavours. +fn install_proxy_forwarding(td: &mut DictData) { + fn fwd_getattr(args: &[Object]) -> Result { + let target = proxy_target(args.first().ok_or_else(|| type_error("missing self"))?)?; + let name = match args.get(1) { + Some(Object::Str(s)) => s.to_string(), + _ => return Err(type_error("attribute name must be string")), + }; + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| type_error("no running interpreter"))?; + // SAFETY: published by an enclosing VM frame on this thread. + let interp = unsafe { &mut *ptr }; + interp.load_attr_public(&target, &name) + } + fn fwd_iter(args: &[Object]) -> Result { + let target = proxy_target(args.first().ok_or_else(|| type_error("missing self"))?)?; + proxy_forward_via_builtin("iter", &target) + } + fn fwd_next(args: &[Object]) -> Result { + let target = proxy_target(args.first().ok_or_else(|| type_error("missing self"))?)?; + proxy_forward_via_builtin("next", &target) + } + fn fwd_len(args: &[Object]) -> Result { + let target = proxy_target(args.first().ok_or_else(|| type_error("missing self"))?)?; + proxy_forward_via_builtin("len", &target) + } + fn fwd_str(args: &[Object]) -> Result { + let target = proxy_target(args.first().ok_or_else(|| type_error("missing self"))?)?; + proxy_forward_via_builtin("str", &target) + } + for (name, f) in [ + ( + "__getattr__", + fwd_getattr as fn(&[Object]) -> Result, + ), + ("__iter__", fwd_iter), + ("__next__", fwd_next), + ("__len__", fwd_len), + ("__str__", fwd_str), + ] { + td.insert(DictKey(Object::from_static(name)), b(name, f)); + } +} + fn proxy_type() -> Rc { PROXY_TYPE.with(|cell| { if let Some(t) = cell.borrow().clone() { return t; } + let mut td = DictData::new(); + install_proxy_forwarding(&mut td); let t = TypeObject::new_with_flags( "weakproxy", vec![crate::builtin_types::builtin_types().object_.clone()], - DictData::new(), + td, TypeFlags { is_exception: false, is_builtin: true, @@ -190,10 +282,12 @@ fn callable_proxy_type() -> Rc { if let Some(t) = cell.borrow().clone() { return t; } + let mut td = DictData::new(); + install_proxy_forwarding(&mut td); let t = TypeObject::new_with_flags( "weakcallableproxy", vec![crate::builtin_types::builtin_types().object_.clone()], - DictData::new(), + td, TypeFlags { is_exception: false, is_builtin: true, @@ -335,7 +429,7 @@ fn make_ref_object(target: Object, callback: Option, kind_tag: u8) -> Ob } Object::Instance(Rc::new(PyInstance { - class, + class: crate::sync::RefCell::new(class), dict, native: None, inline_values: crate::sync::Cell::new(true), diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index bd582ff..d8ed42b 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -107,9 +107,17 @@ impl TypeObject { pub fn new_user( name: &str, bases: Vec>, - dict: DictData, + mut dict: DictData, ) -> Result, RuntimeError> { let is_exception = bases.iter().any(|b| b.flags.is_exception); + // CPython `type_new`: a class that defines `__eq__` without + // defining `__hash__` is unhashable (`__hash__` is set to None + // in the new class's dict). + if dict.contains_key(&DictKey(Object::from_static("__eq__"))) + && !dict.contains_key(&DictKey(Object::from_static("__hash__"))) + { + dict.insert(DictKey(Object::from_static("__hash__")), Object::None); + } Self::new_with_flags( name, bases, @@ -166,6 +174,18 @@ impl TypeObject { }) } + /// The first built-in class in the MRO other than `object` — the + /// moral equivalent of CPython's `solid_base`, which determines + /// instance memory layout for `__class__` assignment checks. + /// `None` for plain `object`-rooted classes. + pub fn solid_base_name(&self) -> Option { + self.mro + .borrow() + .iter() + .find(|t| t.flags.is_builtin && t.name != "object") + .map(|t| t.name.clone()) + } + /// CPython `type.__flags__` (`tp_flags`), computed from this type's /// observable properties. Covers the documented/queried bits: /// inline-values + managed-dict (`test_class`), heap/base/ready/gc, @@ -369,7 +389,10 @@ fn compute_c3( /// descriptors yet; see RFC 0010). #[derive(Debug, Clone)] pub struct PyInstance { - pub class: Rc, + /// The instance's type. Interior-mutable because Python permits + /// `obj.__class__ = OtherClass` for layout-compatible heap types; + /// read through [`PyInstance::cls`]. + pub class: RefCell>, pub dict: Rc>, /// For instances of a subclass of an immutable built-in /// (`int`, `str`, `float`, `bytes`, `tuple`, …) this holds the @@ -392,7 +415,7 @@ pub struct PyInstance { impl PyInstance { pub fn new(class: Rc) -> Self { Self { - class, + class: RefCell::new(class), dict: Rc::new(RefCell::new(DictData::new())), native: None, inline_values: Cell::new(true), @@ -403,10 +426,21 @@ impl PyInstance { /// (subclass of `int`/`str`/…). pub fn with_native(class: Rc, native: Object) -> Self { Self { - class, + class: RefCell::new(class), dict: Rc::new(RefCell::new(DictData::new())), native: Some(native), inline_values: Cell::new(true), } } + + /// The instance's current class (honours `__class__` assignment). + #[inline] + pub fn cls(&self) -> Rc { + self.class.borrow().clone() + } + + /// Re-point the instance at a new class (`obj.__class__ = C`). + pub fn set_cls(&self, class: Rc) { + *self.class.borrow_mut() = class; + } } diff --git a/crates/weavepy-vm/src/vm_singletons.rs b/crates/weavepy-vm/src/vm_singletons.rs index 004759b..8cbb810 100644 --- a/crates/weavepy-vm/src/vm_singletons.rs +++ b/crates/weavepy-vm/src/vm_singletons.rs @@ -38,6 +38,17 @@ pub fn push_pending_finalizer(obj: Object) { }); } +/// Like [`push_pending_finalizer`], but callable from `Drop` impls: +/// tolerates thread-teardown (destroyed TLS) and re-entrant borrows +/// by silently dropping the request. +pub fn try_push_pending_finalizer(obj: Object) { + let _ = PENDING_FINALIZERS.try_with(|cell| { + if let Ok(mut queue) = cell.try_borrow_mut() { + queue.push(obj); + } + }); +} + /// Drain the pending-finalizer queue. The eval loop calls this /// at every eval-breaker tick that has the GC flag set. pub fn drain_pending_finalizers() -> Vec { @@ -45,10 +56,31 @@ pub fn drain_pending_finalizers() -> Vec { } fn make_singleton(name: &'static str) -> Object { + let mut dict = DictData::new(); + // `repr(NotImplemented)` is "NotImplemented", `repr(...)` is + // "Ellipsis" — install a `__repr__` carrying the canonical text so + // every repr path renders the singleton the way CPython does. + let repr_text: &'static str = match name { + "NotImplementedType" => "NotImplemented", + "ellipsis" => "Ellipsis", + other => other, + }; + fn make_repr(text: &'static str) -> Object { + use crate::object::BuiltinFn; + Object::Builtin(Rc::new(BuiltinFn { + name: "__repr__", + call: Box::new(move |_args| Ok(Object::from_static(text))), + call_kw: None, + })) + } + dict.insert( + crate::object::DictKey(Object::from_static("__repr__")), + make_repr(repr_text), + ); let cls = TypeObject::new_with_flags( name, vec![], - DictData::new(), + dict, TypeFlags { is_exception: false, is_builtin: true, From d8305758a4e5217ff14ccb5bf791d47b2c35f277 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Wed, 10 Jun 2026 16:44:49 -0700 Subject: [PATCH 8/9] feat: advance CPython Lib/test conformance wave 2 --- Cargo.lock | 1 + Cargo.toml | 1 + crates/weavepy-compiler/src/lib.rs | 114 +- crates/weavepy-vm/Cargo.toml | 1 + crates/weavepy-vm/src/builtin_types.rs | 37 +- crates/weavepy-vm/src/builtins.rs | 1806 ++++++++++++--- crates/weavepy-vm/src/error.rs | 6 + crates/weavepy-vm/src/lib.rs | 1010 +++++++-- crates/weavepy-vm/src/object.rs | 168 +- crates/weavepy-vm/src/stdlib/mod.rs | 14 +- crates/weavepy-vm/src/stdlib/os.rs | 36 +- .../src/stdlib/python/_collections.py | 260 +++ .../weavepy-vm/src/stdlib/python/asyncio.py | 163 +- .../src/stdlib/python/collections.py | 1970 ++++++++++++----- .../weavepy-vm/src/stdlib/python/inspect.py | 14 + .../weavepy-vm/src/stdlib/python/tempfile.py | 19 +- .../weavepy-vm/src/stdlib/python/types_mod.py | 12 +- .../weavepy-vm/src/stdlib/python/warnings.py | 23 + crates/weavepy-vm/src/stdlib/sys.rs | 126 ++ crates/weavepy-vm/src/stdlib/weakref_real.rs | 32 + tests/regrtest/expectations.toml | 58 +- 21 files changed, 4816 insertions(+), 1055 deletions(-) create mode 100644 crates/weavepy-vm/src/stdlib/python/_collections.py diff --git a/Cargo.lock b/Cargo.lock index 7f2897d..62d5cf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2401,6 +2401,7 @@ dependencies = [ "indexmap", "libc", "md-5", + "memchr", "memmap2 0.9.10", "mio", "num-bigint", diff --git a/Cargo.toml b/Cargo.toml index bce51d8..77b82e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,7 @@ unicode-normalization = "0.1" unicode-properties = "0.1" unicode_names2 = "2.0" memmap2 = "0.9" +memchr = "2.8" libc = "0.2" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } rustls-pki-types = "1.7" diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index 92f7eee..e38ec9e 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -563,6 +563,15 @@ struct Compiler { /// emitted. Drives PEP-657 column tracking in [`Self::emit`]. Updated /// at statement and expression granularity as the compiler descends. current_span: (u32, u32), + /// Number of *live exception values* sitting on the operand stack at + /// the current compile point: a `finally` body (or the unmatched + /// re-raise path of a `try/except`) runs with the propagating + /// exception on the stack until the trailing `RERAISE` pops it. + /// Exception-table entries registered for code nested inside such + /// regions must include these slots in their `depth`, or the + /// dispatch loop would truncate the live exception away and the + /// `RERAISE` would underflow. + exc_on_stack: u32, /// `True` for methods compiled inside a class body. Such methods /// implicitly capture the class's `__class__` cell so `super()` /// works without arguments. @@ -671,6 +680,7 @@ impl Compiler { line_index, current_line: 0, current_span: (0, 0), + exc_on_stack: 0, inside_class_body: false, annotations_initialized: false, code_kind: kind, @@ -1180,6 +1190,13 @@ impl Compiler { if self.kind != CodeKind::Function { return Err(CompileError::ReturnOutsideFunction); } + // PEP 525: async generators cannot return a value (the + // flag is set before the body compiles, so this sees it). + if self.co.is_async_generator && value.is_some() { + return Err(CompileError::SyntaxExact( + "'return' with value in async generator".to_owned(), + )); + } match value { Some(v) => self.compile_expr(v)?, None => { @@ -2208,10 +2225,11 @@ impl Compiler { // Approximate stack depth at handler entry. The dispatch // loop truncates everything above `depth`, so we need to // preserve any state the surrounding control-flow stitched - // into the stack — most importantly, iterators kept live - // across `for` loop iterations. Without full stack-effect - // tracking we simply count active `for` frames. - let body_depth = self.loop_stack.iter().filter(|fr| fr.is_for_loop).count() as u32; + // into the stack — iterators kept live across `for` loop + // iterations, and any propagating exception a surrounding + // `finally` keeps on the stack for its trailing RERAISE. + let body_depth = self.loop_stack.iter().filter(|fr| fr.is_for_loop).count() as u32 + + self.exc_on_stack; // Make the finally body visible to any `return`/`break`/ // `continue` nested inside `body`/`orelse`/handlers. We pop it // before emitting the *direct* normal-/exception-exit copies @@ -2455,15 +2473,18 @@ impl Compiler { let cur = self.next_offset(); self.patch_jump(site, cur); } - // Run finally on the re-raise path before propagating. + // Run finally on the re-raise path before propagating. The + // unmatched exception stays on the stack until RERAISE. let saved = if pushed_finally { self.finally_stack.pop() } else { None }; + self.exc_on_stack += 1; for s in finalbody { self.compile_stmt(s)?; } + self.exc_on_stack -= 1; if let Some(f) = saved { self.finally_stack.push(f); } @@ -2476,9 +2497,12 @@ impl Compiler { let cleanup_start = self.next_offset(); let cleanup_push = self.emit(OpCode::PushExcInfo, 0); let saved = self.finally_stack.pop(); + // The escaping exception is on the stack until RERAISE. + self.exc_on_stack += 1; for s in finalbody { self.compile_stmt(s)?; } + self.exc_on_stack -= 1; if let Some(f) = saved { self.finally_stack.push(f); } @@ -2530,9 +2554,14 @@ impl Compiler { // exception stays put for the trailing `RERAISE 0`. let push_exc_site = self.emit(OpCode::PushExcInfo, 0); let saved = self.finally_stack.pop(); + // The propagating exception is on the stack until RERAISE; + // nested handlers registered inside the finally body must + // preserve that slot. + self.exc_on_stack += 1; for s in finalbody { self.compile_stmt(s)?; } + self.exc_on_stack -= 1; if let Some(f) = saved { self.finally_stack.push(f); } @@ -2565,22 +2594,16 @@ impl Compiler { } return Ok(()); } - // Recurse on multi-item: `with a, b: body` ≡ `with a: with b: body`. - if items.len() > 1 { - let inner = vec![Stmt { - kind: StmtKind::With { - items: items[1..].to_vec(), - body: body.to_vec(), - }, - span: weavepy_lexer::Span::new(0, 0), - }]; - return self.compile_with(&items[..1], &inner); - } - let item = &items[0]; - // The `with` statement's own line — the cleanup/`__exit__` - // sequences below are attributed to it (CPython does the same: - // a traceback through `__exit__` shows the `with` line, not the - // last body statement). + // Multi-item recursion happens at the body site below: + // `with a, b: body` ≡ `with a: with b: body`. + let (item, rest) = items.split_first().expect("nonempty"); + // PEP 657: the whole setup/`__exit__` dance for this item is + // attributed to the context-manager *expression* itself, so a + // traceback through `__init__`/`__enter__`/`__exit__` pinpoints + // the precise manager in `with A(), B(), C():` (CPython + // `testExceptionLocation`). + self.set_line_from(item.context_expr.span.start.0); + self.set_span(item.context_expr.span); let with_line = self.current_line; let with_span = self.current_span; let cm_name = format!(".with_cm{}", self.with_counter); @@ -2589,6 +2612,8 @@ impl Compiler { // Evaluate cm and stash it for later __exit__ access. self.compile_expr(&item.context_expr)?; + self.current_line = with_line; + self.current_span = with_span; self.emit(OpCode::StoreFast, cm_idx); // Call __enter__ and bind (or discard). @@ -2616,8 +2641,12 @@ impl Compiler { }); let body_start = self.next_offset(); - for s in body { - self.compile_stmt(s)?; + if rest.is_empty() { + for s in body { + self.compile_stmt(s)?; + } + } else { + self.compile_with(rest, body)?; } let body_end = self.next_offset(); @@ -2625,7 +2654,7 @@ impl Compiler { // below emits the same call inline. self.finally_stack.pop(); - // Attribute the whole exit path to the `with` line. + // Attribute the whole exit path to this item's expression. self.current_line = with_line; self.current_span = with_span; @@ -2650,7 +2679,8 @@ impl Compiler { // *suppressed* an exception inside a `for` lost the iterator and // the next `FOR_ITER` found an empty stack. This matches the // `body_depth` convention used by `try`/`except` handlers above. - let body_depth = self.loop_stack.iter().filter(|fr| fr.is_for_loop).count() as u32; + let body_depth = self.loop_stack.iter().filter(|fr| fr.is_for_loop).count() as u32 + + self.exc_on_stack; self.co.exception_table.push(ExcHandler { start: body_start, end: body_end, @@ -3553,7 +3583,7 @@ impl Compiler { start: loop_top, end: dance_end, handler: cleanup_target, - depth: 1, + depth: 1 + self.exc_on_stack, }); // Cleanup: pop aiter + exception, then run the `else` clause. self.emit(OpCode::EndAsyncFor, 0); @@ -3576,11 +3606,15 @@ impl Compiler { return Ok(()); } let (head, rest) = items.split_first().expect("nonempty"); - // See `compile_with`: the exit paths are attributed to the - // `async with` statement's own line. + // See `compile_with`: the whole setup/exit dance is attributed + // to this item's context-manager expression (PEP 657). + self.set_line_from(head.context_expr.span.start.0); + self.set_span(head.context_expr.span); let with_line = self.current_line; let with_span = self.current_span; self.compile_expr(&head.context_expr)?; + self.current_line = with_line; + self.current_span = with_span; // BEFORE_ASYNC_WITH leaves [aexit, awaitable(aenter)]. self.emit(OpCode::BeforeAsyncWith, 0); self.compile_await_dance(2); @@ -3650,7 +3684,8 @@ impl Compiler { let handler_start = self.next_offset(); // Preserve enclosing for-loop iterators on the operand stack, the // same depth convention used by `try`/`except` and `compile_with`. - let body_depth = self.loop_stack.iter().filter(|fr| fr.is_for_loop).count() as u32; + let body_depth = self.loop_stack.iter().filter(|fr| fr.is_for_loop).count() as u32 + + self.exc_on_stack; self.co.exception_table.push(ExcHandler { start: body_start, end: body_end, @@ -5349,6 +5384,11 @@ fn collect_reads_stmt(stmt: &Stmt, out: &mut HashSet) { StmtKind::With { items, body } | StmtKind::AsyncWith { items, body } => { for it in items { collect_reads_expr(&it.context_expr, out); + // `with cm as obj.attr:` / `as obj[i]:` reads the + // target's container. + if let Some(t) = &it.optional_vars { + collect_reads_assign_target(t, out); + } } for s in body { collect_reads_stmt(s, out); @@ -5612,6 +5652,16 @@ fn collect_reads_expr(expr: &Expr, out: &mut HashSet) { for g in generators.iter().skip(1) { collect_reads_expr(&g.iter, out); } + // Names free in the comprehension body propagate to the + // enclosing scope (CPython symtable). A non-name target + // (`for tgt[0] in …`) reads its container; filters read + // their condition. + for g in generators { + collect_reads_assign_target(&g.target, out); + for i in &g.ifs { + collect_reads_expr(i, out); + } + } collect_reads_expr(elt, out); } ExprKind::DictComp { @@ -5625,6 +5675,12 @@ fn collect_reads_expr(expr: &Expr, out: &mut HashSet) { for g in generators.iter().skip(1) { collect_reads_expr(&g.iter, out); } + for g in generators { + collect_reads_assign_target(&g.target, out); + for i in &g.ifs { + collect_reads_expr(i, out); + } + } collect_reads_expr(key, out); collect_reads_expr(value, out); } diff --git a/crates/weavepy-vm/Cargo.toml b/crates/weavepy-vm/Cargo.toml index a0a81fc..5523fe5 100644 --- a/crates/weavepy-vm/Cargo.toml +++ b/crates/weavepy-vm/Cargo.toml @@ -52,6 +52,7 @@ unicode-normalization = { workspace = true } unicode-properties = { workspace = true } unicode_names2 = { workspace = true } memmap2 = { workspace = true } +memchr = { workspace = true } libc = { workspace = true } rustls = { workspace = true } rustls-pki-types = { workspace = true } diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index 23035b8..8464688 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -771,6 +771,23 @@ fn concrete_elements(obj: &Object) -> Option> { } } +/// Drain any other iterable (map/filter/generator/range/…) through the +/// running interpreter — the general-protocol fallback for the seeding +/// conversions below (CPython's `PySequence_Tuple` reach). +fn elements_via_interp(obj: &Object) -> Option> { + let ptr = crate::vm_singletons::current_interpreter_ptr()?; + // SAFETY: published by an enclosing VM frame still live on this + // thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + interp.collect_iterable(obj, &globals).ok() +} + +/// `concrete_elements` plus the interpreter-driven fallback. +fn any_elements(obj: &Object) -> Option> { + concrete_elements(obj).or_else(|| elements_via_interp(obj)) +} + /// Build the native payload `object.__new__(cls, value?)` should stash /// on an instance of a value/container built-in subclass, or `None` for /// an ordinary `object` subclass. Mutable containers (`list`/`dict`/ @@ -811,24 +828,24 @@ fn native_seed_for_new(cls: &Rc, value: Option<&Object>) -> Option = value - .and_then(concrete_elements) + .and_then(any_elements) .map(|els| els.iter().filter_map(|o| o.as_i64()).map(|i| i as u8).collect()) .unwrap_or_default(); return Some(Object::Bytes(Rc::from(bytes.as_slice()))); } if is_strict(&bt.tuple_) { - let els = value.and_then(concrete_elements).unwrap_or_default(); + let els = value.and_then(any_elements).unwrap_or_default(); return Some(Object::new_tuple(els)); } if is_strict(&bt.frozenset_) { - let els = value.and_then(concrete_elements).unwrap_or_default(); + let els = value.and_then(any_elements).unwrap_or_default(); return Some(Object::new_frozenset_from(els)); } if is_strict(&bt.list_) { @@ -858,6 +875,18 @@ pub(crate) fn object_new(args: &[Object]) -> Result { )) } }; + // `tuple.__new__(tuple, it)` / `int.__new__(int, x)` … on the *built-in + // class itself* must produce the native value, not a PyInstance shell + // (CPython's per-type `tp_new`). Subclasses keep falling through to the + // payload-seeding path below. + if cls.flags.is_builtin && !Rc::ptr_eq(&cls, &builtin_types().object_) { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still live on this + // thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + return interp.type_call_default(&cls, &args[1..], &[]); + } + } // CPython `object_new` arity policy (bpo-31506): excess arguments // are an error unless exactly one of `__new__`/`__init__` is // overridden (the overriding side owns the signature). diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 2218004..e4cf752 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -63,6 +63,13 @@ pub(crate) fn builtin_type_constructor(name: &str) -> Option> { call_kw: None, })) }; + ($n:literal, $body:expr, $kw:expr) => { + Some(Rc::new(BuiltinFn { + name: $n, + call: Box::new($body), + call_kw: Some(Box::new($kw)), + })) + }; } match name { "str" => ctor!("str", b_str), @@ -75,8 +82,8 @@ pub(crate) fn builtin_type_constructor(name: &str) -> Option> { "dict" => ctor!("dict", b_dict), "set" => ctor!("set", b_set), "frozenset" => ctor!("frozenset", b_frozenset), - "bytes" => ctor!("bytes", b_bytes), - "bytearray" => ctor!("bytearray", b_bytearray), + "bytes" => ctor!("bytes", b_bytes, b_bytes_kw), + "bytearray" => ctor!("bytearray", b_bytearray, b_bytearray_kw), "object" => ctor!("object", b_object), "type" => ctor!("type", b_type), "range" => ctor!("range", b_range), @@ -407,7 +414,7 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "join" => Some(method("join", str_join)), "startswith" => Some(method("startswith", str_startswith)), "endswith" => Some(method("endswith", str_endswith)), - "replace" => Some(method("replace", str_replace)), + "replace" => Some(method_kw("replace", str_replace_kw)), "find" => Some(method("find", str_find)), "rfind" => Some(method("rfind", str_rfind)), "index" => Some(method("index", str_index)), @@ -533,29 +540,47 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { }, Object::Bytes(_) | Object::ByteArray(_) => match name { "decode" => Some(method("decode", bytes_decode)), - "hex" => Some(method("hex", bytes_hex)), + "hex" => Some(method_kw("hex", bytes_hex_kw)), "fromhex" => Some(method("fromhex", bytes_fromhex)), "startswith" => Some(method("startswith", bytes_startswith)), "endswith" => Some(method("endswith", bytes_endswith)), "find" => Some(method("find", bytes_find)), "rfind" => Some(method("rfind", bytes_rfind)), "index" => Some(method("index", bytes_index)), + "rindex" => Some(method("rindex", bytes_rindex)), "count" => Some(method("count", bytes_count)), "lower" => Some(method("lower", bytes_lower)), "upper" => Some(method("upper", bytes_upper)), "strip" => Some(method("strip", bytes_strip)), "lstrip" => Some(method("lstrip", bytes_lstrip)), "rstrip" => Some(method("rstrip", bytes_rstrip)), - "split" => Some(method("split", bytes_split)), - "splitlines" => Some(method("splitlines", bytes_splitlines)), + "split" => Some(method_kw("split", bytes_split_kw)), + "rsplit" => Some(method_kw("rsplit", bytes_rsplit_kw)), + "splitlines" => Some(method_kw("splitlines", bytes_splitlines_kw)), "join" => Some(method("join", bytes_join)), - "replace" => Some(method("replace", bytes_replace)), - "translate" => Some(method("translate", bytes_translate)), + "replace" => Some(method_kw("replace", bytes_replace_kw)), + "translate" => Some(method_kw("translate", bytes_translate_kw)), "maketrans" => Some(method("maketrans", bytes_maketrans)), + "partition" => Some(method("partition", bytes_partition)), + "rpartition" => Some(method("rpartition", bytes_rpartition)), + "removeprefix" => Some(method("removeprefix", bytes_removeprefix)), + "removesuffix" => Some(method("removesuffix", bytes_removesuffix)), + "expandtabs" => Some(method_kw("expandtabs", bytes_expandtabs)), + "center" => Some(method("center", bytes_center)), + "ljust" => Some(method("ljust", bytes_ljust)), + "rjust" => Some(method("rjust", bytes_rjust)), + "zfill" => Some(method("zfill", bytes_zfill)), + "capitalize" => Some(method("capitalize", bytes_capitalize)), + "title" => Some(method("title", bytes_title)), + "swapcase" => Some(method("swapcase", bytes_swapcase)), "isalnum" => Some(method("isalnum", bytes_isalnum)), "isalpha" => Some(method("isalpha", bytes_isalpha)), "isdigit" => Some(method("isdigit", bytes_isdigit)), "isspace" => Some(method("isspace", bytes_isspace)), + "islower" => Some(method("islower", bytes_islower)), + "isupper" => Some(method("isupper", bytes_isupper)), + "istitle" => Some(method("istitle", bytes_istitle)), + "isascii" => Some(method("isascii", bytes_isascii)), // bytearray-only mutators "append" if matches!(obj, Object::ByteArray(_)) => { Some(method("append", bytearray_append)) @@ -570,10 +595,32 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "reverse" if matches!(obj, Object::ByteArray(_)) => { Some(method("reverse", bytearray_reverse)) } + "insert" if matches!(obj, Object::ByteArray(_)) => { + Some(method("insert", bytearray_insert)) + } + "remove" if matches!(obj, Object::ByteArray(_)) => { + Some(method("remove", bytearray_remove)) + } + "copy" if matches!(obj, Object::ByteArray(_)) => { + Some(method("copy", bytearray_copy)) + } // Sequence dunders so direct calls / `hasattr` parity hold. "__contains__" => Some(method("__contains__", obj_contains)), "__len__" => Some(method("__len__", obj_len)), "__getitem__" => Some(method("__getitem__", seq_getitem)), + // PEP 461 `%`-formatting exposed as the number-protocol + // dunders (`bytes_mod` fills CPython's `nb_remainder` slot, + // so both wrappers exist). + "__mod__" => Some(method("__mod__", bytes_dunder_mod)), + "__rmod__" => Some(method("__rmod__", bytes_dunder_rmod)), + "__bytes__" if matches!(obj, Object::Bytes(_)) => { + Some(method("__bytes__", |args| { + match args.first() { + Some(Object::Bytes(b)) => Ok(Object::Bytes(b.clone())), + _ => Err(type_error("__bytes__ requires a bytes receiver")), + } + })) + } _ => None, }, Object::File(_) => match name { @@ -3470,24 +3517,51 @@ pub(crate) fn b_int_from_bytes_cls(args: &[Object]) -> Result) -> Result { + match arg { + Some(Object::Str(s)) => Ok(s.to_string()), + Some(other) => Err(type_error(format!( + "fromhex() argument must be str, not {}", + other.type_name() + ))), + None => Err(type_error( + "descriptor 'fromhex' of 'bytes' object needs an argument", + )), + } +} + +/// CPython's `bytes.fromhex` on a subclass calls the subclass with the +/// parsed result (`PyObject_CallOneArg(type, result)`), so the returned +/// object is an instance of `cls`. +fn fromhex_wrap_subclass( + cls: Option<&Object>, + base_name: &str, + result: Object, +) -> Result { + if let Some(cls_obj @ Object::Type(t)) = cls { + if t.name != base_name { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still live on + // this thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + return interp.call_object_with_globals(cls_obj, &[result], &[], &globals); + } + } + } + Ok(result) +} + pub(crate) fn b_bytes_fromhex_cls(args: &[Object]) -> Result { - let _cls = args.first(); - let s = match args.get(1) { - Some(Object::Str(s)) => s.to_string(), - _ => return Err(type_error("fromhex() argument must be str")), - }; + let s = fromhex_string_arg(args.get(1))?; let bytes = parse_hex_bytes(&s)?; - Ok(Object::new_bytes(bytes)) + fromhex_wrap_subclass(args.first(), "bytes", Object::new_bytes(bytes)) } pub(crate) fn b_bytearray_fromhex_cls(args: &[Object]) -> Result { - let _cls = args.first(); - let s = match args.get(1) { - Some(Object::Str(s)) => s.to_string(), - _ => return Err(type_error("fromhex() argument must be str")), - }; + let s = fromhex_string_arg(args.get(1))?; let bytes = parse_hex_bytes(&s)?; - Ok(Object::new_bytearray(bytes)) + fromhex_wrap_subclass(args.first(), "bytearray", Object::new_bytearray(bytes)) } pub(crate) fn b_float_fromhex_cls(args: &[Object]) -> Result { @@ -3501,28 +3575,30 @@ pub(crate) fn b_float_fromhex_cls(args: &[Object]) -> Result Result, RuntimeError> { - let mut bytes = Vec::new(); - let mut last_high: Option = None; - for c in s.chars() { - if c.is_whitespace() { - if last_high.is_some() { - return Err(value_error("non-hexadecimal number")); - } + // CPython's `_PyBytes_FromHex`: pairs of hex digits, with *ASCII* + // whitespace permitted only between pairs. Error positions are + // character offsets into the original string. + let hex_err = |pos: usize| { + value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {pos}" + )) + }; + let chars: Vec = s.chars().collect(); + let mut bytes = Vec::with_capacity(chars.len() / 2); + let mut i = 0usize; + while i < chars.len() { + let c = chars[i]; + if matches!(c, ' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r') { + i += 1; continue; } - let v = c - .to_digit(16) - .ok_or_else(|| value_error("non-hexadecimal number"))? as u8; - match last_high { - Some(hi) => { - bytes.push((hi << 4) | v); - last_high = None; - } - None => last_high = Some(v), - } - } - if last_high.is_some() { - return Err(value_error("non-hexadecimal number")); + let hi = if c.is_ascii() { c.to_digit(16) } else { None }.ok_or_else(|| hex_err(i))?; + let lo = match chars.get(i + 1) { + Some(c2) if c2.is_ascii() => c2.to_digit(16).ok_or_else(|| hex_err(i + 1))?, + _ => return Err(hex_err(i + 1)), + }; + bytes.push(((hi << 4) | lo) as u8); + i += 2; } Ok(bytes) } @@ -4035,77 +4111,302 @@ fn b_frozenset(args: &[Object]) -> Result { Ok(Object::new_frozenset_from(out)) } -fn b_bytes(args: &[Object]) -> Result { - if args.is_empty() { - return Ok(Object::new_bytes(Vec::new())); - } - match &args[0] { - Object::Int(n) => { - if *n < 0 { - return Err(value_error("negative count")); +/// One item of a `bytes(iterable)` source: an integer in +/// `range(0, 256)` via the `__index__` protocol. +fn byte_item_value(o: &Object) -> Result { + let native = o.native_value(); + match native.as_ref().unwrap_or(o) { + Object::Bool(b) => Ok(u8::from(*b)), + Object::Int(i) if (0..=255).contains(i) => Ok(*i as u8), + Object::Int(_) | Object::Long(_) => { + Err(value_error("bytes must be in range(0, 256)")) + } + inst @ Object::Instance(_) => { + let v = coerce_index_i64(inst)?; + if (0..=255).contains(&v) { + Ok(v as u8) + } else { + Err(value_error("bytes must be in range(0, 256)")) } - Ok(Object::new_bytes(vec![0u8; *n as usize])) } - Object::Str(s) => { - let encoding = args - .get(1) - .and_then(|x| match x { - Object::Str(e) => Some(e.to_string()), - _ => None, - }) - .unwrap_or_else(|| "utf-8".to_owned()); - let errors = args - .get(2) - .and_then(|x| match x { - Object::Str(e) => Some(e.to_string()), - _ => None, - }) - .unwrap_or_else(|| "strict".to_owned()); - let bytes = crate::stdlib::codecs_mod::encode_str(s, &encoding, &errors)?; - Ok(Object::new_bytes(bytes)) + other => Err(type_error(format!( + "'{}' object cannot be interpreted as an integer", + other.type_name() + ))), + } +} + +/// The non-string source conversion shared by `bytes(x)` and +/// `bytearray(x)` — CPython's `PyBytes_FromObject` / +/// `bytearray_init` tail: index-sized count, buffer copy, or +/// iterable of byte values. +fn bytes_from_source_obj(src: &Object, type_name: &str) -> Result, RuntimeError> { + let zero_fill = |n: i64| -> Result, RuntimeError> { + if n < 0 { + return Err(value_error("negative count")); } - Object::Bytes(b) => Ok(Object::Bytes(b.clone())), - Object::ByteArray(b) => Ok(Object::new_bytes(b.borrow().clone())), - other => { - let mut it = other.make_iter()?; + let mut v = Vec::new(); + v.try_reserve_exact(n as usize).map_err(|_| { + RuntimeError::PyException(crate::error::PyException::from_builtin( + "MemoryError", + String::new(), + )) + })?; + v.resize(n as usize, 0); + Ok(v) + }; + match src { + Object::Bytes(b) => Ok(b.to_vec()), + Object::ByteArray(b) => Ok(b.borrow().clone()), + Object::MemoryView(mv) => Ok(mv.to_bytes()), + Object::Bool(b) => zero_fill(i64::from(*b)), + Object::Int(n) => zero_fill(*n), + Object::Long(_) => Err(crate::error::overflow_error( + "cannot fit 'int' into an index-sized integer", + )), + Object::List(items) => { + // CPython re-checks the list length every iteration + // (gh-34973): an item's `__index__` may mutate the list. + let cell = items.clone(); let mut out = Vec::new(); - while let Some(v) = it.next_value() { - match v { - Object::Int(i) if (0..=255).contains(&i) => out.push(i as u8), - _ => return Err(value_error("bytes must be in range(0, 256)")), + let mut i = 0usize; + loop { + let item = { + let l = cell.borrow(); + if i >= l.len() { + break; + } + l[i].clone() + }; + out.push(byte_item_value(&item)?); + i += 1; + } + Ok(out) + } + Object::Tuple(items) => { + let mut out = Vec::with_capacity(items.len()); + for item in items.iter() { + out.push(byte_item_value(item)?); + } + Ok(out) + } + Object::Instance(inst) => { + // `__bytes__` is consulted by `bytes()` only — CPython's + // bytearray skips straight to the count/buffer/iterable + // protocol. + if type_name == "bytes" { + if let Some(method) = crate::instance_method(src, "__bytes__") { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still + // live on this thread; the GIL keeps it exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + let r = + interp.call_object_with_globals(&method, &[], &[], &globals)?; + return bytes_argview(&r).map_err(|_| { + type_error(format!( + "__bytes__ returned non-bytes (type {})", + r.type_name() + )) + }); + } + } + } + // The `__index__` protocol: a TypeError raised *by* the + // hook falls through to the buffer/iterable path + // (gh-29159); any other exception propagates (gh-34974). + let indexable = inst.native.as_ref().map(|n| n.as_i64().is_some()).unwrap_or(false) + || crate::instance_method(src, "__index__").is_some(); + if indexable { + match coerce_index_i64(src) { + Ok(n) => return zero_fill(n), + Err(RuntimeError::PyException(e)) if e.type_name() == "TypeError" => {} + Err(other) => return Err(other), } } - Ok(Object::new_bytes(out)) + // Buffer protocol: a bytes/bytearray subclass instance + // carries its payload natively. + if let Some(native) = &inst.native { + if matches!( + native, + Object::Bytes(_) | Object::ByteArray(_) | Object::MemoryView(_) + ) { + return bytes_from_source_obj(&native.clone(), type_name); + } + } + // Iterable (including legacy `__getitem__` sequences) via + // interpreter reentry; `__iter__` exceptions propagate. + let iterable = crate::instance_method(src, "__iter__").is_some() + || crate::instance_method(src, "__getitem__").is_some() + || inst.native.is_some(); + if !iterable { + return Err(type_error(format!( + "cannot convert '{}' object to {}", + src.type_name(), + type_name + ))); + } + bytes_from_iterable_reentrant(src, type_name) + } + other => { + if other.make_iter().is_err() && !matches!(other, Object::Generator(_)) { + return Err(type_error(format!( + "cannot convert '{}' object to {}", + other.type_name(), + type_name + ))); + } + bytes_from_iterable_reentrant(other, type_name) } } } -fn b_bytearray(args: &[Object]) -> Result { - if args.is_empty() { - return Ok(Object::new_bytearray(Vec::new())); +/// Iterate any object through the running interpreter (generators, +/// sets, user iterables) collecting byte values. +fn bytes_from_iterable_reentrant( + src: &Object, + type_name: &str, +) -> Result, RuntimeError> { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still live on this + // thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + let items = interp.collect_iterable(src, &globals)?; + let mut out = Vec::with_capacity(items.len()); + for item in &items { + out.push(byte_item_value(item)?); + } + Ok(out) + } else { + let mut it = src.make_iter().map_err(|_| { + type_error(format!( + "cannot convert '{}' object to {}", + src.type_name(), + type_name + )) + })?; + let mut out = Vec::new(); + while let Some(v) = it.next_value() { + out.push(byte_item_value(&v)?); + } + Ok(out) } - match &args[0] { - Object::Int(n) => { - if *n < 0 { - return Err(value_error("negative count")); +} + +/// Shared `bytes(...)` / `bytearray(...)` construction — CPython's +/// `bytes_new_impl` / `bytearray_init` argument handling, including +/// the `encoding` / `errors` keyword rules. +fn bytes_construct( + args: &[Object], + kwargs: &[(String, Object)], + type_name: &str, +) -> Result, RuntimeError> { + if args.len() > 3 { + return Err(type_error(format!( + "{type_name}() takes at most 3 arguments ({} given)", + args.len() + ))); + } + let mut source_obj = args.first().cloned(); + let mut encoding_obj = args.get(1).cloned(); + let mut errors_obj = args.get(2).cloned(); + for (k, v) in kwargs { + match k.as_str() { + "source" => source_obj = Some(v.clone()), + "encoding" => encoding_obj = Some(v.clone()), + "errors" => errors_obj = Some(v.clone()), + other => { + return Err(type_error(format!( + "{type_name}() got an unexpected keyword argument '{other}'" + ))) } - Ok(Object::new_bytearray(vec![0u8; *n as usize])) } - Object::Str(s) => Ok(Object::new_bytearray(s.as_bytes().to_vec())), - Object::Bytes(b) => Ok(Object::new_bytearray(b.to_vec())), - Object::ByteArray(b) => Ok(Object::new_bytearray(b.borrow().clone())), - other => { - let mut it = other.make_iter()?; - let mut out = Vec::new(); - while let Some(v) = it.next_value() { - match v { - Object::Int(i) if (0..=255).contains(&i) => out.push(i as u8), - _ => return Err(value_error("bytes must be in range(0, 256)")), - } - } - Ok(Object::new_bytearray(out)) + } + let encoding = match &encoding_obj { + None => None, + Some(Object::Str(s)) => Some(s.to_string()), + Some(o) => { + return Err(type_error(format!( + "{type_name}() argument 'encoding' must be str, not {}", + o.type_name() + ))) + } + }; + let errors = match &errors_obj { + None => None, + Some(Object::Str(s)) => Some(s.to_string()), + Some(o) => { + return Err(type_error(format!( + "{type_name}() argument 'errors' must be str, not {}", + o.type_name() + ))) + } + }; + let Some(src) = source_obj.as_ref() else { + if encoding.is_some() { + return Err(type_error("encoding without a string argument")); + } + if errors.is_some() { + return Err(type_error("errors without a string argument")); + } + return Ok(Vec::new()); + }; + // String sources require an encoding; non-string sources reject one. + let as_str: Option> = match src { + Object::Str(s) => Some(s.clone()), + Object::Instance(inst) => match &inst.native { + Some(Object::Str(s)) => Some(s.clone()), + _ => None, + }, + _ => None, + }; + if let Some(s) = as_str { + let Some(enc) = encoding else { + return Err(type_error("string argument without an encoding")); + }; + return crate::stdlib::codecs_mod::encode_str( + &s, + &enc, + errors.as_deref().unwrap_or("strict"), + ); + } + if encoding.is_some() { + return Err(type_error("encoding without a string argument")); + } + if errors.is_some() { + return Err(type_error("errors without a string argument")); + } + bytes_from_source_obj(src, type_name) +} + +fn b_bytes_kw(args: &[Object], kwargs: &[(String, Object)]) -> Result { + // `bytes(b'…')` with the exact type returns the argument unchanged + // (immutable, so identity is shareable — `test_repeat_id_preserving` + // relies on `bytes(x) is x` style sharing). + if args.len() == 1 && kwargs.is_empty() { + if let Object::Bytes(b) = &args[0] { + return Ok(Object::Bytes(b.clone())); } } + Ok(Object::new_bytes(bytes_construct(args, kwargs, "bytes")?)) +} + +fn b_bytes(args: &[Object]) -> Result { + b_bytes_kw(args, &[]) +} + +fn b_bytearray_kw( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result { + Ok(Object::new_bytearray(bytes_construct( + args, kwargs, "bytearray", + )?)) +} + +fn b_bytearray(args: &[Object]) -> Result { + b_bytearray_kw(args, &[]) } /// Keyword-argument-aware wrapper for `open`. CPython's signature is @@ -4997,20 +5298,42 @@ fn b_chr(args: &[Object]) -> Result { } fn b_ord(args: &[Object]) -> Result { - match one(args, "ord")? { + let arg = one(args, "ord")?; + let native = arg.native_value(); + match native.as_ref().unwrap_or(arg) { Object::Str(s) => { let mut chars = s.chars(); let c = chars .next() - .ok_or_else(|| type_error("ord() expected a character, but empty string given"))?; + .ok_or_else(|| type_error("ord() expected a character, but string of length 0 found"))?; if chars.next().is_some() { - return Err(type_error( - "ord() expected a character, but multi-character string given", - )); + return Err(type_error(format!( + "ord() expected a character, but string of length {} found", + s.chars().count() + ))); } Ok(Object::Int(i64::from(u32::from(c)))) } - _ => Err(type_error("ord() expected string")), + Object::Bytes(b) if b.len() == 1 => Ok(Object::Int(i64::from(b[0]))), + Object::Bytes(b) => Err(type_error(format!( + "ord() expected a character, but string of length {} found", + b.len() + ))), + Object::ByteArray(b) => { + let data = b.borrow(); + if data.len() == 1 { + Ok(Object::Int(i64::from(data[0]))) + } else { + Err(type_error(format!( + "ord() expected a character, but string of length {} found", + data.len() + ))) + } + } + other => Err(type_error(format!( + "ord() expected string of length 1, but {} found", + other.type_name() + ))), } } @@ -5690,6 +6013,10 @@ fn str_match_prefix_suffix( } fn str_replace(args: &[Object]) -> Result { + str_replace_kw(args, &[]) +} + +fn str_replace_kw(args: &[Object], kwargs: &[(String, Object)]) -> Result { let s = str_self(args)?; let from = match args.get(1) { Some(Object::Str(p)) => p, @@ -5699,7 +6026,75 @@ fn str_replace(args: &[Object]) -> Result { Some(Object::Str(p)) => p, _ => return Err(type_error("replace() expected str")), }; - Ok(Object::from_str(s.replace(&**from, to))) + let mut count_obj = args.get(3).cloned(); + for (k, v) in kwargs { + match k.as_str() { + "count" => count_obj = Some(v.clone()), + other => { + return Err(type_error(format!( + "replace() got an unexpected keyword argument '{other}'" + ))) + } + } + } + let count = match count_obj { + None | Some(Object::None) => -1i64, + Some(o) => coerce_index_i64(&o)?, + }; + if count == 0 { + return Ok(Object::from_str(s.to_string())); + } + let out = if count < 0 { + s.replace(&**from, to) + } else if from.is_empty() { + // `str::replacen` with an empty pattern matches between every + // char and at both ends, same as CPython. + let mut out = String::new(); + let mut done = 0i64; + for (i, ch) in s.chars().enumerate() { + let _ = i; + if done < count { + out.push_str(to); + done += 1; + } + out.push(ch); + } + if done < count { + out.push_str(to); + } + out + } else { + s.replacen(&**from, to, count as usize) + }; + Ok(Object::from_str(out)) +} + +/// `ADJUST_INDICES`: negative indices offset by length and floored at +/// 0; `end` clamped to length; `start` left unclamped so a start past +/// the end yields an invalid window (`'abc'.find('', 4) == -1`). +fn str_search_window(args: &[Object], total_chars: i64) -> Option<(i64, i64)> { + let resolve = |arg: Option<&Object>, default: i64| -> i64 { + match arg { + None | Some(Object::None) => default, + Some(o) => match o.as_i64() { + Some(x) => { + if x < 0 { + (x + total_chars).max(0) + } else { + x + } + } + None => default, + }, + } + }; + let start = resolve(args.get(2), 0); + let end = resolve(args.get(3), total_chars).clamp(0, total_chars); + if start > end { + None + } else { + Some((start, end)) + } } fn str_find(args: &[Object]) -> Result { @@ -5709,11 +6104,9 @@ fn str_find(args: &[Object]) -> Result { _ => return Err(type_error("find() expected str")), }; let total_chars = s.chars().count() as i64; - let start = clamp_str_index(args.get(2), 0, total_chars); - let end = clamp_str_index(args.get(3), total_chars, total_chars); - if start > end || start > total_chars { + let Some((start, end)) = str_search_window(args, total_chars) else { return Ok(Object::Int(-1)); - } + }; let start_byte = char_offset_to_byte(s, start as usize); let end_byte = char_offset_to_byte(s, end as usize); let hay = &s[start_byte..end_byte]; @@ -5726,20 +6119,6 @@ fn str_find(args: &[Object]) -> Result { } } -fn clamp_str_index(arg: Option<&Object>, default: i64, len: i64) -> i64 { - match arg { - Some(Object::Int(n)) => { - if *n < 0 { - (len + n).max(0) - } else { - (*n).min(len) - } - } - Some(Object::None) | None => default, - _ => default, - } -} - fn char_offset_to_byte(s: &str, n: usize) -> usize { if n == 0 { return 0; @@ -5920,11 +6299,9 @@ fn str_rfind(args: &[Object]) -> Result { _ => return Err(type_error("rfind() expected str")), }; let total_chars = s.chars().count() as i64; - let start = clamp_str_index(args.get(2), 0, total_chars); - let end = clamp_str_index(args.get(3), total_chars, total_chars); - if start > end { + let Some((start, end)) = str_search_window(args, total_chars) else { return Ok(Object::Int(-1)); - } + }; let start_byte = char_offset_to_byte(s, start as usize); let end_byte = char_offset_to_byte(s, end as usize); let hay = &s[start_byte..end_byte]; @@ -5952,11 +6329,9 @@ fn str_count(args: &[Object]) -> Result { _ => return Err(type_error("count() expected str")), }; let total_chars = s.chars().count() as i64; - let start = clamp_str_index(args.get(2), 0, total_chars); - let end = clamp_str_index(args.get(3), total_chars, total_chars); - if start > end { + let Some((start, end)) = str_search_window(args, total_chars) else { return Ok(Object::Int(0)); - } + }; let start_byte = char_offset_to_byte(s, start as usize); let end_byte = char_offset_to_byte(s, end as usize); Ok(Object::Int( @@ -6773,8 +7148,13 @@ fn dict_fromkeys(args: &[Object]) -> Result { // shape: the bound version receives the dict as ``args[0]``; // the unbound version omits it. Sniff the receiver shape so a // single body handles both. - let (it_idx, value_idx) = match args.first() { - Some(Object::Dict(_)) | Some(Object::Type(_)) => (1usize, 2usize), + // A lone dict argument is the *iterable* of an unbound call + // (`map(dict.fromkeys, list_of_dicts)` — ChainMap.__iter__ does + // this); a dict in slot 0 only marks the bound receiver when more + // arguments follow. + let (it_idx, value_idx) = match (args.first(), args.len()) { + (Some(Object::Type(_)), _) => (1usize, 2usize), + (Some(Object::Dict(_)), n) if n >= 2 => (1usize, 2usize), _ => (0usize, 1usize), }; let it = args @@ -7083,47 +7463,190 @@ fn bytes_argview(arg: &Object) -> Result, RuntimeError> { match arg { Object::Bytes(b) => Ok(b.to_vec()), Object::ByteArray(b) => Ok(b.borrow().clone()), - Object::Str(s) => Ok(s.as_bytes().to_vec()), - Object::Int(i) if (0..=255).contains(i) => Ok(vec![*i as u8]), - _ => Err(type_error("a bytes-like object is required")), + Object::MemoryView(mv) => Ok(mv.to_bytes()), + Object::Instance(inst) => { + // bytes/bytearray subclasses carry their payload natively. + if let Some(native) = &inst.native { + let native = native.clone(); + if matches!( + native, + Object::Bytes(_) | Object::ByteArray(_) | Object::MemoryView(_) + ) { + return bytes_argview(&native); + } + } + // PEP 688: an object exposing `__buffer__` works anywhere a + // bytes-like object is accepted. Reenter the interpreter to + // call it (CPython's PyObject_GetBuffer slot dispatch). + if let Some(method) = crate::instance_method(arg, "__buffer__") { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still live + // on this thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + let r = interp.call_object_with_globals( + &method, + &[Object::Int(0)], + &[], + &globals, + )?; + return match &r { + Object::MemoryView(mv) => Ok(mv.to_bytes()), + Object::Bytes(b) => Ok(b.to_vec()), + Object::ByteArray(b) => Ok(b.borrow().clone()), + _ => Err(type_error(format!( + "__buffer__ returned non-buffer of type '{}'", + r.type_name() + ))), + }; + } + } + Err(type_error(format!( + "a bytes-like object is required, not '{}'", + arg.type_name() + ))) + } + _ => Err(type_error(format!( + "a bytes-like object is required, not '{}'", + arg.type_name() + ))), } } -fn bytes_decode(args: &[Object]) -> Result { - let data = bytes_data(args)?; - let encoding = match args.get(1) { - Some(Object::Str(e)) => e.to_string(), - None => "utf-8".to_owned(), - _ => return Err(type_error("decode() expected str")), - }; - let errors = match args.get(2) { - Some(Object::Str(e)) => e.to_string(), - None => "strict".to_owned(), - _ => "strict".to_owned(), - }; - let s = crate::stdlib::codecs_mod::decode_bytes(&data, &encoding, &errors)?; +/// Needle argument of `bytes.find` / `rfind` / `index` / `rindex` / +/// `count` / `in`: a bytes-like object, or an integer naming a single +/// byte (range-checked like CPython's `_getbytevalue`). Objects with a +/// user `__index__` go through interpreter reentry like CPython's +/// `PyNumber_Index` path. +fn bytes_find_needle(arg: &Object) -> Result, RuntimeError> { + let native = arg.native_value(); + match native.as_ref().unwrap_or(arg) { + Object::Bytes(b) => Ok(b.to_vec()), + Object::ByteArray(b) => Ok(b.borrow().clone()), + Object::MemoryView(mv) => Ok(mv.to_bytes()), + Object::Bool(v) => Ok(vec![u8::from(*v)]), + Object::Int(i) => { + if (0..=255).contains(i) { + Ok(vec![*i as u8]) + } else { + Err(value_error("byte must be in range(0, 256)")) + } + } + Object::Long(_) => Err(value_error("byte must be in range(0, 256)")), + inst @ Object::Instance(_) + if crate::instance_method(inst, "__index__").is_some() => + { + let v = coerce_index_i64(inst)?; + if (0..=255).contains(&v) { + Ok(vec![v as u8]) + } else { + Err(value_error("byte must be in range(0, 256)")) + } + } + _ => Err(type_error(format!( + "argument should be integer or bytes-like object, not '{}'", + arg.type_name() + ))), + } +} + +/// Build a transform result that follows the receiver's type +/// (`bytes.lower() -> bytes`, `bytearray.lower() -> bytearray`). +fn bytes_like_result(args: &[Object], out: Vec) -> Object { + if matches!(args.first(), Some(Object::ByteArray(_))) { + Object::new_bytearray(out) + } else { + Object::new_bytes(out) + } +} + +fn byte_is_pyspace(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | b'\r' | b'\x0b' | b'\x0c') +} + +fn bytes_decode(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let encoding = match args.get(1) { + Some(Object::Str(e)) => e.to_string(), + None => "utf-8".to_owned(), + _ => return Err(type_error("decode() expected str")), + }; + let errors = match args.get(2) { + Some(Object::Str(e)) => e.to_string(), + None => "strict".to_owned(), + _ => "strict".to_owned(), + }; + let s = crate::stdlib::codecs_mod::decode_bytes(&data, &encoding, &errors)?; Ok(Object::from_str(s)) } fn bytes_hex(args: &[Object]) -> Result { - let data = bytes_data(args)?; - let sep: Option = match args.get(1) { + bytes_hex_kw(args, &[]) +} + +fn bytes_hex_kw(args: &[Object], kwargs: &[(String, Object)]) -> Result { + let data = match args.first() { + Some(Object::MemoryView(mv)) => mv.to_bytes(), + _ => bytes_data(args)?, + }; + let mut sep_obj = args.get(1).cloned(); + let mut bps_obj = args.get(2).cloned(); + for (k, v) in kwargs { + match k.as_str() { + "sep" => sep_obj = Some(v.clone()), + "bytes_per_sep" => bps_obj = Some(v.clone()), + other => { + return Err(type_error(format!( + "hex() got an unexpected keyword argument '{other}'" + ))) + } + } + } + let sep: Option = match &sep_obj { + None => None, Some(Object::Str(s)) => { - let bytes = s.as_bytes(); - if bytes.len() != 1 { + let mut chars = s.chars(); + match (chars.next(), chars.next()) { + (Some(c), None) => { + if (c as u32) > 0x7f { + return Err(value_error("sep must be ASCII.")); + } + Some(c as u8) + } + _ => return Err(value_error("sep must be length 1.")), + } + } + Some(Object::Bytes(b)) => { + if b.len() != 1 { return Err(value_error("sep must be length 1.")); } - Some(bytes[0]) + if b[0] > 0x7f { + return Err(value_error("sep must be ASCII.")); + } + Some(b[0]) + } + Some(other) => { + return Err(type_error(format!( + "sep must be str or bytes, not {}", + other.type_name() + ))) } - Some(Object::Bytes(b)) if b.len() == 1 => Some(b[0]), - Some(Object::None) | None => None, - _ => return Err(type_error("sep must be a 1-byte string")), }; - let bytes_per_sep = match args.get(2) { + let bytes_per_sep = match &bps_obj { Some(Object::Int(i)) => *i, Some(Object::Bool(b)) => i64::from(*b), + Some(Object::Long(_)) => { + return Err(crate::error::overflow_error( + "Python int too large to convert to C int", + )) + } None => 1, - _ => return Err(type_error("bytes_per_sep must be int")), + Some(other) => { + return Err(type_error(format!( + "'{}' object cannot be interpreted as an integer", + other.type_name() + ))) + } }; let mut out = String::with_capacity(data.len() * 2); let step = bytes_per_sep.unsigned_abs() as usize; @@ -7160,36 +7683,8 @@ fn bytes_fromhex(args: &[Object]) -> Result { } else { args.first() }; - let s = match s_obj { - Some(Object::Str(s)) => s.to_string(), - _ => return Err(type_error("fromhex() argument must be str")), - }; - let mut bytes = Vec::new(); - let mut last_high: Option = None; - for c in s.chars() { - if c.is_whitespace() { - if last_high.is_some() { - return Err(value_error("non-hexadecimal number")); - } - continue; - } - let v = c.to_digit(16).ok_or_else(|| { - value_error(format!( - "non-hexadecimal number found in fromhex() arg at position {}", - c.len_utf8() - )) - })? as u8; - match last_high { - Some(hi) => { - bytes.push((hi << 4) | v); - last_high = None; - } - None => last_high = Some(v), - } - } - if last_high.is_some() { - return Err(value_error("non-hexadecimal number")); - } + let s = fromhex_string_arg(s_obj)?; + let bytes = parse_hex_bytes(&s)?; // Decide return type based on receiver: bytearray.fromhex returns bytearray; // bytes.fromhex returns bytes. if matches!(args.first(), Some(Object::ByteArray(_))) { @@ -7204,8 +7699,14 @@ fn bytes_startswith(args: &[Object]) -> Result { let target = args .get(1) .ok_or_else(|| type_error("startswith() expected 1 arg"))?; + let (start, end, invalid) = bytes_search_range(args, data.len()); + if invalid { + return Ok(Object::Bool(false)); + } Ok(Object::Bool(bytes_match_prefix_suffix( - &data, target, true, + &data[start..end], + target, + true, )?)) } @@ -7214,8 +7715,14 @@ fn bytes_endswith(args: &[Object]) -> Result { let target = args .get(1) .ok_or_else(|| type_error("endswith() expected 1 arg"))?; + let (start, end, invalid) = bytes_search_range(args, data.len()); + if invalid { + return Ok(Object::Bool(false)); + } Ok(Object::Bool(bytes_match_prefix_suffix( - &data, target, false, + &data[start..end], + target, + false, )?)) } @@ -7224,6 +7731,7 @@ fn bytes_match_prefix_suffix( target: &Object, prefix: bool, ) -> Result { + let name = if prefix { "startswith" } else { "endswith" }; let test = |needle: &[u8]| { if prefix { data.starts_with(needle) @@ -7234,7 +7742,13 @@ fn bytes_match_prefix_suffix( match target { Object::Tuple(parts) => { for item in parts.iter() { - let needle = bytes_argview(item)?; + let needle = bytes_argview(item).map_err(|_| { + type_error(format!( + "tuple for {name} must only contain bytes-like objects, \ + not '{}'", + item.type_name() + )) + })?; if test(&needle) { return Ok(true); } @@ -7242,38 +7756,51 @@ fn bytes_match_prefix_suffix( Ok(false) } _ => { - let needle = bytes_argview(target)?; + let needle = bytes_argview(target).map_err(|_| { + type_error(format!( + "{name} first arg must be bytes or a tuple of bytes, not {}", + target.type_name() + )) + })?; Ok(test(&needle)) } } } /// Resolve the optional `start`/`end` arguments of `bytes.find` and -/// friends (positions 2 and 3) into a clamped `[start, end]` byte -/// window, applying CPython's slice-style negative-index handling. -fn bytes_search_range(args: &[Object], len: usize) -> (usize, usize) { +/// friends (positions 2 and 3) the way CPython's `ADJUST_INDICES` +/// does: negative indices are offset by the length and floored at 0, +/// `end` is clamped to the length but `start` is **not** — a start +/// past the end makes the window invalid (third tuple slot), which +/// matters for empty needles (`b'abc'.find(b'', 4) == -1`). +fn bytes_search_range(args: &[Object], len: usize) -> (usize, usize, bool) { let n = len as i64; let resolve = |o: Option<&Object>, default: i64| -> i64 { match o { None | Some(Object::None) => default, Some(obj) => match obj.as_i64() { - Some(mut x) => { + Some(x) => { if x < 0 { - x += n; + (x + n).max(0) + } else { + x } - x.clamp(0, n) } None => default, }, } }; - let start = resolve(args.get(2), 0).clamp(0, n) as usize; - let end = resolve(args.get(3), n).clamp(0, n) as usize; - (start, end.max(start)) + let raw_start = resolve(args.get(2), 0); + let end = resolve(args.get(3), n).clamp(0, n); + let invalid = raw_start > end; + let start = raw_start.clamp(0, end.max(0)); + (start as usize, end as usize, invalid) } /// Find `sub` within `data[start..end]`, returning the *absolute* /// position (or -1). Mirrors `bytes.find`'s empty-needle behaviour. +/// `memmem` is O(n + m) like CPython's stringlib fastsearch — the +/// suite checks this (`test_adaptive_find` with megabyte needles). fn bytes_find_in(data: &[u8], sub: &[u8], start: usize, end: usize) -> i64 { if start > end || end > data.len() { return -1; @@ -7282,45 +7809,61 @@ fn bytes_find_in(data: &[u8], sub: &[u8], start: usize, end: usize) -> i64 { if sub.is_empty() { return start as i64; } - if sub.len() > hay.len() { - return -1; + memchr::memmem::find(hay, sub).map_or(-1, |i| (start + i) as i64) +} + +/// gh-142560: converting a search argument can run Python code (a user +/// `__index__`) that mutates the receiving bytearray while the search +/// "holds its buffer". CPython raises `BufferError`; we emulate by +/// snapshotting the length around the conversion. +fn bytes_needle_guarded(args: &[Object], arg: &Object) -> Result, RuntimeError> { + if let Some(Object::ByteArray(cell)) = args.first() { + let before = cell.borrow().len(); + let sub = bytes_find_needle(arg)?; + if cell.borrow().len() != before { + return Err(RuntimeError::PyException( + crate::error::PyException::from_builtin( + "BufferError", + "Existing exports of data: object cannot be re-sized", + ), + )); + } + Ok(sub) + } else { + bytes_find_needle(arg) } - hay.windows(sub.len()) - .position(|w| w == sub) - .map_or(-1, |i| (start + i) as i64) } fn bytes_find(args: &[Object]) -> Result { - let data = bytes_data(args)?; - let sub = bytes_argview( + let sub = bytes_needle_guarded( + args, args.get(1) .ok_or_else(|| type_error("find() expected 1 arg"))?, )?; - let (start, end) = bytes_search_range(args, data.len()); + let data = bytes_data(args)?; + let (start, end, invalid) = bytes_search_range(args, data.len()); + if invalid { + return Ok(Object::Int(-1)); + } Ok(Object::Int(bytes_find_in(&data, &sub, start, end))) } fn bytes_rfind(args: &[Object]) -> Result { - let data = bytes_data(args)?; - let sub = bytes_argview( + let sub = bytes_needle_guarded( + args, args.get(1) .ok_or_else(|| type_error("rfind() expected 1 arg"))?, )?; - let (start, end) = bytes_search_range(args, data.len()); - if start > end || end > data.len() { + let data = bytes_data(args)?; + let (start, end, invalid) = bytes_search_range(args, data.len()); + if invalid || end > data.len() { return Ok(Object::Int(-1)); } if sub.is_empty() { return Ok(Object::Int(end as i64)); } - let mut last = -1i64; - if sub.len() <= end - start { - for i in start..=end - sub.len() { - if data[i..i + sub.len()] == sub[..] { - last = i as i64; - } - } - } + let last = memchr::memmem::rfind(&data[start..end], &sub) + .map_or(-1, |i| (start + i) as i64); Ok(Object::Int(last)) } @@ -7331,54 +7874,53 @@ fn bytes_index(args: &[Object]) -> Result { } } +fn bytes_rindex(args: &[Object]) -> Result { + match bytes_rfind(args)? { + Object::Int(i) if i >= 0 => Ok(Object::Int(i)), + _ => Err(value_error("subsection not found")), + } +} + fn bytes_count(args: &[Object]) -> Result { - let data = bytes_data(args)?; - let sub = bytes_argview( + let sub = bytes_needle_guarded( + args, args.get(1) .ok_or_else(|| type_error("count() expected 1 arg"))?, )?; - let (start, end) = bytes_search_range(args, data.len()); + let data = bytes_data(args)?; + let (start, end, invalid) = bytes_search_range(args, data.len()); + if invalid { + return Ok(Object::Int(0)); + } if sub.is_empty() { return Ok(Object::Int((end - start) as i64 + 1)); } - let mut n = 0i64; - let mut i = start; - while i + sub.len() <= end { - if data[i..i + sub.len()] == sub[..] { - n += 1; - i += sub.len(); - } else { - i += 1; - } - } + // Non-overlapping occurrences, like CPython's `stringlib_count`. + let n = memchr::memmem::find_iter(&data[start..end], &sub).count() as i64; Ok(Object::Int(n)) } fn bytes_lower(args: &[Object]) -> Result { - Ok(Object::new_bytes( - bytes_data(args)? - .iter() - .map(|b| b.to_ascii_lowercase()) - .collect::>(), - )) + let out: Vec = bytes_data(args)? + .iter() + .map(|b| b.to_ascii_lowercase()) + .collect(); + Ok(bytes_like_result(args, out)) } fn bytes_upper(args: &[Object]) -> Result { - Ok(Object::new_bytes( - bytes_data(args)? - .iter() - .map(|b| b.to_ascii_uppercase()) - .collect::>(), - )) + let out: Vec = bytes_data(args)? + .iter() + .map(|b| b.to_ascii_uppercase()) + .collect(); + Ok(bytes_like_result(args, out)) } fn bytes_strip(args: &[Object]) -> Result { let data = bytes_data(args)?; let trim_set: Vec = match args.get(1) { - Some(Object::Bytes(b)) => b.to_vec(), - Some(Object::ByteArray(b)) => b.borrow().clone(), None | Some(Object::None) => b" \t\n\r\x0b\x0c".to_vec(), - _ => return Err(type_error("strip() expected bytes")), + Some(other) => bytes_argview(other)?, }; let start = data .iter() @@ -7388,79 +7930,248 @@ fn bytes_strip(args: &[Object]) -> Result { .iter() .rposition(|b| !trim_set.contains(b)) .map_or(start, |i| i + 1); - Ok(Object::new_bytes(data[start..end].to_vec())) + Ok(bytes_like_result(args, data[start..end].to_vec())) } fn bytes_lstrip(args: &[Object]) -> Result { let data = bytes_data(args)?; let trim_set: Vec = match args.get(1) { - Some(Object::Bytes(b)) => b.to_vec(), - Some(Object::ByteArray(b)) => b.borrow().clone(), None | Some(Object::None) => b" \t\n\r\x0b\x0c".to_vec(), - _ => return Err(type_error("lstrip() expected bytes")), + Some(other) => bytes_argview(other)?, }; let start = data .iter() .position(|b| !trim_set.contains(b)) .unwrap_or(data.len()); - Ok(Object::new_bytes(data[start..].to_vec())) + Ok(bytes_like_result(args, data[start..].to_vec())) } fn bytes_rstrip(args: &[Object]) -> Result { let data = bytes_data(args)?; let trim_set: Vec = match args.get(1) { - Some(Object::Bytes(b)) => b.to_vec(), - Some(Object::ByteArray(b)) => b.borrow().clone(), None | Some(Object::None) => b" \t\n\r\x0b\x0c".to_vec(), - _ => return Err(type_error("rstrip() expected bytes")), + Some(other) => bytes_argview(other)?, }; let end = data .iter() .rposition(|b| !trim_set.contains(b)) .map_or(0, |i| i + 1); - Ok(Object::new_bytes(data[..end].to_vec())) + Ok(bytes_like_result(args, data[..end].to_vec())) } -fn bytes_split(args: &[Object]) -> Result { +/// Shared argument parsing for `bytes.split` / `rsplit`: +/// `(sep=None, maxsplit=-1)`, both passable as keywords. +fn bytes_split_args( + args: &[Object], + kwargs: &[(String, Object)], + name: &str, +) -> Result<(Vec, Option>, i64), RuntimeError> { let data = bytes_data(args)?; - let sep: Option> = match args.get(1) { + let mut sep_obj = args.get(1).cloned(); + let mut maxsplit_obj = args.get(2).cloned(); + for (k, v) in kwargs { + match k.as_str() { + "sep" => sep_obj = Some(v.clone()), + "maxsplit" => maxsplit_obj = Some(v.clone()), + other => { + return Err(type_error(format!( + "{name}() got an unexpected keyword argument '{other}'" + ))) + } + } + } + let sep = match sep_obj { None | Some(Object::None) => None, - Some(Object::Bytes(b)) => Some(b.to_vec()), - Some(Object::ByteArray(b)) => Some(b.borrow().clone()), - _ => return Err(type_error("split() expected bytes")), - }; - let parts: Vec> = match sep { - None => data - .split(|c| matches!(c, b' ' | b'\t' | b'\n' | b'\r' | b'\x0b' | b'\x0c')) - .filter(|s| !s.is_empty()) - .map(<[u8]>::to_vec) - .collect(), - Some(sep) if !sep.is_empty() => { - let mut out: Vec> = Vec::new(); - let mut start = 0; + Some(other) => { + // Same reentrancy hazard as the find family (gh-142560): + // converting `sep` can run user code that resizes the + // receiving bytearray. + if let Some(Object::ByteArray(cell)) = args.first() { + let before = cell.borrow().len(); + let sep = bytes_argview(&other)?; + if cell.borrow().len() != before { + return Err(RuntimeError::PyException( + crate::error::PyException::from_builtin( + "BufferError", + "Existing exports of data: object cannot be re-sized", + ), + )); + } + Some(sep) + } else { + Some(bytes_argview(&other)?) + } + } + }; + if let Some(s) = &sep { + if s.is_empty() { + return Err(value_error("empty separator")); + } + } + let maxsplit = match maxsplit_obj { + None => -1, + Some(o) => o + .as_i64() + .ok_or_else(|| type_error("integer argument expected"))?, + }; + Ok((data, sep, maxsplit)) +} + +fn bytes_split_kw( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result { + let (data, sep, maxsplit) = bytes_split_args(args, kwargs, "split")?; + let mut parts: Vec> = Vec::new(); + match sep { + None => { let mut i = 0; - while i + sep.len() <= data.len() { - if data[i..i + sep.len()] == sep[..] { - out.push(data[start..i].to_vec()); - i += sep.len(); - start = i; - } else { + let mut nsplit = 0i64; + while i < data.len() { + while i < data.len() && byte_is_pyspace(data[i]) { i += 1; } + if i >= data.len() { + break; + } + if maxsplit >= 0 && nsplit >= maxsplit { + parts.push(data[i..].to_vec()); + break; + } + let start = i; + while i < data.len() && !byte_is_pyspace(data[i]) { + i += 1; + } + parts.push(data[start..i].to_vec()); + nsplit += 1; } - out.push(data[start..].to_vec()); - out } - _ => vec![data], - }; + Some(sep) => { + let mut start = 0; + let mut nsplit = 0i64; + while maxsplit < 0 || nsplit < maxsplit { + match memchr::memmem::find(&data[start..], &sep) { + Some(rel) => { + parts.push(data[start..start + rel].to_vec()); + start += rel + sep.len(); + nsplit += 1; + } + None => break, + } + } + parts.push(data[start..].to_vec()); + } + } + let is_ba = matches!(args.first(), Some(Object::ByteArray(_))); Ok(Object::new_list( - parts.into_iter().map(Object::new_bytes).collect(), + parts + .into_iter() + .map(|p| { + if is_ba { + Object::new_bytearray(p) + } else { + Object::new_bytes(p) + } + }) + .collect(), + )) +} + +fn bytes_rsplit_kw( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result { + let (data, sep, maxsplit) = bytes_split_args(args, kwargs, "rsplit")?; + let mut parts: Vec> = Vec::new(); + match sep { + None => { + let mut i = data.len(); + let mut nsplit = 0i64; + while i > 0 { + while i > 0 && byte_is_pyspace(data[i - 1]) { + i -= 1; + } + if i == 0 { + break; + } + if maxsplit >= 0 && nsplit >= maxsplit { + parts.push(data[..i].to_vec()); + break; + } + let end = i; + while i > 0 && !byte_is_pyspace(data[i - 1]) { + i -= 1; + } + parts.push(data[i..end].to_vec()); + nsplit += 1; + } + parts.reverse(); + } + Some(sep) => { + let mut end = data.len(); + let mut nsplit = 0i64; + while maxsplit < 0 || nsplit < maxsplit { + match memchr::memmem::rfind(&data[..end], &sep) { + Some(pos) => { + parts.push(data[pos + sep.len()..end].to_vec()); + end = pos; + nsplit += 1; + } + None => break, + } + } + parts.push(data[..end].to_vec()); + parts.reverse(); + } + } + let is_ba = matches!(args.first(), Some(Object::ByteArray(_))); + Ok(Object::new_list( + parts + .into_iter() + .map(|p| { + if is_ba { + Object::new_bytearray(p) + } else { + Object::new_bytes(p) + } + }) + .collect(), )) } fn bytes_splitlines(args: &[Object]) -> Result { + bytes_splitlines_kw(args, &[]) +} + +fn bytes_splitlines_kw( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result { let data = bytes_data(args)?; - let keepends = matches!(args.get(1), Some(Object::Bool(true))); + if args.len() > 2 { + return Err(type_error(format!( + "splitlines() takes at most 1 argument ({} given)", + args.len() - 1 + ))); + } + let mut keepends_obj = args.get(1).cloned(); + for (k, v) in kwargs { + match k.as_str() { + "keepends" => keepends_obj = Some(v.clone()), + other => { + return Err(type_error(format!( + "splitlines() got an unexpected keyword argument '{other}'" + ))) + } + } + } + let keepends = match &keepends_obj { + None => false, + Some(o) => o + .as_i64() + .map(|v| v != 0) + .ok_or_else(|| type_error("an integer is required"))?, + }; let mut out: Vec = Vec::new(); let mut start = 0; let mut i = 0; @@ -7476,7 +8187,7 @@ fn bytes_splitlines(args: &[Object]) -> Result { } else { &data[start..no_eol] }; - out.push(Object::new_bytes(slice.to_vec())); + out.push(bytes_like_result(args, slice.to_vec())); start = end; i = end; } else { @@ -7484,24 +8195,85 @@ fn bytes_splitlines(args: &[Object]) -> Result { } } if start < data.len() { - out.push(Object::new_bytes(data[start..].to_vec())); + out.push(bytes_like_result(args, data[start..].to_vec())); } Ok(Object::new_list(out)) } +/// `bytes.__mod__` / `bytearray.__mod__` — PEP 461 formatting through +/// the running interpreter (instances may need `__bytes__`/`__repr__`). +fn bytes_dunder_mod(args: &[Object]) -> Result { + let receiver = args + .first() + .ok_or_else(|| type_error("__mod__ requires a receiver"))?; + let other = args + .get(1) + .ok_or_else(|| type_error("__mod__ expected 1 argument"))?; + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still live on this + // thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + interp.bytes_percent_format(receiver, other, &globals) + } else { + Err(type_error("bytes %-formatting requires the interpreter")) + } +} + +/// `bytes.__rmod__`: only formats when the *left* operand is bytes-like +/// (then it's really that operand's format), otherwise `NotImplemented`. +fn bytes_dunder_rmod(args: &[Object]) -> Result { + let receiver = args + .first() + .ok_or_else(|| type_error("__rmod__ requires a receiver"))?; + let other = args + .get(1) + .ok_or_else(|| type_error("__rmod__ expected 1 argument"))?; + if matches!(other, Object::Bytes(_) | Object::ByteArray(_)) { + let swapped = [other.clone(), receiver.clone()]; + bytes_dunder_mod(&swapped) + } else { + Ok(crate::vm_singletons::not_implemented()) + } +} + fn bytes_join(args: &[Object]) -> Result { let sep = bytes_data(args)?; let it = args .get(1) .ok_or_else(|| type_error("join() expected iterable"))?; - let mut parts: Vec> = Vec::new(); - let mut iter = it.make_iter()?; - while let Some(v) = iter.next_value() { - match v { - Object::Bytes(b) => parts.push(b.to_vec()), - Object::ByteArray(b) => parts.push(b.borrow().clone()), - _ => return Err(type_error("sequence item: expected bytes")), + // Iterate through the interpreter so user iterables / generators + // work, not just native containers. + let items: Vec = match it { + Object::List(l) => l.borrow().clone(), + Object::Tuple(t) => t.to_vec(), + other => { + if let Some(ptr) = crate::vm_singletons::current_interpreter_ptr() { + // SAFETY: published by an enclosing VM frame still live on + // this thread; the GIL keeps the access exclusive. + let interp = unsafe { &mut *ptr }; + let globals = interp.builtins_dict(); + interp.collect_iterable(other, &globals)? + } else { + let mut iter = other.make_iter()?; + let mut out = Vec::new(); + while let Some(v) = iter.next_value() { + out.push(v); + } + out + } } + }; + let mut parts: Vec> = Vec::with_capacity(items.len()); + for v in &items { + let part = bytes_argview(v).map_err(|_| { + type_error(format!( + "sequence item {}: expected a bytes-like object, {} found", + parts.len(), + v.type_name() + )) + })?; + parts.push(part); } let mut out = Vec::new(); for (i, p) in parts.iter().enumerate() { @@ -7510,10 +8282,17 @@ fn bytes_join(args: &[Object]) -> Result { } out.extend_from_slice(p); } - Ok(Object::new_bytes(out)) + Ok(bytes_like_result(args, out)) } fn bytes_replace(args: &[Object]) -> Result { + bytes_replace_kw(args, &[]) +} + +fn bytes_replace_kw( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result { let data = bytes_data(args)?; let from = bytes_argview( args.get(1) @@ -7523,11 +8302,31 @@ fn bytes_replace(args: &[Object]) -> Result { args.get(2) .ok_or_else(|| type_error("replace() expected 2 args"))?, )?; + let mut max_obj = args.get(3).cloned(); + for (k, v) in kwargs { + match k.as_str() { + "count" => max_obj = Some(v.clone()), + other => { + return Err(type_error(format!( + "replace() got an unexpected keyword argument '{other}'" + ))) + } + } + } + let max = match max_obj { + None | Some(Object::None) => -1i64, + Some(o) => o + .as_i64() + .ok_or_else(|| type_error("integer argument expected"))?, + }; let mut out = Vec::new(); + let mut done = 0i64; let mut i = 0; while i < data.len() { - if i + from.len() <= data.len() && data[i..i + from.len()] == from[..] { + let within_budget = max < 0 || done < max; + if within_budget && i + from.len() <= data.len() && data[i..i + from.len()] == from[..] { out.extend_from_slice(&to); + done += 1; i += from.len().max(1); if from.is_empty() { out.push(data[i - 1]); @@ -7537,7 +8336,12 @@ fn bytes_replace(args: &[Object]) -> Result { i += 1; } } - Ok(Object::new_bytes(out)) + // An empty needle also matches at end-of-string (CPython appends a + // final replacement: `b"ab".replace(b"", b"-") == b"-a-b-"`). + if from.is_empty() && (max < 0 || done < max) { + out.extend_from_slice(&to); + } + Ok(bytes_like_result(args, out)) } /// `bytes.translate(table, /, delete=b'')` and the `bytearray` @@ -7545,9 +8349,32 @@ fn bytes_replace(args: &[Object]) -> Result { /// 256; bytes present in `delete` are dropped first. The receiver's /// type (bytes vs bytearray) is preserved. fn bytes_translate(args: &[Object]) -> Result { + bytes_translate_kw(args, &[]) +} + +fn bytes_translate_kw( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result { let data = bytes_data(args)?; + let mut delete_obj = args.get(2).cloned(); + for (k, v) in kwargs { + match k.as_str() { + "delete" => delete_obj = Some(v.clone()), + other => { + return Err(type_error(format!( + "translate() got an unexpected keyword argument '{other}'" + ))) + } + } + } let table = match args.get(1) { - None | Some(Object::None) => None, + None => { + return Err(type_error( + "translate() takes at least 1 argument (0 given)", + )) + } + Some(Object::None) => None, Some(o) => { let t = bytes_argview(o)?; if t.len() != 256 { @@ -7556,9 +8383,9 @@ fn bytes_translate(args: &[Object]) -> Result { Some(t) } }; - let delete = match args.get(2) { - None | Some(Object::None) => Vec::new(), - Some(o) => bytes_argview(o)?, + let delete = match delete_obj { + None => Vec::new(), + Some(o) => bytes_argview(&o)?, }; let mut out = Vec::with_capacity(data.len()); for &b in &data { @@ -7598,6 +8425,385 @@ fn bytes_maketrans(args: &[Object]) -> Result { Ok(Object::new_bytes(table)) } +fn bytes_partition(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let sep = bytes_argview( + args.get(1) + .ok_or_else(|| type_error("partition() expected 1 arg"))?, + )?; + if sep.is_empty() { + return Err(value_error("empty separator")); + } + let (head, mid, tail) = match memchr::memmem::find(&data, &sep) { + Some(i) => ( + data[..i].to_vec(), + sep.clone(), + data[i + sep.len()..].to_vec(), + ), + None => (data, Vec::new(), Vec::new()), + }; + Ok(Object::new_tuple(vec![ + bytes_like_result(args, head), + bytes_like_result(args, mid), + bytes_like_result(args, tail), + ])) +} + +fn bytes_rpartition(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let sep = bytes_argview( + args.get(1) + .ok_or_else(|| type_error("rpartition() expected 1 arg"))?, + )?; + if sep.is_empty() { + return Err(value_error("empty separator")); + } + let (head, mid, tail) = match memchr::memmem::rfind(&data, &sep) { + Some(i) => ( + data[..i].to_vec(), + sep.clone(), + data[i + sep.len()..].to_vec(), + ), + None => (Vec::new(), Vec::new(), data), + }; + Ok(Object::new_tuple(vec![ + bytes_like_result(args, head), + bytes_like_result(args, mid), + bytes_like_result(args, tail), + ])) +} + +fn bytes_removeprefix(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let prefix = bytes_argview( + args.get(1) + .ok_or_else(|| type_error("removeprefix() expected 1 arg"))?, + )?; + let out = if data.starts_with(&prefix) { + data[prefix.len()..].to_vec() + } else { + data + }; + Ok(bytes_like_result(args, out)) +} + +fn bytes_removesuffix(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let suffix = bytes_argview( + args.get(1) + .ok_or_else(|| type_error("removesuffix() expected 1 arg"))?, + )?; + let out = if !suffix.is_empty() && data.ends_with(&suffix) { + data[..data.len() - suffix.len()].to_vec() + } else { + data + }; + Ok(bytes_like_result(args, out)) +} + +fn bytes_expandtabs( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result { + let data = bytes_data(args)?; + let mut tabsize_obj = args.get(1).cloned(); + for (k, v) in kwargs { + if k == "tabsize" { + tabsize_obj = Some(v.clone()); + } else { + return Err(type_error(format!( + "expandtabs() got an unexpected keyword argument '{k}'" + ))); + } + } + let tabsize = match tabsize_obj { + None => 8, + Some(o) => o + .as_i64() + .ok_or_else(|| type_error("integer argument expected"))?, + }; + let mut out = Vec::with_capacity(data.len()); + let mut col: i64 = 0; + for &b in &data { + match b { + b'\t' => { + if tabsize > 0 { + let pad = tabsize - (col % tabsize); + out.extend(std::iter::repeat_n(b' ', pad as usize)); + col += pad; + } + } + b'\n' | b'\r' => { + out.push(b); + col = 0; + } + _ => { + out.push(b); + col += 1; + } + } + } + Ok(bytes_like_result(args, out)) +} + +/// Shared `center`/`ljust`/`rjust` plumbing: parse `(width, +/// fillchar=b' ')` where fillchar must be a single byte. +fn bytes_pad_args(args: &[Object], name: &str) -> Result<(Vec, i64, u8), RuntimeError> { + let data = bytes_data(args)?; + let width = args + .get(1) + .and_then(|o| o.as_i64()) + .ok_or_else(|| type_error(format!("{name}() expected integer width")))?; + let fill = match args.get(2) { + None => b' ', + Some(o) => { + let v = bytes_argview(o).ok().filter(|v| v.len() == 1); + match v { + Some(v) => v[0], + None => { + return Err(type_error(format!( + "{name}() argument 2 must be a byte string of length 1, \ + not '{}'", + o.type_name() + ))) + } + } + } + }; + Ok((data, width, fill)) +} + +fn bytes_center(args: &[Object]) -> Result { + let (data, width, fill) = bytes_pad_args(args, "center")?; + let len = data.len() as i64; + if width <= len { + return Ok(bytes_like_result(args, data)); + } + // CPython biases the extra fill to the right except when `width` + // is odd (`bytes_center` marg computation). + let marg = width - len; + let left = marg / 2 + (marg & width & 1); + let mut out = Vec::with_capacity(width as usize); + out.extend(std::iter::repeat_n(fill, left as usize)); + out.extend_from_slice(&data); + out.extend(std::iter::repeat_n(fill, (marg - left) as usize)); + Ok(bytes_like_result(args, out)) +} + +fn bytes_ljust(args: &[Object]) -> Result { + let (data, width, fill) = bytes_pad_args(args, "ljust")?; + let mut out = data; + while (out.len() as i64) < width { + out.push(fill); + } + Ok(bytes_like_result(args, out)) +} + +fn bytes_rjust(args: &[Object]) -> Result { + let (data, width, fill) = bytes_pad_args(args, "rjust")?; + let len = data.len() as i64; + let mut out = Vec::with_capacity(width.max(len) as usize); + out.extend(std::iter::repeat_n(fill, (width - len).max(0) as usize)); + out.extend_from_slice(&data); + Ok(bytes_like_result(args, out)) +} + +fn bytes_zfill(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let width = args + .get(1) + .and_then(|o| o.as_i64()) + .ok_or_else(|| type_error("zfill() expected integer width"))?; + let len = data.len() as i64; + if width <= len { + return Ok(bytes_like_result(args, data)); + } + let pad = (width - len) as usize; + let mut out = Vec::with_capacity(width as usize); + let body = if !data.is_empty() && (data[0] == b'+' || data[0] == b'-') { + out.push(data[0]); + &data[1..] + } else { + &data[..] + }; + out.extend(std::iter::repeat_n(b'0', pad)); + out.extend_from_slice(body); + Ok(bytes_like_result(args, out)) +} + +fn bytes_capitalize(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let mut out = Vec::with_capacity(data.len()); + for (i, &b) in data.iter().enumerate() { + out.push(if i == 0 { + b.to_ascii_uppercase() + } else { + b.to_ascii_lowercase() + }); + } + Ok(bytes_like_result(args, out)) +} + +fn bytes_title(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let mut out = Vec::with_capacity(data.len()); + let mut prev_alpha = false; + for &b in &data { + if b.is_ascii_alphabetic() { + out.push(if prev_alpha { + b.to_ascii_lowercase() + } else { + b.to_ascii_uppercase() + }); + prev_alpha = true; + } else { + out.push(b); + prev_alpha = false; + } + } + Ok(bytes_like_result(args, out)) +} + +fn bytes_swapcase(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let out: Vec = data + .iter() + .map(|b| { + if b.is_ascii_uppercase() { + b.to_ascii_lowercase() + } else if b.is_ascii_lowercase() { + b.to_ascii_uppercase() + } else { + *b + } + }) + .collect(); + Ok(bytes_like_result(args, out)) +} + +fn bytes_islower(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let has_cased = data.iter().any(u8::is_ascii_lowercase); + let no_upper = !data.iter().any(u8::is_ascii_uppercase); + Ok(Object::Bool(has_cased && no_upper)) +} + +fn bytes_isupper(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let has_cased = data.iter().any(u8::is_ascii_uppercase); + let no_lower = !data.iter().any(u8::is_ascii_lowercase); + Ok(Object::Bool(has_cased && no_lower)) +} + +fn bytes_istitle(args: &[Object]) -> Result { + let data = bytes_data(args)?; + let mut cased = false; + let mut prev_cased = false; + for &b in &data { + if b.is_ascii_uppercase() { + if prev_cased { + return Ok(Object::Bool(false)); + } + cased = true; + prev_cased = true; + } else if b.is_ascii_lowercase() { + if !prev_cased { + return Ok(Object::Bool(false)); + } + cased = true; + prev_cased = true; + } else { + prev_cased = false; + } + } + Ok(Object::Bool(cased)) +} + +fn bytes_isascii(args: &[Object]) -> Result { + let data = bytes_data(args)?; + Ok(Object::Bool(data.iter().all(u8::is_ascii))) +} + +// ---- bytearray-only mutators beyond append/extend/pop/clear ------ + +fn bytearray_only(args: &[Object], name: &str) -> Result>>, RuntimeError> { + match args.first() { + Some(Object::ByteArray(b)) => Ok(b.clone()), + _ => Err(type_error(format!("{name}() requires a bytearray receiver"))), + } +} + +/// `_getbytevalue`: an int in `range(0, 256)` via the full +/// `__index__` protocol (native unwrap or interpreter reentry). +/// Used by `insert`/`remove`/`append` and bytearray item assignment. +pub(crate) fn bytearray_byte_arg(arg: &Object) -> Result { + let native = arg.native_value(); + match native.as_ref().unwrap_or(arg) { + Object::Bool(v) => Ok(u8::from(*v)), + Object::Int(v) if (0..=255).contains(v) => Ok(*v as u8), + Object::Int(_) | Object::Long(_) => { + Err(value_error("byte must be in range(0, 256)")) + } + inst @ Object::Instance(_) + if crate::instance_method(inst, "__index__").is_some() => + { + let v = coerce_index_i64(inst)?; + if (0..=255).contains(&v) { + Ok(v as u8) + } else { + Err(value_error("byte must be in range(0, 256)")) + } + } + other => Err(type_error(format!( + "'{}' object cannot be interpreted as an integer", + other.type_name() + ))), + } +} + +fn bytearray_insert(args: &[Object]) -> Result { + let cell = bytearray_only(args, "insert")?; + let pos = args + .get(1) + .and_then(|o| o.as_i64()) + .ok_or_else(|| type_error("insert() expected integer index"))?; + let byte = bytearray_byte_arg( + args.get(2) + .ok_or_else(|| type_error("insert() expected 2 args"))?, + )?; + let mut data = cell.borrow_mut(); + let len = data.len() as i64; + let idx = if pos < 0 { + (len + pos).max(0) + } else { + pos.min(len) + } as usize; + data.insert(idx, byte); + Ok(Object::None) +} + +fn bytearray_remove(args: &[Object]) -> Result { + let cell = bytearray_only(args, "remove")?; + let byte = bytearray_byte_arg( + args.get(1) + .ok_or_else(|| type_error("remove() expected 1 arg"))?, + )?; + let mut data = cell.borrow_mut(); + match data.iter().position(|b| *b == byte) { + Some(i) => { + data.remove(i); + Ok(Object::None) + } + None => Err(value_error("value not found in bytearray")), + } +} + +fn bytearray_copy(args: &[Object]) -> Result { + let cell = bytearray_only(args, "copy")?; + let data = cell.borrow().clone(); + Ok(Object::new_bytearray(data)) +} + fn bytes_isalnum(args: &[Object]) -> Result { let data = bytes_data(args)?; Ok(Object::Bool( diff --git a/crates/weavepy-vm/src/error.rs b/crates/weavepy-vm/src/error.rs index 336cc12..89bb610 100644 --- a/crates/weavepy-vm/src/error.rs +++ b/crates/weavepy-vm/src/error.rs @@ -36,6 +36,11 @@ pub struct PyException { /// re-raises preserve the original traceback — the re-raise /// location is not recorded). pub suppress_tb_once: bool, + /// True once implicit-context chaining has been decided for this + /// exception (CPython chains exactly once, in `_PyErr_SetObject` at + /// the raise site). Propagation through Rust boundaries must not + /// re-chain — user code may have set `__context__ = None` since. + pub context_settled: bool, } impl PyException { @@ -46,6 +51,7 @@ impl PyException { context: None, cause: None, suppress_tb_once: false, + context_settled: false, } } diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 55f923d..697dde2 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -764,6 +764,27 @@ impl Interpreter { /// (containers, caches, other bindings) skips the reap and leaves /// the object to the cycle collector. fn prompt_reap_dropped(&mut self, dropped: Object) { + // A discarded never-driven `asend`/`athrow`/`aclose` awaitable + // warns at finalization (gh-113753; CPython's + // `async_gen_asend_finalize` family). + if let Object::AsyncGenAwait(a) = &dropped { + if !a.started.get() && !a.consumed.get() && Rc::strong_count(a) <= 1 { + a.consumed.set(true); + let method = match a.kind { + crate::object::AgenAwaitKind::Send => "asend", + crate::object::AgenAwaitKind::Throw => "athrow", + crate::object::AgenAwaitKind::Close => "aclose", + }; + let qualname = match &a.agen { + Object::AsyncGenerator(g) => g.qualname.borrow().clone(), + other => other.type_name_owned(), + }; + let _ = self.emit_runtime_warning(format!( + "coroutine method '{method}' of '{qualname}' was never awaited" + )); + } + return; + } let finalizable = match &dropped { Object::Instance(i) => i.cls().lookup("__del__").is_some(), Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { @@ -846,6 +867,35 @@ impl Interpreter { /// Call `obj.__del__()` if present, routing any raised exception /// through the unraisable hook. Used by both the cycle-GC drain and /// the shutdown pass. + /// CPython `_PyErr_WarnUnawaitedCoroutine`: prefer the Python hook + /// `warnings._warn_unawaited_coroutine` (it appends the `cr_origin` + /// creation traceback); a failing hook reports through + /// `sys.unraisablehook` with the coroutine as the object, and the + /// plain RuntimeWarning is still issued as a fallback. + fn warn_unawaited_coroutine(&mut self, obj: &Object, qualname: &str) { + let mut warned = false; + if let Some(hook) = self.module_attr("warnings", "_warn_unawaited_coroutine") { + let globals = self.builtins.clone(); + match self.call(&hook, &[obj.clone()], &[], &globals) { + Ok(_) => warned = true, + Err(err) => { + let outer = Rc::new(RefCell::new(DictData::new())); + let context_repr = + self.repr_of(obj, &outer).unwrap_or_else(|_| obj.repr()); + self.write_unraisable(&err, obj, &context_repr); + } + } + } + if !warned { + let message = format!("coroutine '{qualname}' was never awaited"); + if let Err(err) = self.emit_runtime_warning(message) { + let outer = Rc::new(RefCell::new(DictData::new())); + let context_repr = self.repr_of(obj, &outer).unwrap_or_else(|_| obj.repr()); + self.write_unraisable(&err, obj, &context_repr); + } + } + } + fn invoke_finalizer(&mut self, obj: &Object) { // A coroutine that was created but never driven: CPython's // `_PyGen_Finalize` emits the "was never awaited" @@ -855,13 +905,30 @@ impl Interpreter { if let Object::Coroutine(g) = obj { if matches!(&*g.state.borrow(), GeneratorState::Created(_)) { *g.state.borrow_mut() = GeneratorState::Finished; - let message = - format!("coroutine '{}' was never awaited", g.qualname.borrow()); - if let Err(err) = self.emit_runtime_warning(message) { + let qualname = g.qualname.borrow().clone(); + self.warn_unawaited_coroutine(obj, &qualname); + return; + } + } + // Mark "finalize ran" first (CPython sets the GC FINALIZED bit + // before tp_finalize): whatever happens below runs at most + // once; a generator left suspended by its finalizer will not + // be resurrected and re-finalized on its next drop. + if let Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) = obj { + g.finalize_ran.set(true); + } + // PEP 525: an async generator whose firstiter hook captured a + // finalizer routes finalization through that hook (CPython + // `_PyGen_Finalize`); asyncio's hook schedules `aclose()` on + // the owning loop instead of closing synchronously here. + if let Object::AsyncGenerator(g) = obj { + let finalizer = g.finalizer.borrow().clone(); + if !matches!(finalizer, Object::None) && !g.is_finished() { + let globals = self.builtins.clone(); + if let Err(err) = self.call(&finalizer, &[obj.clone()], &[], &globals) { let outer = Rc::new(RefCell::new(DictData::new())); - let context_repr = self - .repr_of(obj, &outer) - .unwrap_or_else(|_| obj.repr()); + let context_repr = + self.repr_of(obj, &outer).unwrap_or_else(|_| obj.repr()); self.write_unraisable(&err, obj, &context_repr); } return; @@ -1351,12 +1418,11 @@ impl Interpreter { // fresh exception — including ones raised from C // (here: Rust opcodes and builtins) — to the // currently handled exception. `RAISE_VARARGS` - // already did this at the raise site; a fresh - // Rust-raised error is recognisable by its empty - // traceback and unset context/cause. Re-raises - // carry `suppress_tb_once` or an existing dict - // `__context__` and are left alone. - if exc.context.is_none() + // already did this at the raise site + // (`context_settled`); a fresh Rust-raised error + // has empty traceback and unset context/cause. + if !exc.context_settled + && exc.context.is_none() && exc.cause.is_none() && exc.traceback.is_empty() && !exc.suppress_tb_once @@ -1584,6 +1650,14 @@ impl Interpreter { )) } 2 => { + // Clearing the frame of a never-awaited coroutine + // finalizes it — CPython's `_PyGen_Finalize` emits + // the "was never awaited" RuntimeWarning (bpo-45813). + if matches!(g.kind, crate::object::CoroutineKind::Coroutine) { + let obj = Object::Coroutine(g.clone()); + let qualname = g.qualname.borrow().clone(); + self.warn_unawaited_coroutine(&obj, &qualname); + } // Tear down the never-started generator: its frame // locals (bound arguments) die now, like CPython's // refcount-driven dealloc. Drop the snapshot's own @@ -2268,8 +2342,22 @@ impl Interpreter { frame.push(Object::Cell(cell)); } OpCode::LoadAttr => { + // `f().cr_frame`: the receiver temporary dies when the + // attribute load pops it — finalize promptly so a + // never-awaited coroutine warns here (bpo-45813). + let reap = match frame.stack.last() { + Some( + o @ (Object::Generator(_) + | Object::Coroutine(_) + | Object::AsyncGenerator(_)), + ) => Some(o.clone()), + _ => None, + }; let v = self.specialized_load_attr(frame, cache_pc, ins.arg)?; frame.push(v); + if let Some(r) = reap { + self.prompt_reap_dropped(r); + } } OpCode::StoreAttr => { self.specialized_store_attr(frame, cache_pc, ins.arg)?; @@ -2291,7 +2379,28 @@ impl Interpreter { &frame.globals.clone(), )? } else { - self.binary_subscr(&v, &i)? + // `dict.__getitem__` on a subclass dispatches a + // user-defined `__missing__(key)` instead of raising + // (CPython `dict_subscript`). + match self.binary_subscr(&v, &i) { + Err(RuntimeError::PyException(exc)) + if exc.type_name() == "KeyError" + && matches!(v.native_value(), Some(Object::Dict(_))) => + { + match instance_method(&v, "__missing__") { + Some(miss) => self.call( + &miss, + std::slice::from_ref(&i), + &[], + &frame.globals.clone(), + )?, + None => { + return Err(RuntimeError::PyException(exc)) + } + } + } + r => r?, + } } } else if let Object::Type(ty) = &v { // `Foo[args]` — CPython looks up `__getitem__` @@ -2483,7 +2592,20 @@ impl Interpreter { frame.push(Object::Bool(result)); } OpCode::PopTop => { - frame.pop()?; + let v = frame.pop()?; + // Discarding the last reference to a temporary mirrors + // CPython's refcount-driven finalization (`f()` as a + // statement finalizes the result immediately). + if matches!( + v, + Object::Generator(_) + | Object::Coroutine(_) + | Object::AsyncGenerator(_) + | Object::Instance(_) + | Object::AsyncGenAwait(_) + ) { + self.prompt_reap_dropped(v); + } } OpCode::CopyTop => { let v = frame.top()?.clone(); @@ -3106,15 +3228,24 @@ impl Interpreter { let mut exc = match ins.arg { 0 => { // Re-raise the currently-handled exception. A bare - // `raise` preserves the original traceback: the - // re-raise site is *not* recorded (CPython RERAISE). - let mut top = frame - .exc_handlers + // `raise` preserves the original traceback (the + // re-raise site is *not* recorded) and — unlike a + // fresh `raise e` — never re-chains `__context__` + // (CPython RERAISE; an explicit + // `e.__context__ = None` must survive). The active + // exception is thread-state-wide in CPython + // (`sys.exc_info()`), not frame-local: a helper + // called from inside an `except:` block can + // re-raise the caller's exception. + let mut top = self + .exc_info_stack + .borrow() .last() - .map(|(_, pe)| pe.clone()) + .cloned() .ok_or_else(|| runtime_error("No active exception to re-raise"))?; top.suppress_tb_once = true; - top + top.context_settled = true; + return Err(RuntimeError::PyException(top)); } 1 => { let arg = frame.pop()?; @@ -3212,15 +3343,41 @@ impl Interpreter { }; let mut pe = Self::normalize_exception(exc, None)?; // Re-raises keep the original traceback; the RERAISE - // site itself is not recorded (matches CPython). + // site itself is not recorded, and `__context__` is not + // re-chained (matches CPython). pe.suppress_tb_once = true; + pe.context_settled = true; Self::sync_exc_attrs(&pe); return Err(RuntimeError::PyException(pe)); } OpCode::BeforeWith => { let cm = frame.pop()?; - let exit_method = self.load_attr(&cm, "__exit__")?; - let enter_method = self.load_attr(&cm, "__enter__")?; + // CPython `BEFORE_WITH`: a missing protocol method is a + // TypeError naming the protocol, not an AttributeError + // (`__enter__` checked first, then `__exit__`). + let enter_method = self.load_attr(&cm, "__enter__").map_err(|err| { + if matches!(&err, RuntimeError::PyException(e) if e.type_name() == "AttributeError") + { + type_error(format!( + "'{}' object does not support the context manager protocol", + cm.type_name() + )) + } else { + err + } + })?; + let exit_method = self.load_attr(&cm, "__exit__").map_err(|err| { + if matches!(&err, RuntimeError::PyException(e) if e.type_name() == "AttributeError") + { + type_error(format!( + "'{}' object does not support the context manager protocol \ + (missed __exit__ method)", + cm.type_name() + )) + } else { + err + } + })?; let entered = self.call(&enter_method, &[], &[], &frame.globals)?; // Stack on exit: [exit_method, entered_value] frame.push(exit_method); @@ -3717,6 +3874,9 @@ impl Interpreter { /// `__suppress_context__` (handled in `sync_exc_attrs`), which /// governs *display*, not whether `__context__` exists. fn attach_implicit_context(&self, exc: &mut PyException) { + // Chaining is decided exactly once per raise; later propagation + // through Rust boundaries must leave the result alone. + exc.context_settled = true; let stack = self.exc_info_stack.borrow(); let Some(ctx) = stack.last() else { return; @@ -3748,6 +3908,7 @@ impl Interpreter { /// The suspended generator's active handled exception is the top of /// the entries we detached on suspend (`frame.saved_exc_info`). fn chain_thrown_context(exc: &mut PyException, frame: &Frame) { + exc.context_settled = true; let Some(active) = frame.saved_exc_info.last() else { return; }; @@ -4065,6 +4226,11 @@ impl Interpreter { "await" if prefix == "cr_" || prefix == "ag_" => { return Ok(self.gen_yieldfrom(g)) } + // PEP-style origin tracking + // (`sys.set_coroutine_origin_tracking_depth`). + "origin" if prefix == "cr_" => { + return Ok(g.origin.borrow().clone()) + } _ => {} } } @@ -4824,10 +4990,13 @@ impl Interpreter { // (`type('D', (A,), {})`, enum's functional API) would inherit // its base's or metaclass's `__name__`. if name == "__name__" || name == "__qualname__" { - if let Some(v) = ty.dict.borrow().get(&DictKey(Object::from_str(name))) { - return Ok(v.clone()); + match ty.dict.borrow().get(&DictKey(Object::from_str(name))) { + // A getset under `__name__` (generator/coroutine types) + // describes *instances*; the type's own name comes from + // the synthetic, as with CPython's `type.__name__`. + Some(v) if !matches!(v, Object::Property(_)) => return Ok(v.clone()), + _ => return Ok(Object::from_str(&ty.name)), } - return Ok(Object::from_str(&ty.name)); } let meta = ty.metaclass_or_type(); @@ -4844,19 +5013,10 @@ impl Interpreter { // (2) Look up the name in `ty` itself (and its MRO). if let Some(attr) = ty.lookup(name) { - // A `__name__`/`__qualname__` *getset* in a class dict - // describes instances (e.g. `coroutine.__name__`); for the - // class itself CPython's metaclass getset (`type.__name__`) - // takes precedence and reports the type's own name. Fall - // through to the synthetic below. - let meta_owned = matches!(name, "__name__" | "__qualname__") - && matches!(attr, Object::Property(_)); - if !meta_owned { - // Apply the descriptor protocol with no instance: classmethods - // bind to the class, plain functions stay as functions, - // staticmethods unwrap, properties remain themselves. - return self.descriptor_get(&attr, &Object::None, &owner); - } + // Apply the descriptor protocol with no instance: classmethods + // bind to the class, plain functions stay as functions, + // staticmethods unwrap, properties remain themselves. + return self.descriptor_get(&attr, &Object::None, &owner); } // (3) Fall-through to (possibly non-data) metaclass attribute. @@ -7406,6 +7566,25 @@ impl Interpreter { self.load_attr(obj, name) } + /// Crate-visible attribute store (weakproxy `__setattr__` forwarding). + pub(crate) fn store_attr_public( + &mut self, + obj: &Object, + name: &str, + value: Object, + ) -> Result<(), RuntimeError> { + self.store_attr(obj, name, value) + } + + /// Crate-visible attribute delete (weakproxy `__delattr__` forwarding). + pub(crate) fn delete_attr_public( + &mut self, + obj: &Object, + name: &str, + ) -> Result<(), RuntimeError> { + self.delete_attr(obj, name) + } + /// Crate-visible `str()` for builtins that need full dispatch /// (e.g. `BaseException.__str__` rendering a nested exception arg). pub(crate) fn stringify_public( @@ -7427,12 +7606,24 @@ impl Interpreter { return Self::require_str_result(r, "__str__"); } // A subclass of a built-in (`class S(str)`, `class F(float)`, …) - // with no custom `__str__` inherits the base type's `__str__`, - // i.e. it stringifies its native payload rather than falling back - // to `object.__str__` (the `` repr). + // with no custom `__str__` inherits the base type's `__str__` — + // but only the value types actually *have* a `tp_str` in CPython. + // Container bases (tuple/list/dict/…) don't, so `str(x)` falls + // through `object.__str__` to `repr(x)`, which dispatches a user + // `__repr__` when defined (namedtuple relies on this). if let Some(native) = &inst.native { - let native = native.clone(); - return self.stringify(&native, globals); + if matches!( + native, + Object::Int(_) + | Object::Bool(_) + | Object::Long(_) + | Object::Float(_) + | Object::Complex(_) + | Object::Str(_) + ) { + let native = native.clone(); + return self.stringify(&native, globals); + } } return self.repr_of(v, globals); } @@ -7728,15 +7919,10 @@ impl Interpreter { Err(not_awaitable(&value)) } } - // An async generator that surfaced through `__anext__` is - // already drivable via SEND; `await agen()` itself is an error. - Object::AsyncGenerator(_) => { - if ctx == 1 { - Ok(value) - } else { - Err(not_awaitable(&value)) - } - } + // Async generators implement no `__await__`; `anext()` wraps + // them in an `AsyncGenAwait` before any GET_AWAITABLE sees + // them, so a bare agen here is always an error. + Object::AsyncGenerator(_) => Err(not_awaitable(&value)), // The deferred `asend`/`athrow`/`aclose` awaitable is already a // drivable awaitable (SEND applies the op via `step_agen_await`). Object::AsyncGenAwait(_) => Ok(value), @@ -7828,12 +8014,16 @@ impl Interpreter { ) -> Result { match aiter { Object::AsyncGenerator(_) => { - // The async generator is itself the awaitable for the - // next yield (cooperative model — we don't allocate a - // fresh `async_generator_asend` like CPython does). - // `SEND` knows how to translate `StopIteration` into - // `StopAsyncIteration` for async generators. - Ok(aiter.clone()) + // A deferred `async_generator_asend` awaitable, like + // CPython's. Returning the agen itself would make + // `await anext(agen)` indistinguishable from the illegal + // `await agen` (which must raise TypeError). + self.agen_init_hooks(aiter)?; + Ok(make_agen_await( + aiter, + crate::object::AgenAwaitKind::Send, + vec![Object::None], + )) } Object::Instance(_) if instance_method(aiter, "__anext__").is_some() => { let method = instance_method(aiter, "__anext__").expect("checked"); @@ -8047,6 +8237,29 @@ impl Interpreter { } } + /// PEP 525 `async_gen_init_hooks`: the first time any of + /// `__anext__`/`asend`/`athrow`/`aclose` produces an awaitable for + /// this agen, capture the thread's *finalizer* hook on the + /// generator and invoke the *firstiter* hook with it. CPython does + /// this at awaitable-creation time (not first drive), and an + /// exception from the hook propagates to the caller. + fn agen_init_hooks(&mut self, agen: &Object) -> Result<(), RuntimeError> { + let Object::AsyncGenerator(g) = agen else { + return Ok(()); + }; + if g.hooks_inited.get() { + return Ok(()); + } + g.hooks_inited.set(true); + let (firstiter, finalizer) = crate::stdlib::sys::asyncgen_hooks(); + *g.finalizer.borrow_mut() = finalizer; + if !matches!(firstiter, Object::None) { + let globals = self.builtins.clone(); + self.call(&firstiter, &[agen.clone()], &[], &globals)?; + } + Ok(()) + } + /// CPython's `AWAITABLE_STATE_CLOSED` error: driving (or throwing into) /// an awaitable that already completed. fn agen_await_reuse_error(kind: crate::object::AgenAwaitKind) -> RuntimeError { @@ -8067,11 +8280,14 @@ impl Interpreter { let Object::AsyncGenerator(g) = &a.agen else { return None; }; + // Mid-await suspension *or* the agen's own body executing right + // now (e.g. `await anext(me)` from inside the body) both count + // as "already running" in CPython's `ag_running_async` sense. let mid_await = matches!( &*g.state.borrow(), GeneratorState::Suspended(boxed) if boxed.downcast_ref::().is_some_and(|f| !f.agen_yielded_value) - ); + ) || matches!(&*g.state.borrow(), GeneratorState::Running); if !mid_await { return None; } @@ -8106,15 +8322,42 @@ impl Interpreter { let outcome = if first { match a.kind { AgenAwaitKind::Send => { - let value = a.args.first().cloned().unwrap_or(Object::None); + // CPython `async_gen_asend_send`: a non-None value + // passed to the awaitable's own `send` replaces the + // stored payload — and a just-started agen then + // raises the usual non-None TypeError. + let value = if matches!(send_value, Object::None) { + a.args.first().cloned().unwrap_or(Object::None) + } else { + send_value.clone() + }; self.gen_method_send(&a.agen, value) } AgenAwaitKind::Throw => self.gen_method_throw(&a.agen, &a.args), - AgenAwaitKind::Close => self.gen_method_close(&a.agen), + AgenAwaitKind::Close => { + // Deliver GeneratorExit as a throw (not a one-shot + // close): the agen's cleanup may legitimately + // suspend on inner awaits (`finally: await sleep()`), + // which pass through below and keep us drivable. + if matches!( + &a.agen, + Object::AsyncGenerator(g) if g.is_finished() + ) { + a.consumed.set(true); + return Err(stop_iteration()); + } + let bt = crate::builtin_types::builtin_types(); + let exc_inst = crate::builtin_types::make_exception_with_class( + bt.generator_exit.clone(), + "", + ); + self.gen_method_throw(&a.agen, &[exc_inst]) + } } } else { self.gen_method_send(&a.agen, send_value) }; + let is_close = matches!(a.kind, AgenAwaitKind::Close); match outcome { Ok(value) => { // Did the agen suspend on an inner `await` (passing `value` @@ -8127,16 +8370,35 @@ impl Interpreter { return Ok(value); } } + a.consumed.set(true); + if is_close { + // A real `yield` while GeneratorExit is pending — + // the agen refused to exit. + return Err(crate::error::runtime_error( + "async generator ignored GeneratorExit", + )); + } // The agen yielded a consumer value / the op completed: // express completion as `StopIteration(value)` so the SEND // handler short-circuits. - a.consumed.set(true); Err(stop_iteration_with(value)) } // `StopAsyncIteration` (agen finished) and genuine exceptions // raised in the agen body propagate out of the await. Err(e) => { a.consumed.set(true); + if is_close + && matches!( + &e, + RuntimeError::PyException(pe) if matches!( + pe.type_name().as_str(), + "GeneratorExit" | "StopAsyncIteration" + ) + ) + { + // Clean exit: awaiting `aclose()` completes with None. + return Err(stop_iteration()); + } Err(e) } } @@ -8515,8 +8777,11 @@ impl Interpreter { let exc_inst = crate::builtin_types::make_exception_with_class(bt.generator_exit.clone(), ""); return match self.gen_method_throw(&a.agen, &[exc_inst]) { + // CPython `gen_close` on the asend/athrow awaitable + // reports it as a *coroutine* ignoring the exit (the + // agen-level `aclose` path says "async generator"). Ok(_yielded) => Err(crate::error::runtime_error( - "async generator ignored GeneratorExit", + "coroutine ignored GeneratorExit", )), Err(RuntimeError::PyException(exc)) if matches!( @@ -8615,10 +8880,25 @@ impl Interpreter { self.generator_throw(g, exc) } _ => { - // Non-generator iterators don't have `.throw()`; - // CPython just re-raises the exception out of the - // delegation. - Err(RuntimeError::PyException(exc)) + let globals = self.builtins.clone(); + if exc.type_name() == "GeneratorExit" { + // CPython `gen_close_iter`: ask the sub-iterator to + // close itself, then deliver the GeneratorExit to + // the delegating frame (its finally blocks run). + if let Ok(close_m) = self.load_attr(sub_iter, "close") { + self.call(&close_m, &[], &[], &globals)?; + } + return Err(RuntimeError::PyException(exc)); + } + // A custom awaitable/iterator with its own `throw` + // (e.g. the `coroutine_wrapper` from `__await__`, or + // types.coroutine's _GeneratorWrapper) handles the + // exception itself; without one, the exception is + // raised at the delegating frame's yield-from point. + match self.load_attr(sub_iter, "throw") { + Ok(throw_m) => self.call(&throw_m, &[exc.instance.clone()], &[], &globals), + Err(_) => Err(RuntimeError::PyException(exc)), + } } } } @@ -8962,6 +9242,67 @@ impl Interpreter { items.borrow_mut().extend(extra); return Ok(a.clone()); } + // PEP 584 `dict |= other` updates in place; unlike the binary + // `|` it accepts anything `dict.update` does (a mapping or an + // iterable of key/value pairs). + (Object::Dict(dst), BinOpKind::BitOr) => { + let src: Vec<(DictKey, Object)> = match b { + Object::Dict(s) => { + s.borrow().iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } + Object::Instance(_) => match b.native_value() { + Some(Object::Dict(s)) => { + s.borrow().iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } + _ => { + let pairs = self.collect_iterable(b, globals)?; + let mut out = Vec::with_capacity(pairs.len()); + for p in pairs { + let kv = self.collect_iterable(&p, globals)?; + match <[Object; 2]>::try_from(kv) { + Ok([k, v]) => out.push((DictKey(k), v)), + Err(_) => { + return Err(type_error( + "cannot convert dictionary update sequence \ + element to a key/value pair", + )) + } + } + } + out + } + }, + Object::List(_) | Object::Tuple(_) => { + let pairs = self.collect_iterable(b, globals)?; + let mut out = Vec::with_capacity(pairs.len()); + for p in pairs { + let kv = self.collect_iterable(&p, globals)?; + match <[Object; 2]>::try_from(kv) { + Ok([k, v]) => out.push((DictKey(k), v)), + Err(_) => { + return Err(type_error( + "cannot convert dictionary update sequence \ + element to a key/value pair", + )) + } + } + } + out + } + _ => { + return Err(type_error(format!( + "unsupported operand type(s) for |=: 'dict' and '{}'", + b.type_name_owned() + ))) + } + }; + let mut d = dst.borrow_mut(); + for (k, v) in src { + d.insert(k, v); + } + drop(d); + return Ok(a.clone()); + } // `set`/`frozenset` in-place set algebra. `frozenset` is // immutable, so it falls through to the binary path which // returns a fresh object; only mutable `set` mutates here. @@ -8986,8 +9327,64 @@ impl Interpreter { buf.borrow_mut().extend_from_slice(&extra); return Ok(a.clone()); } + Object::MemoryView(mv) => { + let extra = mv.to_bytes(); + buf.borrow_mut().extend_from_slice(&extra); + return Ok(a.clone()); + } _ => {} }, + // `bytearray *= n` repeats in place, preserving identity + // (CPython's `bytearray_irepeat`). + (Object::ByteArray(buf), BinOpKind::Mult) => { + let n = match b { + Object::Int(n) => Some(*n), + Object::Bool(v) => Some(i64::from(*v)), + inst @ Object::Instance(_) + if instance_method(inst, "__index__").is_some() + || inst.native_value().is_some() => + { + Some(crate::builtins::coerce_index_i64(inst)?) + } + _ => None, + }; + if let Some(n) = n { + let mut data = buf.borrow_mut(); + if n <= 0 { + data.clear(); + } else { + let unit = data.len(); + checked_repeat_count(unit, n, "bytes")?; + let original = data.clone(); + for _ in 1..n { + data.extend_from_slice(&original); + } + } + drop(data); + return Ok(a.clone()); + } + } + // `list *= n` likewise repeats in place. + (Object::List(items), BinOpKind::Mult) => { + if let Some(n) = match b { + Object::Int(n) => Some(*n), + Object::Bool(v) => Some(i64::from(*v)), + _ => None, + } { + let mut data = items.borrow_mut(); + if n <= 0 { + data.clear(); + } else { + checked_repeat_count(data.len(), n, "list")?; + let original = data.clone(); + for _ in 1..n { + data.extend_from_slice(&original); + } + } + drop(data); + return Ok(a.clone()); + } + } _ => {} } self.dispatch_binary_op(a, b, op, globals) @@ -9079,7 +9476,7 @@ impl Interpreter { /// latin-1 so it can share the text `%`-engine, then re-encoded; the /// result type follows the left operand. `%s`/`%b` dispatch `__bytes__` /// (and `%a`/`%r` `__repr__`) on user instances via the VM. - fn bytes_percent_format( + pub(crate) fn bytes_percent_format( &mut self, a: &Object, b: &Object, @@ -9112,6 +9509,18 @@ impl Interpreter { } }; let rendered = percent_format_with(&template, b, PercentMode::Bytes, &mut resolve)?; + // gh-142557: a `%a`/`%r`/`%s` callback may have mutated a bytearray + // template while we were formatting from it. + if let Object::ByteArray(t) = a { + if t.borrow().len() != template.len() { + return Err(RuntimeError::PyException( + crate::error::PyException::from_builtin( + "BufferError", + "Existing exports of data: object cannot be re-sized", + ), + )); + } + } let out: Vec = rendered.chars().map(|c| c as u8).collect(); Ok(match a { Object::ByteArray(_) => Object::new_bytearray(out), @@ -10389,7 +10798,10 @@ impl Interpreter { if let Some(setattr) = inst.cls().lookup("__setattr__") { if matches!( setattr, - Object::Function(_) | Object::BoundMethod(_) | Object::Instance(_) + Object::Function(_) + | Object::BoundMethod(_) + | Object::Instance(_) + | Object::Builtin(_) ) { self.call( &setattr, @@ -10529,7 +10941,10 @@ impl Interpreter { if let Some(delattr) = inst.cls().lookup("__delattr__") { if matches!( delattr, - Object::Function(_) | Object::BoundMethod(_) | Object::Instance(_) + Object::Function(_) + | Object::BoundMethod(_) + | Object::Instance(_) + | Object::Builtin(_) ) { self.call( &delattr, @@ -10576,6 +10991,23 @@ impl Interpreter { } Ok(()) } + // `del module.attr` removes the name from the module dict + // (CPython `module_setattro` with a NULL value). + Object::Module(m) => { + let removed = m + .dict + .borrow_mut() + .shift_remove(&DictKey(Object::from_str(name))) + .is_some(); + if removed { + Ok(()) + } else { + Err(attribute_error(format!( + "module '{}' has no attribute '{}'", + m.name, name + ))) + } + } _ => Err(type_error(format!( "'{}' object has no attribute '{}'", obj.type_name(), @@ -10743,6 +11175,38 @@ impl Interpreter { }, _ => container, }; + let is_sequence = matches!( + container, + Object::List(_) + | Object::Tuple(_) + | Object::Str(_) + | Object::Bytes(_) + | Object::ByteArray(_) + | Object::Range(_) + | Object::MemoryView(_) + ); + // Sequence indices honour the full `__index__` protocol; slices + // are pre-resolved so the container is not borrowed while user + // `__index__` code runs. + let coerced_index; + let index = match index { + Object::Slice(s) if is_sequence && slice_needs_resolution(s) => { + coerced_index = Object::Slice(Rc::new(resolve_slice_ints(s)?)); + &coerced_index + } + inst @ Object::Instance(_) + if is_sequence && instance_method(inst, "__index__").is_some() => + { + coerced_index = Object::Int(crate::builtins::coerce_index_i64(inst)?); + &coerced_index + } + Object::Long(_) if is_sequence => { + return Err(index_error( + "cannot fit 'int' into an index-sized integer", + )) + } + _ => index, + }; match (container, index) { (Object::List(items), Object::Int(i)) => { let items = items.borrow(); @@ -10862,6 +11326,14 @@ impl Interpreter { .cloned() .ok_or_else(|| key_error(key.repr())) } + (Object::Bytes(_), other) => Err(type_error(format!( + "byte indices must be integers or slices, not {}", + other.type_name() + ))), + (Object::ByteArray(_), other) => Err(type_error(format!( + "bytearray indices must be integers or slices, not {}", + other.type_name() + ))), (_, _) => Err(type_error(format!( "'{}' object is not subscriptable with '{}'", container.type_name(), @@ -10903,6 +11375,32 @@ impl Interpreter { }, _ => container, }; + let is_sequence = matches!( + container, + Object::List(_) | Object::ByteArray(_) | Object::MemoryView(_) + ); + // Sequence indices honour the full `__index__` protocol; slices + // are pre-resolved so the container is not borrowed while user + // `__index__` code runs (gh-91153). + let coerced_index; + let index = match index { + Object::Slice(s) if is_sequence && slice_needs_resolution(s) => { + coerced_index = Object::Slice(Rc::new(resolve_slice_ints(s)?)); + &coerced_index + } + inst @ Object::Instance(_) + if is_sequence && instance_method(inst, "__index__").is_some() => + { + coerced_index = Object::Int(crate::builtins::coerce_index_i64(inst)?); + &coerced_index + } + Object::Long(_) if is_sequence => { + return Err(index_error( + "cannot fit 'int' into an index-sized integer", + )) + } + _ => index, + }; match (container, index) { (Object::List(items), Object::Int(i)) => { let mut items = items.borrow_mut(); @@ -10936,15 +11434,54 @@ impl Interpreter { Ok(()) } (Object::ByteArray(b), Object::Int(i)) => { - let mut b = b.borrow_mut(); - let idx = normalize_index(*i, b.len())?; - let byte = match value { - Object::Int(v) if (0..=255).contains(&v) => v as u8, - _ => return Err(value_error("byte must be in 0..256")), + // Convert the value *before* touching the buffer: a user + // `__index__` can run Python that resizes this bytearray + // (gh-91153), so the bounds check happens afterwards + // against the current length. + let i = *i; + let byte = crate::builtins::bytearray_byte_arg(&value)?; + let mut data = b.borrow_mut(); + let idx = normalize_index(i, data.len())?; + data[idx] = byte; + Ok(()) + } + (Object::ByteArray(b), Object::Slice(s)) => { + // `ba[i:j:k] = bytes-like / iterable of ints` — collect + // the RHS into raw bytes via the buffer fast paths or + // the full iteration protocol, then splice like a list. + let replacement: Vec = match &value { + Object::Bytes(src) => src.to_vec(), + Object::ByteArray(src) => src.borrow().clone(), + _ => { + let items = self.collect_iterable(&value, globals)?; + let mut out = Vec::with_capacity(items.len()); + for item in items { + out.push(crate::builtins::bytearray_byte_arg(&item)?); + } + out + } }; - b[idx] = byte; + let mut data = b.borrow_mut(); + let mut objs: Vec = + data.iter().map(|byte| Object::Int(i64::from(*byte))).collect(); + let repl_objs: Vec = replacement + .iter() + .map(|byte| Object::Int(i64::from(*byte))) + .collect(); + apply_slice_assignment(&mut objs, s, repl_objs)?; + *data = objs + .into_iter() + .map(|o| match o { + Object::Int(v) => v as u8, + _ => unreachable!("bytearray slice splice produced non-int"), + }) + .collect(); Ok(()) } + (Object::ByteArray(_), other) => Err(type_error(format!( + "bytearray indices must be integers or slices, not {}", + other.type_name() + ))), _ => Err(type_error(format!( "'{}' object does not support item assignment", container.type_name() @@ -10965,6 +11502,21 @@ impl Interpreter { _ => None, }; let index = unwrapped.as_ref().unwrap_or(index); + let is_sequence = matches!(container, Object::List(_) | Object::ByteArray(_)); + let coerced_index; + let index = match index { + Object::Slice(s) if is_sequence && slice_needs_resolution(s) => { + coerced_index = Object::Slice(Rc::new(resolve_slice_ints(s)?)); + &coerced_index + } + inst @ Object::Instance(_) + if is_sequence && instance_method(inst, "__index__").is_some() => + { + coerced_index = Object::Int(crate::builtins::coerce_index_i64(inst)?); + &coerced_index + } + _ => index, + }; match (container, index) { (Object::List(items), Object::Int(i)) => { let mut items = items.borrow_mut(); @@ -11044,31 +11596,52 @@ impl Interpreter { } ".u.agen_aiter" => Ok(receiver.clone()), ".u.agen_anext" => match &receiver { - Object::AsyncGenerator(_) => Ok(make_agen_await( - &receiver, - crate::object::AgenAwaitKind::Send, - vec![Object::None], - )), + Object::AsyncGenerator(_) => { + self.agen_init_hooks(&receiver)?; + Ok(make_agen_await( + &receiver, + crate::object::AgenAwaitKind::Send, + vec![Object::None], + )) + } other => Err(type_error(format!( "__anext__ requires an async_generator, got '{}'", other.type_name() ))), }, - ".u.agen_send" => Ok(make_agen_await( - &receiver, - crate::object::AgenAwaitKind::Send, - vec![rest.first().cloned().unwrap_or(Object::None)], - )), - ".u.agen_throw" => Ok(make_agen_await( - &receiver, - crate::object::AgenAwaitKind::Throw, - rest.to_vec(), - )), - ".u.agen_close" => Ok(make_agen_await( - &receiver, - crate::object::AgenAwaitKind::Close, - Vec::new(), - )), + ".u.agen_send" => { + self.agen_init_hooks(&receiver)?; + Ok(make_agen_await( + &receiver, + crate::object::AgenAwaitKind::Send, + vec![rest.first().cloned().unwrap_or(Object::None)], + )) + } + ".u.agen_throw" => { + // CPython warns at the `athrow(...)` call + // itself, before the awaitable is driven. + if rest.len() > 1 { + self.emit_deprecation_warning( + "the (type, exc, tb) signature of athrow() is deprecated, \ + use the single-arg signature instead." + .to_owned(), + )?; + } + self.agen_init_hooks(&receiver)?; + Ok(make_agen_await( + &receiver, + crate::object::AgenAwaitKind::Throw, + rest.to_vec(), + )) + } + ".u.agen_close" => { + self.agen_init_hooks(&receiver)?; + Ok(make_agen_await( + &receiver, + crate::object::AgenAwaitKind::Close, + Vec::new(), + )) + } _ => Err(RuntimeError::Internal(format!( "unknown unbound gen sentinel {}", b.name @@ -11627,11 +12200,13 @@ impl Interpreter { // path, so they are unaffected.) ".agen_anext" => match &bm.receiver { Object::AsyncGenerator(_) => { + let receiver = bm.receiver.clone(); + self.agen_init_hooks(&receiver)?; return Ok(make_agen_await( - &bm.receiver, + &receiver, crate::object::AgenAwaitKind::Send, vec![Object::None], - )) + )); } other => { return Err(type_error(format!( @@ -11648,23 +12223,38 @@ impl Interpreter { // eagerly and returned the result, yielding `await // None`). See [`Self::step_agen_await`]. ".agen_send" => { + let receiver = bm.receiver.clone(); let value = args.first().cloned().unwrap_or(Object::None); + self.agen_init_hooks(&receiver)?; return Ok(make_agen_await( - &bm.receiver, + &receiver, crate::object::AgenAwaitKind::Send, vec![value], )); } ".agen_throw" => { + // CPython warns at the `athrow(...)` call + // itself, before the awaitable is driven. + if args.len() > 1 { + self.emit_deprecation_warning( + "the (type, exc, tb) signature of athrow() is deprecated, \ + use the single-arg signature instead." + .to_owned(), + )?; + } + let receiver = bm.receiver.clone(); + self.agen_init_hooks(&receiver)?; return Ok(make_agen_await( - &bm.receiver, + &receiver, crate::object::AgenAwaitKind::Throw, args.to_vec(), )); } ".agen_close" => { + let receiver = bm.receiver.clone(); + self.agen_init_hooks(&receiver)?; return Ok(make_agen_await( - &bm.receiver, + &receiver, crate::object::AgenAwaitKind::Close, Vec::new(), )); @@ -13208,6 +13798,9 @@ impl Interpreter { } if let Some(builtin) = self.builtin_constructor_for(&cls) { if !kwargs.is_empty() { + if let Some(call_kw) = &builtin.call_kw { + return call_kw(args, kwargs); + } return Err(type_error(format!( "{}() does not accept keyword arguments", cls.name @@ -13581,9 +14174,11 @@ impl Interpreter { "" }; let given_verb = if provided == 1 { "was" } else { "were" }; + // CPython renders these with `co_qualname` (`Class.meth()` / + // `outer..f()`), not the bare name. return Err(type_error(format!( "{}() takes {} positional argument{} but {} {} given", - f.name, sig, plural, provided, given_verb + code.qualname, sig, plural, provided, given_verb ))); } // Keyword args: match by name. Unmatched ones go into the @@ -13752,6 +14347,33 @@ impl Interpreter { false, ); if code.is_generator || code.is_coroutine || code.is_async_generator { + // `cr_origin`: when origin tracking is on, snapshot the + // creation call stack (caller-outwards, like CPython's + // `compute_cr_origin`) before the bootstrap touches the + // frame stack. + let cr_origin = if code.is_coroutine { + let depth = crate::stdlib::sys::coroutine_origin_tracking_depth(); + if depth > 0 { + let stack = self.frame_stack.borrow(); + let frames: Vec = stack + .iter() + .rev() + .take(depth as usize) + .map(|py| { + Object::new_tuple(vec![ + Object::from_str(py.code.filename.clone()), + Object::Int(i64::from(py.current_lineno())), + Object::from_str(py.code.name.clone()), + ]) + }) + .collect(); + Some(Object::new_tuple(frames)) + } else { + None + } + } else { + None + }; // Run the bootstrap so the frame is past // `RETURN_GENERATOR; POP_TOP; RESUME`. We then wrap the // frame in a PyGenerator and hand it back to the caller. @@ -13785,6 +14407,9 @@ impl Interpreter { gen_code, Box::new(frame), )); + if let Some(origin) = cr_origin { + *gen.origin.borrow_mut() = origin; + } let obj = if code.is_coroutine { Object::Coroutine(gen) } else if code.is_async_generator { @@ -15087,29 +15712,76 @@ fn current_package(globals: &Rc>) -> Option { None } +/// Resolve one slice bound to an `i64` the way `PySlice_Unpack` does: +/// `None` → `Ok(None)`, ints/bools directly, big ints clamped to the +/// extremes, and `__index__`-able objects through interpreter reentry. +pub(crate) fn slice_bound_index(o: &Object) -> Result, RuntimeError> { + match o { + Object::None => Ok(None), + Object::Int(i) => Ok(Some(*i)), + Object::Bool(b) => Ok(Some(i64::from(*b))), + Object::Long(l) => { + use num_traits::{Signed, ToPrimitive}; + Ok(Some(l.to_i64().unwrap_or(if l.is_negative() { + i64::MIN + } else { + i64::MAX + }))) + } + Object::Instance(_) => Ok(Some(crate::builtins::coerce_index_i64(o).map_err( + |_| { + type_error( + "slice indices must be integers or None or have an __index__ method", + ) + }, + )?)), + _ => Err(type_error( + "slice indices must be integers or None or have an __index__ method", + )), + } +} + +/// True when a slice has a bound that needs active resolution (a big +/// int, bool, or `__index__`-able object rather than `None`/`Int`). +fn slice_needs_resolution(s: &PySlice) -> bool { + [&s.start, &s.stop, &s.step] + .iter() + .any(|o| !matches!(o, Object::None | Object::Int(_))) +} + +/// Pre-resolve every bound of a slice to `None`/`Int`. Run *before* +/// borrowing the target container: `__index__` reentry can execute +/// arbitrary Python (including code that mutates the container). +fn resolve_slice_ints(s: &PySlice) -> Result { + let resolve = |o: &Object| -> Result { + Ok(match slice_bound_index(o)? { + None => Object::None, + Some(v) => Object::Int(v), + }) + }; + Ok(PySlice { + start: resolve(&s.start)?, + stop: resolve(&s.stop)?, + step: resolve(&s.step)?, + }) +} + fn apply_slice_assignment( data: &mut Vec, s: &PySlice, replacement: Vec, ) -> Result<(), RuntimeError> { let len = data.len() as i64; - let step = match &s.step { - Object::None => 1i64, - Object::Int(i) => *i, - _ => return Err(type_error("slice indices must be integers or None")), + let step = match slice_bound_index(&s.step)? { + None => 1i64, + Some(v) => v, }; if step == 0 { return Err(value_error("slice step cannot be zero")); } - let extract = |o: &Object, default: i64| -> Result { - match o { - Object::None => Ok(default), - Object::Int(i) => Ok(*i), - _ => Err(type_error("slice indices must be integers or None")), - } - }; - let start_raw = extract(&s.start, if step > 0 { 0 } else { len - 1 })?; - let stop_raw = extract(&s.stop, if step > 0 { len } else { -1 })?; + let start_raw = + slice_bound_index(&s.start)?.unwrap_or(if step > 0 { 0 } else { len - 1 }); + let stop_raw = slice_bound_index(&s.stop)?.unwrap_or(if step > 0 { len } else { -1 }); let norm = |x: i64| -> i64 { if x < 0 { ((x + len).max(0)).min(len) @@ -16463,6 +17135,16 @@ pub(crate) fn percent_format_with( let positional: Vec = match value { Object::Tuple(items) => items.to_vec(), Object::Dict(_) => Vec::new(), + // A *tuple subclass* spreads as the argument pack too (CPython + // PyTuple_Check) — namedtuple's `repr_fmt % self` depends on it. + Object::Instance(_) + if matches!(value.native_value(), Some(Object::Tuple(_))) => + { + match value.native_value() { + Some(Object::Tuple(items)) => items.to_vec(), + _ => unreachable!(), + } + } other => vec![other.clone()], }; while i < bytes.len() { @@ -16576,13 +17258,35 @@ pub(crate) fn percent_format_with( continue; } let item = if let Some(k) = mapping_key { - match value { - Object::Dict(d) => d + // In bytes mode the mapping is keyed by *bytes* (the raw + // template slice, latin-1); in str mode by text. + let key_obj = match mode { + PercentMode::Bytes => { + Object::new_bytes(k.chars().map(|c| c as u8).collect::>()) + } + PercentMode::Str => Object::from_str(&k), + }; + // Unwrap dict subclasses to their payload (CPython only + // needs `mp_subscript` here). + let native; + let mapping = match value { + Object::Dict(d) => Some(d), + Object::Instance(_) => match value.native_value() { + Some(Object::Dict(d)) => { + native = d; + Some(&native) + } + _ => None, + }, + _ => None, + }; + match mapping { + Some(d) => d .borrow() - .get(&DictKey(Object::from_str(&k))) + .get(&DictKey(key_obj.clone())) .cloned() - .ok_or_else(|| key_error(format!("'{k}'")))?, - _ => return Err(type_error("format requires a mapping")), + .ok_or_else(|| key_error(key_obj.repr()))?, + None => return Err(type_error("format requires a mapping")), } } else { let v = positional @@ -16695,9 +17399,11 @@ pub(crate) fn percent_format_with( }, _ if percent_is_real(&item) => item.clone(), _ => { + // gh-130928: `%i` reports as `%d` in the error. + let kind_msg = if kind == 'i' { 'd' } else { kind }; return Err(type_error(format!( - "%{kind} format: a real number is required, not {}", - item.type_name() + "%{kind_msg} format: a real number is required, not {}", + item.type_name_owned() ))) } }; @@ -16709,7 +17415,7 @@ pub(crate) fn percent_format_with( if !percent_is_int(&item) { return Err(type_error(format!( "%{kind} format: an integer is required, not {}", - item.type_name() + item.type_name_owned() ))); } format_via_spec_percent(&item, &spec)? @@ -16719,10 +17425,10 @@ pub(crate) fn percent_format_with( return Err(type_error(match mode { PercentMode::Bytes => format!( "float argument required, not {}", - item.type_name() + item.type_name_owned() ), PercentMode::Str => { - format!("must be real number, not {}", item.type_name()) + format!("must be real number, not {}", item.type_name_owned()) } })); } @@ -17952,7 +18658,12 @@ fn detect_yield_from_subiter(frame: &Frame) -> Option { Object::Generator(_) | Object::Coroutine(_) | Object::AsyncGenerator(_) + | Object::AsyncGenAwait(_) | Object::Iter(_) => Some(top.clone()), + // A custom awaitable (`__await__` returning its own iterator, + // e.g. coroutine_wrapper or types.coroutine's wrapper): SEND + // delegates to it, so throw/close delegate too. + Object::Instance(_) => Some(top.clone()), _ => None, } } @@ -18131,7 +18842,7 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result { - let times = if *n < 0 { 0 } else { *n as usize }; + let times = checked_repeat_count(x.len(), *n, "string")?; let mut out = String::with_capacity(x.len() * times); for _ in 0..times { out.push_str(x); @@ -18185,7 +18896,12 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result { - let times = if *n < 0 { 0 } else { *n as usize }; + // `b * 1` returns the operand itself — bytes are immutable, so + // CPython shares the object (`test_repeat_id_preserving`). + if *n == 1 { + return Ok(Object::Bytes(x.clone())); + } + let times = checked_repeat_count(x.len(), *n, "bytes")?; let mut out = Vec::with_capacity(x.len() * times); for _ in 0..times { out.extend_from_slice(x); @@ -18193,8 +18909,8 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result { - let times = if *n < 0 { 0 } else { *n as usize }; let body = x.borrow().clone(); + let times = checked_repeat_count(body.len(), *n, "bytes")?; let mut out = Vec::with_capacity(body.len() * times); for _ in 0..times { out.extend_from_slice(&body); @@ -18205,6 +18921,15 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result Ok(intersect_sets(&a.borrow(), &b.borrow())), (O::Set(a), O::Set(b), B::Sub) => Ok(difference_sets(&a.borrow(), &b.borrow())), (O::Set(a), O::Set(b), B::BitXor) => Ok(symmetric_diff_sets(&a.borrow(), &b.borrow())), + + // PEP 584 — `dict | dict` merges left-to-right into a new dict. + (O::Dict(x), O::Dict(y), B::BitOr) => { + let mut out = x.borrow().clone(); + for (k, v) in y.borrow().iter() { + out.insert(k.clone(), v.clone()); + } + Ok(Object::Dict(Rc::new(RefCell::new(out)))) + } (O::FrozenSet(a), O::FrozenSet(b), B::BitOr) => Ok(union_sets(a, b)), (O::FrozenSet(a), O::FrozenSet(b), B::BitAnd) => Ok(intersect_sets(a, b)), (O::FrozenSet(a), O::FrozenSet(b), B::Sub) => Ok(difference_sets(a, b)), @@ -18216,8 +18941,8 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result { - let times = if *n < 0 { 0 } else { *n as usize }; let body = x.borrow().clone(); + let times = checked_repeat_count(body.len(), *n, "list")?; let mut out = Vec::with_capacity(body.len() * times); for _ in 0..times { out.extend(body.iter().cloned()); @@ -18229,6 +18954,14 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result { + let times = checked_repeat_count(x.len(), *n, "tuple")?; + let mut out: Vec = Vec::with_capacity(x.len() * times); + for _ in 0..times { + out.extend(x.iter().cloned()); + } + Ok(Object::new_tuple(out)) + } // PEP 604 — type union via `|`. Matches `Type | Type`, // `Type | None`, `Type | UnionType`, and the symmetric forms. @@ -18254,6 +18987,23 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result Result { + let times = if n < 0 { 0 } else { n as usize }; + if item_len == 0 { + return Ok(times.min(1)); + } + match item_len.checked_mul(times) { + Some(total) if total <= isize::MAX as usize => Ok(times), + _ => Err(crate::error::overflow_error(format!( + "repeated {what} is too long" + ))), + } +} + /// Return `true` if `obj` can participate in a PEP 604 `X | Y` union /// construction — a real type, the runtime singleton `None` /// (interpreted as `type(None)`), or an existing PEP 604 union we diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 6bd7694..fd1e3d2 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -793,6 +793,23 @@ pub struct PyGenerator { /// finishes and the frame is dropped. pub code: Object, pub state: RefCell, + /// `cr_origin` — for coroutines created while + /// `sys.set_coroutine_origin_tracking_depth(n)` is active: a tuple + /// of `(filename, lineno, funcname)` triples for the creation call + /// stack (most recent first). `None` when tracking is off. + pub origin: RefCell, + /// PEP 525 `sys.set_asyncgen_hooks` bookkeeping (async generators + /// only). `hooks_inited` flips on the first `__anext__`/`asend`/ + /// `athrow`/`aclose`, at which point the thread's *finalizer* hook + /// is captured here so finalization can route through the event + /// loop that first iterated the generator. + pub hooks_inited: crate::sync::Cell, + pub finalizer: RefCell, + /// CPython's "tp_finalize already ran" GC bit: `invoke_finalizer` + /// sets it before finalizing so a generator left suspended by its + /// finalizer (e.g. a PEP 525 hook that declined to close it) is + /// not resurrected and re-finalized forever on the next drop. + pub finalize_ran: crate::sync::Cell, } impl PyGenerator { @@ -809,6 +826,10 @@ impl PyGenerator { kind, code, state: RefCell::new(GeneratorState::Created(frame)), + origin: RefCell::new(Object::None), + hooks_inited: crate::sync::Cell::new(false), + finalizer: RefCell::new(Object::None), + finalize_ran: crate::sync::Cell::new(false), } } @@ -839,12 +860,22 @@ impl Drop for PyGenerator { let prev = std::mem::replace(&mut *state, GeneratorState::Finished); drop(state); if let GeneratorState::Suspended(frame) = prev { + // Finalized once already (CPython's `_PyGC_FINALIZED` bit): + // drop the frame for real instead of looping forever + // through resurrection → finalize → drop. + if self.finalize_ran.get() { + return; + } let resurrected = Rc::new(PyGenerator { name: RefCell::new(self.name.borrow().clone()), qualname: RefCell::new(self.qualname.borrow().clone()), kind: self.kind, code: self.code.clone(), state: RefCell::new(GeneratorState::Suspended(frame)), + origin: RefCell::new(self.origin.borrow().clone()), + hooks_inited: crate::sync::Cell::new(self.hooks_inited.get()), + finalizer: RefCell::new(self.finalizer.borrow().clone()), + finalize_ran: crate::sync::Cell::new(self.finalize_ran.get()), }); let obj = match self.kind { CoroutineKind::Generator => Object::Generator(resurrected), @@ -1298,6 +1329,13 @@ pub enum PyIterator { data: Rc<[u8]>, index: usize, }, + /// Live view over a bytearray (CPython's `bytearray_iterator` + /// tracks the buffer, so clearing the bytearray exhausts a + /// half-consumed iterator — issue 27443). + ByteArray { + data: Rc>>, + index: usize, + }, /// Lazy `enumerate(...)`. Holds a *shared* handle to the wrapped /// iterator so consuming the enumerate also advances the original /// (CPython: `enumerate(it)` yields from the same `it`, leaving it @@ -1384,6 +1422,11 @@ impl PyIterator { *index += 1; Some(Object::Int(i64::from(v))) } + PyIterator::ByteArray { data, index } => { + let v = data.borrow().get(*index).copied()?; + *index += 1; + Some(Object::Int(i64::from(v))) + } PyIterator::Enumerate { inner, count } => { let v = inner.borrow_mut().next_value()?; let i = *count; @@ -1425,6 +1468,9 @@ impl PyIterator { PyIterator::Str { s, index } => Some(s[(*index).min(s.len())..].chars().count()), PyIterator::DictKeys { keys, index } => Some(keys.len().saturating_sub(*index)), PyIterator::Bytes { data, index } => Some(data.len().saturating_sub(*index)), + PyIterator::ByteArray { data, index } => { + Some(data.borrow().len().saturating_sub(*index)) + } PyIterator::Enumerate { inner, .. } => inner.borrow().remaining(), PyIterator::Reversed { index, .. } => Some((*index + 1).max(0) as usize), PyIterator::Range { @@ -1475,6 +1521,11 @@ impl PyIterator { .get(*index..) .map(|rest| rest.iter().map(|b| Object::Int(i64::from(*b))).collect()) .unwrap_or_default(), + PyIterator::ByteArray { data, index } => data + .borrow() + .get(*index..) + .map(|rest| rest.iter().map(|b| Object::Int(i64::from(*b))).collect()) + .unwrap_or_default(), PyIterator::Range { current, stop, @@ -1546,9 +1597,9 @@ impl PyIterator { /// elements. pub fn reduce_remaining(&self) -> Object { match self { - PyIterator::Tuple { .. } | PyIterator::Bytes { .. } => { - Object::new_tuple(self.remaining_items()) - } + PyIterator::Tuple { .. } + | PyIterator::Bytes { .. } + | PyIterator::ByteArray { .. } => Object::new_tuple(self.remaining_items()), PyIterator::Str { s, index } => { let start = (*index).min(s.len()); Object::from_str(&s[start..]) @@ -1855,6 +1906,16 @@ impl Object { .partial_cmp(&(i64::from(*b) as f64)) .ok_or_else(|| value_error("cannot order with NaN"))?), (O::Str(a), O::Str(b)) => Ok(a.cmp(b)), + // bytes/bytearray order lexicographically by byte value; + // the four mixed combinations all compare (CPython's + // shared `bytes_richcompare` buffer path). + (O::Bytes(a), O::Bytes(b)) => Ok(a.as_ref().cmp(b.as_ref())), + (O::Bytes(a), O::ByteArray(b)) => Ok(a.as_ref()[..].cmp(&b.borrow()[..])), + (O::ByteArray(a), O::Bytes(b)) => Ok(a.borrow()[..].cmp(&b.as_ref()[..])), + (O::ByteArray(a), O::ByteArray(b)) => { + let bv = b.borrow().clone(); + Ok(a.borrow()[..].cmp(&bv[..])) + } (O::Tuple(a), O::Tuple(b)) => seq_cmp(a, b), (O::List(a), O::List(b)) => { let a = a.borrow(); @@ -1888,26 +1949,27 @@ impl Object { Object::Dict(d) => Ok(d.borrow().contains_key(&DictKey(item.clone()))), Object::Set(s) => Ok(s.borrow().contains(&DictKey(item.clone()))), Object::FrozenSet(s) => Ok(s.contains(&DictKey(item.clone()))), - Object::Bytes(haystack) => match item { - Object::Int(i) => Ok(*i >= 0 && *i <= 255 && haystack.contains(&(*i as u8))), - Object::Bytes(needle) => Ok(bytes_contains(haystack, needle)), - Object::ByteArray(needle) => Ok(bytes_contains(haystack, &needle.borrow())), - _ => Err(type_error( - "a bytes-like object is required, not '".to_owned() + item.type_name() + "'", - )), - }, - Object::ByteArray(haystack) => match item { - Object::Int(i) => { - Ok(*i >= 0 && *i <= 255 && haystack.borrow().contains(&(*i as u8))) + Object::Bytes(haystack) => { + bytes_membership(haystack, item) + } + Object::ByteArray(haystack) => { + // Clone out: converting `item` can reenter Python (a user + // `__index__`) that mutates this bytearray. CPython holds + // the buffer during conversion and raises BufferError if + // it is resized (gh-142560). + let before = haystack.borrow().len(); + let hay: Vec = haystack.borrow().clone(); + let result = bytes_membership(&hay, item); + if haystack.borrow().len() != before { + return Err(RuntimeError::PyException( + crate::error::PyException::from_builtin( + "BufferError", + "Existing exports of data: object cannot be re-sized", + ), + )); } - Object::Bytes(needle) => Ok(bytes_contains(&haystack.borrow(), needle)), - Object::ByteArray(needle) => { - Ok(bytes_contains(&haystack.borrow(), &needle.borrow())) - } - _ => Err(type_error( - "a bytes-like object is required, not '".to_owned() + item.type_name() + "'", - )), - }, + result + } Object::Range(r) => { if let Object::Int(i) = item { if r.step > 0 { @@ -1995,13 +2057,10 @@ impl Object { data: b.clone(), index: 0, }), - Object::ByteArray(b) => { - let snapshot: Rc<[u8]> = Rc::from(b.borrow().as_slice()); - Ok(PyIterator::Bytes { - data: snapshot, - index: 0, - }) - } + Object::ByteArray(b) => Ok(PyIterator::ByteArray { + data: b.clone(), + index: 0, + }), Object::MemoryView(mv) => { if mv.released.get() { return Err(value_error("memoryview: released")); @@ -2926,10 +2985,39 @@ fn bytes_contains(haystack: &[u8], needle: &[u8]) -> bool { if needle.is_empty() { return true; } - if needle.len() > haystack.len() { - return false; + memchr::memmem::find(haystack, needle).is_some() +} + +/// `x in bytes` / `x in bytearray`: a byte value (int in +/// `range(0, 256)`, out-of-range is `ValueError`) or a bytes-like +/// needle. Anything else is the CPython `TypeError`. +fn bytes_membership(haystack: &[u8], item: &Object) -> Result { + let native = item.native_value(); + match native.as_ref().unwrap_or(item) { + Object::Bool(v) => Ok(haystack.contains(&u8::from(*v))), + Object::Int(i) => { + if (0..=255).contains(i) { + Ok(haystack.contains(&(*i as u8))) + } else { + Err(value_error("byte must be in range(0, 256)")) + } + } + Object::Long(_) => Err(value_error("byte must be in range(0, 256)")), + Object::Bytes(needle) => Ok(bytes_contains(haystack, needle)), + Object::ByteArray(needle) => Ok(bytes_contains(haystack, &needle.borrow())), + Object::MemoryView(mv) => Ok(bytes_contains(haystack, &mv.to_bytes())), + inst @ Object::Instance(_) if crate::instance_method(inst, "__index__").is_some() => { + let v = crate::builtins::coerce_index_i64(inst)?; + if (0..=255).contains(&v) { + Ok(haystack.contains(&(v as u8))) + } else { + Err(value_error("byte must be in range(0, 256)")) + } + } + _ => Err(type_error( + "a bytes-like object is required, not '".to_owned() + item.type_name() + "'", + )), } - haystack.windows(needle.len()).any(|w| w == needle) } /// CPython's `Py_UNICODE_ISPRINTABLE`: every character is printable @@ -2955,13 +3043,23 @@ pub(crate) fn char_is_printable(c: char) -> bool { } fn bytes_repr(b: &[u8]) -> String { + // CPython prefers single quotes, switching to double quotes when + // the data contains a single quote but no double quote. + let quote = if b.contains(&b'\'') && !b.contains(&b'"') { + b'"' + } else { + b'\'' + }; let mut out = String::with_capacity(b.len() + 3); out.push('b'); - out.push('\''); + out.push(quote as char); for &c in b { match c { b'\\' => out.push_str("\\\\"), - b'\'' => out.push_str("\\'"), + c if c == quote => { + out.push('\\'); + out.push(c as char); + } b'\n' => out.push_str("\\n"), b'\r' => out.push_str("\\r"), b'\t' => out.push_str("\\t"), @@ -2969,7 +3067,7 @@ fn bytes_repr(b: &[u8]) -> String { _ => out.push_str(&format!("\\x{c:02x}")), } } - out.push('\''); + out.push(quote as char); out } diff --git a/crates/weavepy-vm/src/stdlib/mod.rs b/crates/weavepy-vm/src/stdlib/mod.rs index c8350f5..4ec0fca 100644 --- a/crates/weavepy-vm/src/stdlib/mod.rs +++ b/crates/weavepy-vm/src/stdlib/mod.rs @@ -196,14 +196,22 @@ fn frozen_sources() -> &'static [FrozenSource] { source: include_str!("python/_seqtools.py"), is_package: false, }, - // `collections` is a package so `collections.abc` resolves; the - // verbatim CPython `_collections_abc` carries the ABC definitions - // and `collections.abc` re-exports them (RFC 0037 WS8). + // `collections` is the verbatim CPython package init; the + // `_collections` accelerator below supplies `deque`/`defaultdict` + // (which have no pure-Python fallback in the real module), while + // `OrderedDict`/`namedtuple` run the reference pure-Python paths. + // The verbatim CPython `_collections_abc` carries the ABC + // definitions and `collections.abc` re-exports them (RFC 0037 WS8). FrozenSource { name: "collections", source: include_str!("python/collections.py"), is_package: true, }, + FrozenSource { + name: "_collections", + source: include_str!("python/_collections.py"), + is_package: false, + }, FrozenSource { name: "_collections_abc", source: include_str!("python/_collections_abc.py"), diff --git a/crates/weavepy-vm/src/stdlib/os.rs b/crates/weavepy-vm/src/stdlib/os.rs index 1fd5650..0891bcf 100644 --- a/crates/weavepy-vm/src/stdlib/os.rs +++ b/crates/weavepy-vm/src/stdlib/os.rs @@ -363,7 +363,7 @@ pub fn build_path(_cache: &ModuleCache) -> Rc { ); d.insert( DictKey(Object::from_static("realpath")), - builtin("realpath", path_abspath), + builtin("realpath", path_realpath), ); d.insert( DictKey(Object::from_static("relpath")), @@ -1439,6 +1439,40 @@ fn path_abspath(args: &[Object]) -> Result { Ok(Object::from_str(abs.to_string_lossy().into_owned())) } +/// `os.path.realpath` — resolve symlinks via `fs::canonicalize` +/// (CPython's non-strict mode: a nonexistent tail rides lexically on +/// the longest resolvable prefix). +fn path_realpath(args: &[Object]) -> Result { + let s = first_path(args, "realpath")?; + let p = PathBuf::from(&s); + let abs = if p.is_absolute() { + p + } else { + std::env::current_dir() + .map_err(|e| os_error(format!("realpath: {e}")))? + .join(p) + }; + if let Ok(c) = std::fs::canonicalize(&abs) { + return Ok(Object::from_str(c.to_string_lossy().into_owned())); + } + let mut prefix = abs.clone(); + let mut tail: Vec = Vec::new(); + while prefix.file_name().is_some() { + if let Ok(c) = std::fs::canonicalize(&prefix) { + let mut out = c; + for t in tail.iter().rev() { + out.push(t); + } + return Ok(Object::from_str(normpath_lexical( + &out.to_string_lossy(), + ))); + } + tail.push(prefix.file_name().expect("checked above").to_owned()); + prefix.pop(); + } + Ok(Object::from_str(normpath_lexical(&abs.to_string_lossy()))) +} + fn path_normpath(args: &[Object]) -> Result { let s = first_path(args, "normpath")?; let normalised = normpath_lexical(&s); diff --git a/crates/weavepy-vm/src/stdlib/python/_collections.py b/crates/weavepy-vm/src/stdlib/python/_collections.py new file mode 100644 index 0000000..25f5e00 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/python/_collections.py @@ -0,0 +1,260 @@ +"""WeavePy's `_collections` accelerator module. + +CPython implements `deque`, `defaultdict`, `OrderedDict`, `_tuplegetter` +and `_count_elements` in C here; the verbatim `collections/__init__.py` +imports each inside `try/except ImportError` and falls back to its +pure-Python definitions when absent. + +WeavePy supplies the two containers that have *no* pure-Python fallback +in the real module — `deque` and `defaultdict` — plus `_count_elements`. +`OrderedDict` and `_tuplegetter` are intentionally omitted so the +reference pure-Python implementations run instead. +""" + +__all__ = ["deque", "defaultdict", "_count_elements"] + + +def _count_elements(mapping, iterable): + """Tally elements from the iterable (Counter's inner loop).""" + mapping_get = mapping.get + for elem in iterable: + mapping[elem] = mapping_get(elem, 0) + 1 + + +class defaultdict(dict): + """dict subclass that calls a factory function to supply missing values.""" + + def __init__(self, default_factory=None, /, *args, **kwds): + if default_factory is not None and not callable(default_factory): + raise TypeError("first argument must be callable or None") + dict.__init__(self, *args, **kwds) + self.default_factory = default_factory + + def __missing__(self, key): + if self.default_factory is None: + raise KeyError(key) + self[key] = value = self.default_factory() + return value + + def __repr__(self): + return ( + f"{type(self).__name__}({self.default_factory!r}, {dict.__repr__(self)})" + ) + + def copy(self): + return type(self)(self.default_factory, self) + + __copy__ = copy + + def __reduce__(self): + if self.default_factory is None: + args = () + else: + args = (self.default_factory,) + return type(self), args, None, None, iter(self.items()) + + def __or__(self, other): + if not isinstance(other, dict): + return NotImplemented + new = self.copy() + new.update(other) + return new + + def __ror__(self, other): + if not isinstance(other, dict): + return NotImplemented + new = type(self)(self.default_factory, other) + new.update(self) + return new + + +class deque: + """list-like container with fast appends and pops on either end. + + Pure-Python stand-in for CPython's doubly-linked-block C deque; it + keeps the public API (append/appendleft, pop/popleft, maxlen + discipline, rotate, +, *, comparison, …) over a plain list. + """ + + def __init__(self, iterable=(), maxlen=None): + if maxlen is not None: + if not isinstance(maxlen, int): + raise TypeError("an integer is required") + if maxlen < 0: + raise ValueError("maxlen must be non-negative") + self._data = [] + self._maxlen = maxlen + self.extend(iterable) + + @property + def maxlen(self): + return self._maxlen + + def append(self, x): + self._data.append(x) + if self._maxlen is not None and len(self._data) > self._maxlen: + del self._data[0] + + def appendleft(self, x): + self._data.insert(0, x) + if self._maxlen is not None and len(self._data) > self._maxlen: + self._data.pop() + + def pop(self): + if not self._data: + raise IndexError("pop from an empty deque") + return self._data.pop() + + def popleft(self): + if not self._data: + raise IndexError("pop from an empty deque") + return self._data.pop(0) + + def extend(self, iterable): + for item in iterable: + self.append(item) + + def extendleft(self, iterable): + for item in iterable: + self.appendleft(item) + + def rotate(self, n=1): + if not self._data: + return + size = len(self._data) + n = n % size + if n == 0: + return + self._data = self._data[-n:] + self._data[:-n] + + def clear(self): + del self._data[:] + + def copy(self): + return type(self)(self._data, self._maxlen) + + __copy__ = copy + + def count(self, value): + return sum(1 for item in self._data if item == value) + + def index(self, value, start=0, stop=None): + if stop is None: + stop = len(self._data) + n = len(self._data) + if start < 0: + start = max(0, start + n) + if stop < 0: + stop += n + for i in range(start, min(stop, n)): + if self._data[i] == value: + return i + raise ValueError(f"{value!r} is not in deque") + + def insert(self, i, x): + if self._maxlen is not None and len(self._data) >= self._maxlen: + raise IndexError("deque already at its maximum size") + self._data.insert(i, x) + + def remove(self, value): + for i, item in enumerate(self._data): + if item == value: + del self._data[i] + return + raise ValueError("deque.remove(x): x not in deque") + + def reverse(self): + self._data.reverse() + + def __len__(self): + return len(self._data) + + def __bool__(self): + return bool(self._data) + + def __iter__(self): + return iter(self._data) + + def __reversed__(self): + return reversed(self._data) + + def __contains__(self, x): + return x in self._data + + def __getitem__(self, idx): + if isinstance(idx, slice): + raise TypeError("sequence index must be integer, not 'slice'") + return self._data[idx] + + def __setitem__(self, idx, value): + self._data[idx] = value + + def __delitem__(self, idx): + del self._data[idx] + + def __add__(self, other): + if not isinstance(other, deque): + return NotImplemented + new = self.copy() + new.extend(other._data) + return new + + def __iadd__(self, other): + self.extend(other) + return self + + def __mul__(self, n): + if not isinstance(n, int): + return NotImplemented + return type(self)(self._data * n, self._maxlen) + + __rmul__ = __mul__ + + def __imul__(self, n): + self._data *= n + if self._maxlen is not None and len(self._data) > self._maxlen: + del self._data[: len(self._data) - self._maxlen] + return self + + def _cmp_seq(self, other): + return other._data if isinstance(other, deque) else NotImplemented + + def __eq__(self, other): + if not isinstance(other, deque): + return NotImplemented + return self._data == other._data + + def __ne__(self, other): + if not isinstance(other, deque): + return NotImplemented + return self._data != other._data + + def __lt__(self, other): + if not isinstance(other, deque): + return NotImplemented + return self._data < other._data + + def __le__(self, other): + if not isinstance(other, deque): + return NotImplemented + return self._data <= other._data + + def __gt__(self, other): + if not isinstance(other, deque): + return NotImplemented + return self._data > other._data + + def __ge__(self, other): + if not isinstance(other, deque): + return NotImplemented + return self._data >= other._data + + __hash__ = None + + def __reduce__(self): + return type(self), (list(self._data), self._maxlen) + + def __repr__(self): + if self._maxlen is None: + return f"{type(self).__name__}({self._data!r})" + return f"{type(self).__name__}({self._data!r}, maxlen={self._maxlen})" diff --git a/crates/weavepy-vm/src/stdlib/python/asyncio.py b/crates/weavepy-vm/src/stdlib/python/asyncio.py index 4272261..b1d3497 100644 --- a/crates/weavepy-vm/src/stdlib/python/asyncio.py +++ b/crates/weavepy-vm/src/stdlib/python/asyncio.py @@ -36,9 +36,11 @@ spins up a fresh one. """ +import sys as _sys import time as _time import selectors as _selectors import socket as _socket +import weakref as _weakref # ---- Exceptions --------------------------------------------------- @@ -171,11 +173,31 @@ def get_name(self): def set_name(self, value): self._name = str(value) + def get_coro(self): + return self._coro + + def get_stack(self, *, limit=None): + """Frames where the task's coroutine is currently suspended + (CPython walks `cr_await` chains; a single frame is the common + observable shape). Done tasks have no stack.""" + if self.done(): + return [] + frame = getattr(self._coro, "cr_frame", None) + if frame is None: + frame = getattr(self._coro, "gi_frame", None) + return [frame] if frame is not None else [] + def cancel(self, msg=None): if self.done(): return False self._must_cancel = True self._cancel_message = msg + # Wake a task parked on an inner future so the cancellation is + # delivered promptly (CPython cancels `_fut_waiter`, which + # reschedules the step with CancelledError). + waiting = self._waiting_on + if waiting is not None and not waiting.done(): + waiting.cancel(msg) return True def _step(self, value, exc=None): @@ -270,6 +292,11 @@ def __init__(self): self._selector = _selectors.DefaultSelector() # fd -> (reader_cb, writer_cb) self._fd_callbacks = {} + # PEP 525: async generators first-iterated while this loop runs. + # Weak — the loop must not keep otherwise-dead agens alive (their + # finalization is exactly what routes through the hooks). + self._asyncgens = _weakref.WeakSet() + self._asyncgens_shutdown_called = False # ---- inspection ----------------------------------------------- @@ -328,9 +355,18 @@ def create_future(self): # ---- run loop ------------------------------------------------- def run_forever(self): + global _running_loop if self._running: raise RuntimeError("event loop is already running") + if _running_loop is not None: + raise RuntimeError( + "Cannot run the event loop while another loop is running") + old_agen_hooks = _sys.get_asyncgen_hooks() + _sys.set_asyncgen_hooks( + firstiter=self._asyncgen_firstiter_hook, + finalizer=self._asyncgen_finalizer_hook) self._running = True + _running_loop = self try: while self._running: if (not self._ready and not self._scheduled @@ -339,6 +375,45 @@ def run_forever(self): self._run_once() finally: self._running = False + _running_loop = None + _sys.set_asyncgen_hooks(*old_agen_hooks) + + # ---- PEP 525 async generator finalization --------------------- + + def _asyncgen_firstiter_hook(self, agen): + if self._asyncgens_shutdown_called: + import warnings + warnings.warn( + "asynchronous generator {!r} was scheduled after " + "loop.shutdown_asyncgens() call".format(agen), + ResourceWarning) + self._asyncgens.add(agen) + + def _asyncgen_finalizer_hook(self, agen): + self._asyncgens.discard(agen) + if not self.is_closed(): + self.call_soon(self.create_task, agen.aclose()) + + async def shutdown_asyncgens(self): + """Shutdown all active asynchronous generators.""" + self._asyncgens_shutdown_called = True + closing_agens = list(self._asyncgens) + self._asyncgens.clear() + if not closing_agens: + return + results = await gather( + *[ag.aclose() for ag in closing_agens], + return_exceptions=True) + for result, agen in zip(results, closing_agens): + if isinstance(result, CancelledError): + continue + if isinstance(result, BaseException): + self.call_exception_handler({ + "message": "an error occurred during closing of " + "asynchronous generator {!r}".format(agen), + "exception": result, + "asyncgen": agen, + }) def run_until_complete(self, future): if not isinstance(future, Future): @@ -549,18 +624,37 @@ def _try_accept(): self.remove_reader(sock.fileno()) def _handle_exception(self, exc): - if self._exception_handler is not None: - self._exception_handler(self, {"exception": exc}) - else: - try: - import sys - sys.stderr.write("asyncio task exception: {}\n".format(exc)) - except Exception: - pass + self.call_exception_handler({ + "message": "Exception in callback", + "exception": exc, + }) def set_exception_handler(self, handler): self._exception_handler = handler + def get_exception_handler(self): + return self._exception_handler + + def default_exception_handler(self, context): + message = context.get("message") or \ + "Unhandled exception in event loop" + exc = context.get("exception") + try: + _sys.stderr.write("{}: {!r}\n".format(message, exc)) + except Exception: + pass + + def call_exception_handler(self, context): + if self._exception_handler is None: + self.default_exception_handler(context) + else: + try: + self._exception_handler(self, context) + except BaseException: + # A broken handler must not take the loop down + # (CPython falls back to the default handler). + self.default_exception_handler(context) + class _Handle: """A cancellation handle returned by `call_soon` / `call_later`.""" @@ -583,9 +677,18 @@ def cancelled(self): _current_loop = None +# The loop currently inside `run_forever` (CPython tracks this in a +# thread-local set by `events._set_running_loop`). `sleep`, `gather`, +# `create_task` etc. must schedule on *this* loop — not the module-level +# "current" loop, which can be a different object when user code drives +# `new_event_loop().run_until_complete(...)` directly. +_running_loop = None + def get_event_loop(): global _current_loop + if _running_loop is not None: + return _running_loop if _current_loop is None or _current_loop.is_closed(): _current_loop = EventLoop() return _current_loop @@ -601,9 +704,9 @@ def set_event_loop(loop): def get_running_loop(): - if _current_loop is None or not _current_loop.is_running(): + if _running_loop is None: raise RuntimeError("no running event loop") - return _current_loop + return _running_loop # ---- event loop policy -------------------------------------------- @@ -666,8 +769,31 @@ def run(coro, *, debug=None): try: return loop.run_until_complete(coro) finally: - loop.close() - set_event_loop(None) + try: + _cancel_all_tasks(loop) + loop.run_until_complete(loop.shutdown_asyncgens()) + finally: + loop.close() + set_event_loop(None) + + +def _cancel_all_tasks(loop): + to_cancel = [t for t in loop._tasks if not t.done()] + if not to_cancel: + return + for task in to_cancel: + task.cancel() + loop.run_until_complete(gather(*to_cancel, return_exceptions=True)) + for task in to_cancel: + if task.cancelled(): + continue + if task.exception() is not None: + loop.call_exception_handler({ + "message": "unhandled exception during asyncio.run() " + "shutdown", + "exception": task.exception(), + "task": task, + }) def ensure_future(obj, *, loop=None): @@ -700,8 +826,21 @@ def all_tasks(loop=None): # ---- sleep -------------------------------------------------------- +class _Sleep0: + """Awaitable that yields to the event loop exactly once (CPython's + `__sleep0`). `sleep(0)` must reschedule the task — code like + `finally: await sleep(0)` relies on other ready tasks (e.g. a + pending aclose) running before it resumes.""" + + def __await__(self): + yield + + __iter__ = __await__ + + async def sleep(delay, result=None): if delay <= 0: + await _Sleep0() return result loop = get_event_loop() fut = _SleepFuture(loop=loop) diff --git a/crates/weavepy-vm/src/stdlib/python/collections.py b/crates/weavepy-vm/src/stdlib/python/collections.py index bcf5b82..e5a280e 100644 --- a/crates/weavepy-vm/src/stdlib/python/collections.py +++ b/crates/weavepy-vm/src/stdlib/python/collections.py @@ -1,646 +1,1600 @@ -"""WeavePy's pure-Python ``collections`` module. - -The shape mirrors CPython's public API closely enough that everyday -code (``defaultdict(list)``, ``Counter('hello').most_common()``, -``deque(maxlen=3)``) works without modification. - -The implementations intentionally favour clarity over micro-optimised -behaviour. They are designed to run on top of WeavePy's own bytecode -interpreter, not to be the fastest possible Python. -""" +'''This module implements specialized container datatypes providing +alternatives to Python's general purpose built-in containers, dict, +list, set, and tuple. + +* namedtuple factory function for creating tuple subclasses with named fields +* deque list-like container with fast appends and pops on either end +* ChainMap dict-like class for creating a single view of multiple mappings +* Counter dict subclass for counting hashable objects +* OrderedDict dict subclass that remembers the order entries were added +* defaultdict dict subclass that calls a factory function to supply missing values +* UserDict wrapper around dictionary objects for easier dict subclassing +* UserList wrapper around list objects for easier list subclassing +* UserString wrapper around string objects for easier string subclassing + +''' __all__ = [ - "deque", - "OrderedDict", - "defaultdict", - "Counter", - "ChainMap", - "namedtuple", - "UserDict", - "UserList", - "UserString", + 'ChainMap', + 'Counter', + 'OrderedDict', + 'UserDict', + 'UserList', + 'UserString', + 'defaultdict', + 'deque', + 'namedtuple', ] -# `UserDict`/`UserList`/`UserString` are verbatim CPython and depend on -# `collections.abc`, so they live in a sibling frozen module (imported at -# the end of this file, after the package is otherwise initialised) to keep -# the import graph acyclic. +import _collections_abc +import sys as _sys +_sys.modules['collections.abc'] = _collections_abc +abc = _collections_abc -def _count_elements(mapping, iterable): - """Tally elements from the iterable. +from itertools import chain as _chain +from itertools import repeat as _repeat +from itertools import starmap as _starmap +from keyword import iskeyword as _iskeyword +from operator import eq as _eq +from operator import itemgetter as _itemgetter +from reprlib import recursive_repr as _recursive_repr +from _weakref import proxy as _proxy - The pure-Python fallback CPython ships when the ``_collections`` - C accelerator is unavailable; ``test_collections`` imports it by - name to exercise ``Counter`` behaviour. - """ - mapping_get = mapping.get - for elem in iterable: - mapping[elem] = mapping_get(elem, 0) + 1 +try: + from _collections import deque +except ImportError: + pass +else: + _collections_abc.MutableSequence.register(deque) +try: + from _collections import _deque_iterator +except ImportError: + pass -class deque: - """Double-ended queue with optional maximum length. +try: + from _collections import defaultdict +except ImportError: + pass - Supports the operations exercised by typical Python code: append, - appendleft, pop, popleft, extend, extendleft, rotate, clear, copy, - indexing, iteration, len, contains, and equality. - """ - def __init__(self, iterable=None, maxlen=None): - if maxlen is not None and maxlen < 0: - raise ValueError("maxlen must be non-negative") - self._data = [] - self._maxlen = maxlen - if iterable is not None: - for item in iterable: - self.append(item) +################################################################################ +### OrderedDict +################################################################################ - @property - def maxlen(self): - return self._maxlen - - def append(self, x): - self._data.append(x) - if self._maxlen is not None and len(self._data) > self._maxlen: - del self._data[0] - - def appendleft(self, x): - self._data.insert(0, x) - if self._maxlen is not None and len(self._data) > self._maxlen: - self._data.pop() - - def pop(self): - if not self._data: - raise IndexError("pop from an empty deque") - return self._data.pop() - - def popleft(self): - if not self._data: - raise IndexError("pop from an empty deque") - return self._data.pop(0) - - def extend(self, iterable): - for item in iterable: - self.append(item) - - def extendleft(self, iterable): - for item in iterable: - self.appendleft(item) - - def rotate(self, n=1): - if not self._data: - return - size = len(self._data) - n = n % size - if n == 0: - return - self._data = self._data[-n:] + self._data[:-n] +class _OrderedDictKeysView(_collections_abc.KeysView): - def clear(self): - self._data = [] + def __reversed__(self): + yield from reversed(self._mapping) - def copy(self): - return deque(self._data, self._maxlen) - - def count(self, value): - return sum(1 for item in self._data if item == value) - - def index(self, value, start=0, stop=None): - if stop is None: - stop = len(self._data) - for i in range(start, stop): - if self._data[i] == value: - return i - raise ValueError(repr(value) + " is not in deque") - - def insert(self, i, x): - if self._maxlen is not None and len(self._data) >= self._maxlen: - raise IndexError("deque already at its maximum size") - self._data.insert(i, x) - - def remove(self, value): - for i, item in enumerate(self._data): - if item == value: - del self._data[i] - return - raise ValueError(repr(value) + " not in deque") +class _OrderedDictItemsView(_collections_abc.ItemsView): - def reverse(self): - self._data.reverse() + def __reversed__(self): + for key in reversed(self._mapping): + yield (key, self._mapping[key]) - def __len__(self): - return len(self._data) +class _OrderedDictValuesView(_collections_abc.ValuesView): + + def __reversed__(self): + for key in reversed(self._mapping): + yield self._mapping[key] + +class _Link(object): + __slots__ = 'prev', 'next', 'key', '__weakref__' + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as regular dictionaries. + + # The internal self.__map dict maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # The sentinel is in self.__hardroot with a weakref proxy in self.__root. + # The prev links are weakref proxies (to prevent circular references). + # Individual links are kept alive by the hard reference in self.__map. + # Those hard references disappear when a key is deleted from an OrderedDict. + + def __new__(cls, /, *args, **kwds): + "Create the ordered dict object and set up the underlying structures." + self = dict.__new__(cls) + self.__hardroot = _Link() + self.__root = root = _proxy(self.__hardroot) + root.prev = root.next = root + self.__map = {} + return self + + def __init__(self, other=(), /, **kwds): + '''Initialize an ordered dictionary. The signature is the same as + regular dictionaries. Keyword argument order is preserved. + ''' + self.__update(other, **kwds) + + def __setitem__(self, key, value, + dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link at the end of the linked list, + # and the inherited dictionary is updated with the new key/value pair. + if key not in self: + self.__map[key] = link = Link() + root = self.__root + last = root.prev + link.prev, link.next, link.key = last, root, key + last.next = link + root.prev = proxy(link) + dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which gets + # removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link = self.__map.pop(key) + link_prev = link.prev + link_next = link.next + link_prev.next = link_next + link_next.prev = link_prev + link.prev = None + link.next = None def __iter__(self): - return iter(self._data) + 'od.__iter__() <==> iter(od)' + # Traverse the linked list in order. + root = self.__root + curr = root.next + while curr is not root: + yield curr.key + curr = curr.next def __reversed__(self): - return reversed(self._data) + 'od.__reversed__() <==> reversed(od)' + # Traverse the linked list in reverse order. + root = self.__root + curr = root.prev + while curr is not root: + yield curr.key + curr = curr.prev - def __contains__(self, x): - return x in self._data + def clear(self): + 'od.clear() -> None. Remove all items from od.' + root = self.__root + root.prev = root.next = root + self.__map.clear() + dict.clear(self) - def __getitem__(self, idx): - return self._data[idx] + def popitem(self, last=True): + '''Remove and return a (key, value) pair from the dictionary. - def __setitem__(self, idx, value): - self._data[idx] = value + Pairs are returned in LIFO order if last is true or FIFO order if false. + ''' + if not self: + raise KeyError('dictionary is empty') + root = self.__root + if last: + link = root.prev + link_prev = link.prev + link_prev.next = root + root.prev = link_prev + else: + link = root.next + link_next = link.next + root.next = link_next + link_next.prev = root + key = link.key + del self.__map[key] + value = dict.pop(self, key) + return key, value - def __eq__(self, other): - if isinstance(other, deque): - return self._data == other._data - return NotImplemented + def move_to_end(self, key, last=True): + '''Move an existing element to the end (or beginning if last is false). + + Raise KeyError if the element does not exist. + ''' + link = self.__map[key] + link_prev = link.prev + link_next = link.next + soft_link = link_next.prev + link_prev.next = link_next + link_next.prev = link_prev + root = self.__root + if last: + last = root.prev + link.prev = last + link.next = root + root.prev = soft_link + last.next = link + else: + first = root.next + link.prev = root + link.next = first + first.prev = soft_link + root.next = link + + def __sizeof__(self): + sizeof = _sys.getsizeof + n = len(self) + 1 # number of links including root + size = sizeof(self.__dict__) # instance dictionary + size += sizeof(self.__map) * 2 # internal dict and inherited dict + size += sizeof(self.__hardroot) * n # link objects + size += sizeof(self.__root) * n # proxy objects + return size + + update = __update = _collections_abc.MutableMapping.update + + def keys(self): + "D.keys() -> a set-like object providing a view on D's keys" + return _OrderedDictKeysView(self) + + def items(self): + "D.items() -> a set-like object providing a view on D's items" + return _OrderedDictItemsView(self) + + def values(self): + "D.values() -> an object providing a view on D's values" + return _OrderedDictValuesView(self) + + __ne__ = _collections_abc.MutableMapping.__ne__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise KeyError + is raised. + + ''' + marker = self.__marker + result = dict.pop(self, key, marker) + if result is not marker: + # The same as in __delitem__(). + link = self.__map.pop(key) + link_prev = link.prev + link_next = link.next + link_prev.next = link_next + link_next.prev = link_prev + link.prev = None + link.next = None + return result + if default is marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + '''Insert key with a value of default if key is not in the dictionary. + Return the value for key if key is in the dictionary, else default. + ''' + if key in self: + return self[key] + self[key] = default + return default + + @_recursive_repr() def __repr__(self): - if self._maxlen is None: - return "deque(" + repr(self._data) + ")" - return "deque(" + repr(self._data) + ", maxlen=" + repr(self._maxlen) + ")" + 'od.__repr__() <==> repr(od)' + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, dict(self.items())) + + def __reduce__(self): + 'Return state information for pickling' + state = self.__getstate__() + if state: + if isinstance(state, tuple): + state, slots = state + else: + slots = {} + state = state.copy() + slots = slots.copy() + for k in vars(OrderedDict()): + state.pop(k, None) + slots.pop(k, None) + if slots: + state = state, slots + else: + state = state or None + return self.__class__, (), state, None, iter(self.items()) + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) -class _MappingMixin: - """Internal helper: provides the common mapping wire-up our pure- - Python container types share. Composes a plain ``dict`` instead of - inheriting, sidestepping WeavePy's current lack of MRO dispatch - onto built-in types.""" + @classmethod + def fromkeys(cls, iterable, value=None): + '''Create a new ordered dictionary with keys from iterable and values set to value. + ''' + self = cls() + for key in iterable: + self[key] = value + return self - def __init__(self): - self._data = {} + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. - def __getitem__(self, key): - try: - return self._data[key] - except KeyError: - miss = getattr(self, "__missing__", None) - if miss is not None: - return miss(key) - raise + ''' + if isinstance(other, OrderedDict): + return dict.__eq__(self, other) and all(map(_eq, self, other)) + return dict.__eq__(self, other) - def __setitem__(self, key, value): - self._data[key] = value + def __ior__(self, other): + self.update(other) + return self - def __delitem__(self, key): - del self._data[key] + def __or__(self, other): + if not isinstance(other, dict): + return NotImplemented + new = self.__class__(self) + new.update(other) + return new - def __contains__(self, key): - return key in self._data + def __ror__(self, other): + if not isinstance(other, dict): + return NotImplemented + new = self.__class__(other) + new.update(self) + return new - def __len__(self): - return len(self._data) - def __iter__(self): - return iter(self._data) +try: + from _collections import OrderedDict +except ImportError: + # Leave the pure Python version in place. + pass - def keys(self): - return list(self._data.keys()) - def values(self): - return list(self._data.values()) +################################################################################ +### namedtuple +################################################################################ - def items(self): - return list(self._data.items()) +try: + from _collections import _tuplegetter +except ImportError: + _tuplegetter = lambda index, doc: property(_itemgetter(index), doc=doc) - def get(self, key, default=None): - return self._data.get(key, default) +def namedtuple(typename, field_names, *, rename=False, defaults=None, module=None): + """Returns a new subclass of tuple with named fields. + + >>> Point = namedtuple('Point', ['x', 'y']) + >>> Point.__doc__ # docstring for the new class + 'Point(x, y)' + >>> p = Point(11, y=22) # instantiate with positional args or keywords + >>> p[0] + p[1] # indexable like a plain tuple + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessible by name + 33 + >>> d = p._asdict() # convert to a dictionary + >>> d['x'] + 11 + >>> Point(**d) # convert from a dictionary + Point(x=11, y=22) + >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields + Point(x=100, y=22) - def pop(self, key, *args): - if args: - return self._data.pop(key, args[0]) - return self._data.pop(key) - - def update(self, other=None, **kwargs): - if other is not None: - if hasattr(other, "items"): - for k, v in other.items(): - self._data[k] = v - else: - for k, v in other: - self._data[k] = v - for k, v in kwargs.items(): - self._data[k] = v + """ - def clear(self): - self._data.clear() + # Validate the field names. At the user's option, either generate an error + # message or automatically replace the field name with a valid name. + if isinstance(field_names, str): + field_names = field_names.replace(',', ' ').split() + field_names = list(map(str, field_names)) + typename = _sys.intern(str(typename)) + if rename: + seen = set() + for index, name in enumerate(field_names): + if (not name.isidentifier() + or _iskeyword(name) + or name.startswith('_') + or name in seen): + field_names[index] = f'_{index}' + seen.add(name) + + for name in [typename] + field_names: + if type(name) is not str: + raise TypeError('Type names and field names must be strings') + if not name.isidentifier(): + raise ValueError('Type names and field names must be valid ' + f'identifiers: {name!r}') + if _iskeyword(name): + raise ValueError('Type names and field names cannot be a ' + f'keyword: {name!r}') + + seen = set() + for name in field_names: + if name.startswith('_') and not rename: + raise ValueError('Field names cannot start with an underscore: ' + f'{name!r}') + if name in seen: + raise ValueError(f'Encountered duplicate field name: {name!r}') + seen.add(name) + + field_defaults = {} + if defaults is not None: + defaults = tuple(defaults) + if len(defaults) > len(field_names): + raise TypeError('Got more default values than field names') + field_defaults = dict(reversed(list(zip(reversed(field_names), + reversed(defaults))))) + + # Variables used in the methods and docstrings + field_names = tuple(map(_sys.intern, field_names)) + num_fields = len(field_names) + arg_list = ', '.join(field_names) + if num_fields == 1: + arg_list += ',' + repr_fmt = '(' + ', '.join(f'{name}=%r' for name in field_names) + ')' + tuple_new = tuple.__new__ + _dict, _tuple, _len, _map, _zip = dict, tuple, len, map, zip + + # Create all the named tuple methods to be added to the class namespace + + namespace = { + '_tuple_new': tuple_new, + '__builtins__': {}, + '__name__': f'namedtuple_{typename}', + } + code = f'lambda _cls, {arg_list}: _tuple_new(_cls, ({arg_list}))' + __new__ = eval(code, namespace) + __new__.__name__ = '__new__' + __new__.__doc__ = f'Create new instance of {typename}({arg_list})' + if defaults is not None: + __new__.__defaults__ = defaults -class OrderedDict(_MappingMixin): - """Dict that remembers insertion order. + @classmethod + def _make(cls, iterable): + result = tuple_new(cls, iterable) + if _len(result) != num_fields: + raise TypeError(f'Expected {num_fields} arguments, got {len(result)}') + return result - WeavePy's built-in ``dict`` already preserves insertion order, so - this is mostly the additional ``move_to_end`` / - ``popitem(last=...)`` semantics.""" + _make.__func__.__doc__ = (f'Make a new {typename} object from a sequence ' + 'or iterable') - def __init__(self, *args, **kwargs): - _MappingMixin.__init__(self) - if len(args) > 1: - raise TypeError( - f"expected at most 1 positional argument, got {len(args)}" - ) - if args: - self.update(args[0]) - if kwargs: - self.update(kwargs) + def _replace(self, /, **kwds): + result = self._make(_map(kwds.pop, field_names, self)) + if kwds: + raise TypeError(f'Got unexpected field names: {list(kwds)!r}') + return result - def setdefault(self, key, default=None): - if key in self._data: - return self._data[key] - self._data[key] = default - return default + _replace.__doc__ = (f'Return a new {typename} object replacing specified ' + 'fields with new values') - def __eq__(self, other): - if isinstance(other, OrderedDict): - return (list(self.items()) == list(other.items())) - return dict(self) == other + def __repr__(self): + 'Return a nicely formatted representation string' + return self.__class__.__name__ + repr_fmt % self + + def _asdict(self): + 'Return a new dict which maps field names to their values.' + return _dict(_zip(self._fields, self)) + + def __getnewargs__(self): + 'Return self as a plain tuple. Used by copy and pickle.' + return _tuple(self) + + # Modify function metadata to help with introspection and debugging + for method in ( + __new__, + _make.__func__, + _replace, + __repr__, + _asdict, + __getnewargs__, + ): + method.__qualname__ = f'{typename}.{method.__name__}' + + # Build-up the class namespace dictionary + # and use type() to build the result class + class_namespace = { + '__doc__': f'{typename}({arg_list})', + '__slots__': (), + '_fields': field_names, + '_field_defaults': field_defaults, + '__new__': __new__, + '_make': _make, + '__replace__': _replace, + '_replace': _replace, + '__repr__': __repr__, + '_asdict': _asdict, + '__getnewargs__': __getnewargs__, + '__match_args__': field_names, + } + for index, name in enumerate(field_names): + doc = _sys.intern(f'Alias for field number {index}') + class_namespace[name] = _tuplegetter(index, doc) + + result = type(typename, (tuple,), class_namespace) + + # For pickling to work, the __module__ variable needs to be set to the frame + # where the named tuple is created. Bypass this step in environments where + # sys._getframe is not defined (Jython for example) or sys._getframe is not + # defined for arguments greater than 0 (IronPython), or where the user has + # specified a particular module. + if module is None: + try: + module = _sys._getframemodulename(1) or '__main__' + except AttributeError: + try: + module = _sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + if module is not None: + result.__module__ = module - def __ne__(self, other): - return not self == other + return result - def copy(self): - return OrderedDict(self) - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for k in iterable: - d[k] = value - return d +######################################################################## +### Counter +######################################################################## - def move_to_end(self, key, last=True): - if key not in self._data: - raise KeyError(key) - value = self._data.pop(key) - if last: - self._data[key] = value - else: - items = [(key, value)] - for k in list(self._data.keys()): - items.append((k, self._data.pop(k))) - for k, v in items: - self._data[k] = v +def _count_elements(mapping, iterable): + 'Tally elements from the iterable.' + mapping_get = mapping.get + for elem in iterable: + mapping[elem] = mapping_get(elem, 0) + 1 - def popitem(self, last=True): - if not self._data: - raise KeyError("dictionary is empty") - keys = list(self._data.keys()) - key = keys[-1] if last else keys[0] - value = self._data.pop(key) - return (key, value) +try: # Load C helper function if available + from _collections import _count_elements +except ImportError: + pass + +class Counter(dict): + '''Dict subclass for counting hashable items. Sometimes called a bag + or multiset. Elements are stored as dictionary keys and their counts + are stored as dictionary values. + + >>> c = Counter('abcdeabcdabcaba') # count elements from a string + + >>> c.most_common(3) # three most common elements + [('a', 5), ('b', 4), ('c', 3)] + >>> sorted(c) # list all unique elements + ['a', 'b', 'c', 'd', 'e'] + >>> ''.join(sorted(c.elements())) # list elements with repetitions + 'aaaaabbbbcccdde' + >>> sum(c.values()) # total of all counts + 15 + + >>> c['a'] # count of letter 'a' + 5 + >>> for elem in 'shazam': # update counts from an iterable + ... c[elem] += 1 # by adding 1 to each element's count + >>> c['a'] # now there are seven 'a' + 7 + >>> del c['b'] # remove all 'b' + >>> c['b'] # now there are zero 'b' + 0 + + >>> d = Counter('simsalabim') # make another counter + >>> c.update(d) # add in the second counter + >>> c['a'] # now there are nine 'a' + 9 + + >>> c.clear() # empty the counter + >>> c + Counter() + + Note: If a count is set to zero or reduced to zero, it will remain + in the counter until the entry is deleted or the counter is cleared: + + >>> c = Counter('aaabbc') + >>> c['b'] -= 2 # reduce the count of 'b' by two + >>> c.most_common() # 'b' is still in, but its count is zero + [('a', 3), ('c', 1), ('b', 0)] + + ''' + # References: + # http://en.wikipedia.org/wiki/Multiset + # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html + # http://www.java2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm + # http://code.activestate.com/recipes/259174/ + # Knuth, TAOCP Vol. II section 4.6.3 + + def __init__(self, iterable=None, /, **kwds): + '''Create a new, empty Counter object. And if given, count elements + from an input iterable. Or, initialize the count from another mapping + of elements to their counts. + + >>> c = Counter() # a new, empty counter + >>> c = Counter('gallahad') # a new counter from an iterable + >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping + >>> c = Counter(a=4, b=2) # a new counter from keyword args + + ''' + super().__init__() + self.update(iterable, **kwds) - def __repr__(self): - items = ", ".join(repr(k) + ": " + repr(v) for k, v in self.items()) - return "OrderedDict({" + items + "})" - - -class defaultdict(_MappingMixin): - """Dict that creates missing values via a ``default_factory``.""" - - def __init__(self, default_factory=None, *args, **kwargs): - if default_factory is not None and not callable(default_factory): - raise TypeError("first argument must be callable or None") - _MappingMixin.__init__(self) - self.default_factory = default_factory - if args: - src = args[0] - if hasattr(src, "keys"): - for k in src.keys(): - self[k] = src[k] - else: - for k, v in src: - self[k] = v - for k, v in kwargs.items(): - self[k] = v + def __missing__(self, key): + 'The count of elements not in the Counter is zero.' + # Needed so that self[missing_item] does not raise KeyError + return 0 - def __getitem__(self, key): - if key in self._data: - return self._data[key] - if self.default_factory is None: - raise KeyError(key) - value = self.default_factory() - self._data[key] = value - return value + def total(self): + 'Sum of the counts' + return sum(self.values()) - def __repr__(self): - return ( - "defaultdict(" - + repr(self.default_factory) - + ", " - + repr(self._data) - + ")" - ) + def most_common(self, n=None): + '''List the n most common elements and their counts from the most + common to the least. If n is None, then list all element counts. - def copy(self): - new = defaultdict(self.default_factory) - for k, v in self.items(): - new[k] = v - return new + >>> Counter('abracadabra').most_common(3) + [('a', 5), ('b', 2), ('r', 2)] + + ''' + # Emulate Bag.sortedByCount from Smalltalk + if n is None: + return sorted(self.items(), key=_itemgetter(1), reverse=True) + # Lazy import to speedup Python startup time + import heapq + return heapq.nlargest(n, self.items(), key=_itemgetter(1)) -class Counter(_MappingMixin): - """Pure-Python ``Counter`` backed by an internal dict.""" + def elements(self): + '''Iterator over elements repeating each as many times as its count. - def __init__(self, iterable=None, **kwargs): - _MappingMixin.__init__(self) - if iterable is not None: - self.update(iterable) - if kwargs: - self.update(kwargs) + >>> c = Counter('ABCABC') + >>> sorted(c.elements()) + ['A', 'A', 'B', 'B', 'C', 'C'] + + Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + + >>> import math + >>> prime_factors = Counter({2: 2, 3: 3, 17: 1}) + >>> math.prod(prime_factors.elements()) + 1836 + + Note, if an element's count has been set to zero or is a negative + number, elements() will ignore it. + + ''' + # Emulate Bag.do from Smalltalk and Multiset.begin from C++. + return _chain.from_iterable(_starmap(_repeat, self.items())) + + # Override dict methods where necessary + + @classmethod + def fromkeys(cls, iterable, v=None): + # There is no equivalent method for counters because the semantics + # would be ambiguous in cases such as Counter.fromkeys('aaabbc', v=2). + # Initializing counters to zero values isn't necessary because zero + # is already the default value for counter lookups. Initializing + # to one is easily accomplished with Counter(set(iterable)). For + # more exotic cases, create a dictionary first using a dictionary + # comprehension or dict.fromkeys(). + raise NotImplementedError( + 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') + + def update(self, iterable=None, /, **kwds): + '''Like dict.update() but add counts instead of replacing them. + + Source can be an iterable, a dictionary, or another Counter instance. + + >>> c = Counter('which') + >>> c.update('witch') # add elements from another iterable + >>> d = Counter('watch') + >>> c.update(d) # add elements from another counter + >>> c['h'] # four 'h' in which, witch, and watch + 4 + + ''' + # The regular dict.update() operation makes no sense here because the + # replace behavior results in some of the original untouched counts + # being mixed-in with all of the other counts for a mismash that + # doesn't have a straight-forward interpretation in most counting + # contexts. Instead, we implement straight-addition. Both the inputs + # and outputs are allowed to contain zero and negative counts. - def update(self, iterable=None, **kwargs): if iterable is not None: - if hasattr(iterable, "items"): - for key, value in iterable.items(): - self._data[key] = self._data.get(key, 0) + value + if isinstance(iterable, _collections_abc.Mapping): + if self: + self_get = self.get + for elem, count in iterable.items(): + self[elem] = count + self_get(elem, 0) + else: + # fast path when counter is empty + super().update(iterable) else: - for item in iterable: - self._data[item] = self._data.get(item, 0) + 1 - if kwargs: - for key, value in kwargs.items(): - self._data[key] = self._data.get(key, 0) + value - - def subtract(self, iterable=None, **kwargs): + _count_elements(self, iterable) + if kwds: + self.update(kwds) + + def subtract(self, iterable=None, /, **kwds): + '''Like dict.update() but subtracts counts instead of replacing them. + Counts can be reduced below zero. Both the inputs and outputs are + allowed to contain zero and negative counts. + + Source can be an iterable, a dictionary, or another Counter instance. + + >>> c = Counter('which') + >>> c.subtract('witch') # subtract elements from another iterable + >>> c.subtract(Counter('watch')) # subtract elements from another counter + >>> c['h'] # 2 in which, minus 1 in witch, minus 1 in watch + 0 + >>> c['w'] # 1 in which, minus 1 in witch, minus 1 in watch + -1 + + ''' if iterable is not None: - if hasattr(iterable, "items"): - for key, value in iterable.items(): - self._data[key] = self._data.get(key, 0) - value + self_get = self.get + if isinstance(iterable, _collections_abc.Mapping): + for elem, count in iterable.items(): + self[elem] = self_get(elem, 0) - count else: - for item in iterable: - self._data[item] = self._data.get(item, 0) - 1 - if kwargs: - for key, value in kwargs.items(): - self._data[key] = self._data.get(key, 0) - value - - def most_common(self, n=None): - items = list(self._data.items()) - items.sort(key=lambda kv: kv[1], reverse=True) - if n is None: - return items - return items[:n] + for elem in iterable: + self[elem] = self_get(elem, 0) - 1 + if kwds: + self.subtract(kwds) - def elements(self): - for key, count in self._data.items(): - i = 0 - while i < count: - yield key - i += 1 + def copy(self): + 'Return a shallow copy.' + return self.__class__(self) - def total(self): - return sum(self._data.values()) + def __reduce__(self): + return self.__class__, (dict(self),) - def __missing__(self, key): - return 0 + def __delitem__(self, elem): + 'Like dict.__delitem__() but does not raise KeyError for missing values.' + if elem in self: + super().__delitem__(elem) def __repr__(self): - return "Counter(" + repr(self._data) + ")" + if not self: + return f'{self.__class__.__name__}()' + try: + # dict() preserves the ordering returned by most_common() + d = dict(self.most_common()) + except TypeError: + # handle case where values are not orderable + d = dict(self) + return f'{self.__class__.__name__}({d!r})' + + # Multiset-style mathematical operations discussed in: + # Knuth TAOCP Volume II section 4.6.3 exercise 19 + # and at http://en.wikipedia.org/wiki/Multiset + # + # Outputs guaranteed to only include positive counts. + # + # To strip negative and zero counts, add-in an empty counter: + # c += Counter() + # + # Results are ordered according to when an element is first + # encountered in the left operand and then by the order + # encountered in the right operand. + # + # When the multiplicities are all zero or one, multiset operations + # are guaranteed to be equivalent to the corresponding operations + # for regular sets. + # Given counter multisets such as: + # cp = Counter(a=1, b=0, c=1) + # cq = Counter(c=1, d=0, e=1) + # The corresponding regular sets would be: + # sp = {'a', 'c'} + # sq = {'c', 'e'} + # All of the following relations would hold: + # set(cp + cq) == sp | sq + # set(cp - cq) == sp - sq + # set(cp | cq) == sp | sq + # set(cp & cq) == sp & sq + # (cp == cq) == (sp == sq) + # (cp != cq) == (sp != sq) + # (cp <= cq) == (sp <= sq) + # (cp < cq) == (sp < sq) + # (cp >= cq) == (sp >= sq) + # (cp > cq) == (sp > sq) + + def __eq__(self, other): + 'True if all counts agree. Missing counts are treated as zero.' + if not isinstance(other, Counter): + return NotImplemented + return all(self[e] == other[e] for c in (self, other) for e in c) + + def __ne__(self, other): + 'True if any counts disagree. Missing counts are treated as zero.' + if not isinstance(other, Counter): + return NotImplemented + return not self == other + + def __le__(self, other): + 'True if all counts in self are a subset of those in other.' + if not isinstance(other, Counter): + return NotImplemented + return all(self[e] <= other[e] for c in (self, other) for e in c) + + def __lt__(self, other): + 'True if all counts in self are a proper subset of those in other.' + if not isinstance(other, Counter): + return NotImplemented + return self <= other and self != other + + def __ge__(self, other): + 'True if all counts in self are a superset of those in other.' + if not isinstance(other, Counter): + return NotImplemented + return all(self[e] >= other[e] for c in (self, other) for e in c) + + def __gt__(self, other): + 'True if all counts in self are a proper superset of those in other.' + if not isinstance(other, Counter): + return NotImplemented + return self >= other and self != other def __add__(self, other): + '''Add counts from two counters. + + >>> Counter('abbb') + Counter('bcc') + Counter({'b': 4, 'c': 2, 'a': 1}) + + ''' if not isinstance(other, Counter): return NotImplemented result = Counter() - for k, v in self._data.items(): - new = v + other._data.get(k, 0) - if new > 0: - result._data[k] = new - for k, v in other._data.items(): - if k not in self._data and v > 0: - result._data[k] = v + for elem, count in self.items(): + newcount = count + other[elem] + if newcount > 0: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count return result def __sub__(self, other): + ''' Subtract count, but keep only results with positive counts. + + >>> Counter('abbbc') - Counter('bccd') + Counter({'b': 2, 'a': 1}) + + ''' if not isinstance(other, Counter): return NotImplemented result = Counter() - for k, v in self._data.items(): - new = v - other._data.get(k, 0) - if new > 0: - result._data[k] = new + for elem, count in self.items(): + newcount = count - other[elem] + if newcount > 0: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count < 0: + result[elem] = 0 - count return result def __or__(self, other): + '''Union is the maximum of value in either of the input counters. + + >>> Counter('abbb') | Counter('bcc') + Counter({'b': 3, 'c': 2, 'a': 1}) + + ''' if not isinstance(other, Counter): return NotImplemented result = Counter() - for k, v in self._data.items(): - other_v = other._data.get(k, 0) - best = v if v > other_v else other_v - if best > 0: - result._data[k] = best - for k, v in other._data.items(): - if k not in self._data and v > 0: - result._data[k] = v + for elem, count in self.items(): + other_count = other[elem] + newcount = other_count if count < other_count else count + if newcount > 0: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count return result def __and__(self, other): + ''' Intersection is the minimum of corresponding counts. + + >>> Counter('abbb') & Counter('bcc') + Counter({'b': 1}) + + ''' if not isinstance(other, Counter): return NotImplemented result = Counter() - for k, v in self._data.items(): - other_v = other._data.get(k, 0) - best = v if v < other_v else other_v - if best > 0: - result._data[k] = best + for elem, count in self.items(): + other_count = other[elem] + newcount = count if count < other_count else other_count + if newcount > 0: + result[elem] = newcount return result def __pos__(self): + 'Adds an empty counter, effectively stripping negative and zero counts' result = Counter() - for k, v in self._data.items(): - if v > 0: - result._data[k] = v + for elem, count in self.items(): + if count > 0: + result[elem] = count return result def __neg__(self): + '''Subtracts from an empty counter. Strips positive and zero counts, + and flips the sign on negative counts. + + ''' result = Counter() - for k, v in self._data.items(): - if v < 0: - result._data[k] = -v + for elem, count in self.items(): + if count < 0: + result[elem] = 0 - count return result + def _keep_positive(self): + '''Internal method to strip elements with a negative or zero count''' + nonpositive = [elem for elem, count in self.items() if not count > 0] + for elem in nonpositive: + del self[elem] + return self + + def __iadd__(self, other): + '''Inplace add from another counter, keeping only positive counts. + + >>> c = Counter('abbb') + >>> c += Counter('bcc') + >>> c + Counter({'b': 4, 'c': 2, 'a': 1}) + + ''' + for elem, count in other.items(): + self[elem] += count + return self._keep_positive() + + def __isub__(self, other): + '''Inplace subtract counter, but keep only results with positive counts. + + >>> c = Counter('abbbc') + >>> c -= Counter('bccd') + >>> c + Counter({'b': 2, 'a': 1}) + + ''' + for elem, count in other.items(): + self[elem] -= count + return self._keep_positive() + + def __ior__(self, other): + '''Inplace union is the maximum of value from either counter. + + >>> c = Counter('abbb') + >>> c |= Counter('bcc') + >>> c + Counter({'b': 3, 'c': 2, 'a': 1}) + + ''' + for elem, other_count in other.items(): + count = self[elem] + if other_count > count: + self[elem] = other_count + return self._keep_positive() + + def __iand__(self, other): + '''Inplace intersection is the minimum of corresponding counts. + + >>> c = Counter('abbb') + >>> c &= Counter('bcc') + >>> c + Counter({'b': 1}) -class ChainMap: - """View multiple dicts as a single mapping.""" + ''' + for elem, count in self.items(): + other_count = other[elem] + if other_count < count: + self[elem] = other_count + return self._keep_positive() + + +######################################################################## +### ChainMap +######################################################################## + +class ChainMap(_collections_abc.MutableMapping): + ''' A ChainMap groups multiple dicts (or other mappings) together + to create a single, updateable view. + + The underlying mappings are stored in a list. That list is public and can + be accessed or updated using the *maps* attribute. There is no other + state. + + Lookups search the underlying mappings successively until a key is found. + In contrast, writes, updates, and deletions only operate on the first + mapping. + + ''' def __init__(self, *maps): - if not maps: - maps = ({},) - self.maps = list(maps) + '''Initialize a ChainMap by setting *maps* to the given mappings. + If no mappings are provided, a single empty dictionary is used. - def __getitem__(self, key): - for m in self.maps: - if key in m: - return m[key] + ''' + self.maps = list(maps) or [{}] # always at least one map + + def __missing__(self, key): raise KeyError(key) + def __getitem__(self, key): + for mapping in self.maps: + try: + return mapping[key] # can't use 'key in mapping' with defaultdict + except KeyError: + pass + return self.__missing__(key) # support subclasses that define __missing__ + + def get(self, key, default=None): + return self[key] if key in self else default + + def __len__(self): + return len(set().union(*self.maps)) # reuses stored hash values if possible + + def __iter__(self): + d = {} + for mapping in map(dict.fromkeys, reversed(self.maps)): + d |= mapping # reuses stored hash values if possible + return iter(d) + + def __contains__(self, key): + return any(key in m for m in self.maps) + + def __bool__(self): + return any(self.maps) + + @_recursive_repr() + def __repr__(self): + return f'{self.__class__.__name__}({", ".join(map(repr, self.maps))})' + + @classmethod + def fromkeys(cls, iterable, value=None, /): + 'Create a new ChainMap with keys from iterable and values set to value.' + return cls(dict.fromkeys(iterable, value)) + + def copy(self): + 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]' + return self.__class__(self.maps[0].copy(), *self.maps[1:]) + + __copy__ = copy + + def new_child(self, m=None, **kwargs): # like Django's Context.push() + '''New ChainMap with a new map followed by all previous maps. + If no map is provided, an empty dict is used. + Keyword arguments update the map or new empty dict. + ''' + if m is None: + m = kwargs + elif kwargs: + m.update(kwargs) + return self.__class__(m, *self.maps) + + @property + def parents(self): # like Django's Context.pop() + 'New ChainMap from maps[1:].' + return self.__class__(*self.maps[1:]) + def __setitem__(self, key, value): self.maps[0][key] = value def __delitem__(self, key): - if key in self.maps[0]: + try: del self.maps[0][key] - else: - raise KeyError(key) + except KeyError: + raise KeyError(f'Key not found in the first mapping: {key!r}') - def __contains__(self, key): - for m in self.maps: - if key in m: - return True - return False + def popitem(self): + 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.' + try: + return self.maps[0].popitem() + except KeyError: + raise KeyError('No keys found in the first mapping.') + + def pop(self, key, *args): + 'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].' + try: + return self.maps[0].pop(key, *args) + except KeyError: + raise KeyError(f'Key not found in the first mapping: {key!r}') + + def clear(self): + 'Clear maps[0], leaving maps[1:] intact.' + self.maps[0].clear() + + def __ior__(self, other): + self.maps[0].update(other) + return self + + def __or__(self, other): + if not isinstance(other, _collections_abc.Mapping): + return NotImplemented + m = self.copy() + m.maps[0].update(other) + return m + + def __ror__(self, other): + if not isinstance(other, _collections_abc.Mapping): + return NotImplemented + m = dict(other) + for child in reversed(self.maps): + m.update(child) + return self.__class__(m) + + +################################################################################ +### UserDict +################################################################################ + +class UserDict(_collections_abc.MutableMapping): + + # Start by filling-out the abstract methods + def __init__(self, dict=None, /, **kwargs): + self.data = {} + if dict is not None: + self.update(dict) + if kwargs: + self.update(kwargs) def __len__(self): - seen = set() - for m in self.maps: - for k in m: - seen.add(k) - return len(seen) + return len(self.data) + + def __getitem__(self, key): + if key in self.data: + return self.data[key] + if hasattr(self.__class__, "__missing__"): + return self.__class__.__missing__(self, key) + raise KeyError(key) + + def __setitem__(self, key, item): + self.data[key] = item + + def __delitem__(self, key): + del self.data[key] def __iter__(self): - seen = set() - for m in self.maps: - for k in m: - if k not in seen: - seen.add(k) - yield k + return iter(self.data) + + # Modify __contains__ and get() to work like dict + # does when __missing__ is present. + def __contains__(self, key): + return key in self.data def get(self, key, default=None): - try: + if key in self: return self[key] - except KeyError: - return default + return default - def keys(self): - return list(iter(self)) - def values(self): - return [self[k] for k in self] + # Now, add the methods in dicts but not in MutableMapping + def __repr__(self): + return repr(self.data) - def items(self): - return [(k, self[k]) for k in self] + def __or__(self, other): + if isinstance(other, UserDict): + return self.__class__(self.data | other.data) + if isinstance(other, dict): + return self.__class__(self.data | other) + return NotImplemented - def new_child(self, m=None): - if m is None: - m = {} - return ChainMap(m, *self.maps) + def __ror__(self, other): + if isinstance(other, UserDict): + return self.__class__(other.data | self.data) + if isinstance(other, dict): + return self.__class__(other | self.data) + return NotImplemented - @property - def parents(self): - return ChainMap(*self.maps[1:]) + def __ior__(self, other): + if isinstance(other, UserDict): + self.data |= other.data + else: + self.data |= other + return self + + def __copy__(self): + inst = self.__class__.__new__(self.__class__) + inst.__dict__.update(self.__dict__) + # Create a copy and avoid triggering descriptors + inst.__dict__["data"] = self.__dict__["data"].copy() + return inst + + def copy(self): + if self.__class__ is UserDict: + return UserDict(self.data.copy()) + import copy + data = self.data + try: + self.data = {} + c = copy.copy(self) + finally: + self.data = data + c.update(self) + return c + + @classmethod + def fromkeys(cls, iterable, value=None): + d = cls() + for key in iterable: + d[key] = value + return d + + +################################################################################ +### UserList +################################################################################ + +class UserList(_collections_abc.MutableSequence): + """A more or less complete user-defined wrapper around list objects.""" + + def __init__(self, initlist=None): + self.data = [] + if initlist is not None: + # XXX should this accept an arbitrary sequence? + if type(initlist) == type(self.data): + self.data[:] = initlist + elif isinstance(initlist, UserList): + self.data[:] = initlist.data[:] + else: + self.data = list(initlist) def __repr__(self): - return "ChainMap(" + ", ".join(repr(m) for m in self.maps) + ")" + return repr(self.data) + def __lt__(self, other): + return self.data < self.__cast(other) -def namedtuple(typename, field_names, *, rename=False, defaults=None, module=None): - """Return a new lightweight class with the given fields. + def __le__(self, other): + return self.data <= self.__cast(other) - The result mirrors CPython's ``namedtuple`` API surface — iteration, - indexing, ``_asdict``, ``_replace``, and ``_fields`` — without - inheriting from the built-in ``tuple`` type, which would require - full MRO dispatch onto built-in types. - """ + def __eq__(self, other): + return self.data == self.__cast(other) - if isinstance(field_names, str): - field_names = field_names.replace(",", " ").split() - field_names = list(field_names) + def __gt__(self, other): + return self.data > self.__cast(other) - if rename: - seen = set() - for i, name in enumerate(field_names): - if ( - not name.isidentifier() - or name.startswith("_") - or name in seen - ): - field_names[i] = "_" + str(i) - seen.add(field_names[i]) + def __ge__(self, other): + return self.data >= self.__cast(other) - if defaults is not None: - defaults = tuple(defaults) - if len(defaults) > len(field_names): - raise TypeError("got more defaults than field names") - field_defaults = dict( - zip(field_names[-len(defaults):], defaults) - ) - else: - field_defaults = {} - - class _NT: - _fields = tuple(field_names) - _field_defaults = field_defaults - __match_args__ = tuple(field_names) - - @classmethod - def _make(cls, iterable): - return cls(*iterable) - - def __getnewargs__(self): - return tuple(self._values) - - def __init__(self, *args, **kwargs): - values = list(args) - i = len(values) - while i < len(field_names): - name = field_names[i] - if name in kwargs: - values.append(kwargs[name]) - elif name in field_defaults: - values.append(field_defaults[name]) - else: - raise TypeError( - "missing required argument: " + repr(name) - ) - i += 1 - if len(values) != len(field_names): - raise TypeError("wrong number of arguments") - self._values = tuple(values) - for name, value in zip(field_names, values): - setattr(self, name, value) - - def __iter__(self): - return iter(self._values) - - def __getitem__(self, index): - return self._values[index] - - def __len__(self): - return len(self._values) - - def __eq__(self, other): - if isinstance(other, _NT): - return self._values == other._values - if isinstance(other, tuple): - return self._values == other - return NotImplemented + def __cast(self, other): + return other.data if isinstance(other, UserList) else other - def _asdict(self): - return dict(zip(field_names, self._values)) - - def _replace(self, **changes): - values = list(self._values) - for i, name in enumerate(field_names): - if name in changes: - values[i] = changes.pop(name) - if changes: - # Match CPython: leftover keys are reported as a TypeError - # ("Got unexpected field names: [...]"). - raise TypeError( - "Got unexpected field names: " + repr(list(changes)) - ) - return type(self)(*values) - - def __replace__(self, **changes): - return self._replace(**changes) - - def __repr__(self): - parts = [] - for name, value in zip(field_names, self._values): - parts.append(name + "=" + repr(value)) - return typename + "(" + ", ".join(parts) + ")" - - _NT.__name__ = typename - _NT.__qualname__ = typename - if module is not None: - _NT.__module__ = module + def __contains__(self, item): + return item in self.data + + def __len__(self): + return len(self.data) + + def __getitem__(self, i): + if isinstance(i, slice): + return self.__class__(self.data[i]) + else: + return self.data[i] + + def __setitem__(self, i, item): + self.data[i] = item + + def __delitem__(self, i): + del self.data[i] + + def __add__(self, other): + if isinstance(other, UserList): + return self.__class__(self.data + other.data) + elif isinstance(other, type(self.data)): + return self.__class__(self.data + other) + return self.__class__(self.data + list(other)) + + def __radd__(self, other): + if isinstance(other, UserList): + return self.__class__(other.data + self.data) + elif isinstance(other, type(self.data)): + return self.__class__(other + self.data) + return self.__class__(list(other) + self.data) + + def __iadd__(self, other): + if isinstance(other, UserList): + self.data += other.data + elif isinstance(other, type(self.data)): + self.data += other + else: + self.data += list(other) + return self + + def __mul__(self, n): + return self.__class__(self.data * n) + + __rmul__ = __mul__ + + def __imul__(self, n): + self.data *= n + return self + + def __copy__(self): + inst = self.__class__.__new__(self.__class__) + inst.__dict__.update(self.__dict__) + # Create a copy and avoid triggering descriptors + inst.__dict__["data"] = self.__dict__["data"][:] + return inst + + def append(self, item): + self.data.append(item) + + def insert(self, i, item): + self.data.insert(i, item) + + def pop(self, i=-1): + return self.data.pop(i) + + def remove(self, item): + self.data.remove(item) + + def clear(self): + self.data.clear() + + def copy(self): + return self.__class__(self) + + def count(self, item): + return self.data.count(item) + + def index(self, item, *args): + return self.data.index(item, *args) + + def reverse(self): + self.data.reverse() + + def sort(self, /, *args, **kwds): + self.data.sort(*args, **kwds) + + def extend(self, other): + if isinstance(other, UserList): + self.data.extend(other.data) + else: + self.data.extend(other) + + +################################################################################ +### UserString +################################################################################ + +class UserString(_collections_abc.Sequence): + + def __init__(self, seq): + if isinstance(seq, str): + self.data = seq + elif isinstance(seq, UserString): + self.data = seq.data[:] + else: + self.data = str(seq) + + def __str__(self): + return str(self.data) + + def __repr__(self): + return repr(self.data) + + def __int__(self): + return int(self.data) + + def __float__(self): + return float(self.data) + + def __complex__(self): + return complex(self.data) + + def __hash__(self): + return hash(self.data) + + def __getnewargs__(self): + return (self.data[:],) + + def __eq__(self, string): + if isinstance(string, UserString): + return self.data == string.data + return self.data == string + + def __lt__(self, string): + if isinstance(string, UserString): + return self.data < string.data + return self.data < string + + def __le__(self, string): + if isinstance(string, UserString): + return self.data <= string.data + return self.data <= string + + def __gt__(self, string): + if isinstance(string, UserString): + return self.data > string.data + return self.data > string + + def __ge__(self, string): + if isinstance(string, UserString): + return self.data >= string.data + return self.data >= string + + def __contains__(self, char): + if isinstance(char, UserString): + char = char.data + return char in self.data + + def __len__(self): + return len(self.data) + + def __getitem__(self, index): + return self.__class__(self.data[index]) + + def __add__(self, other): + if isinstance(other, UserString): + return self.__class__(self.data + other.data) + elif isinstance(other, str): + return self.__class__(self.data + other) + return self.__class__(self.data + str(other)) + + def __radd__(self, other): + if isinstance(other, str): + return self.__class__(other + self.data) + return self.__class__(str(other) + self.data) + + def __mul__(self, n): + return self.__class__(self.data * n) + + __rmul__ = __mul__ + + def __mod__(self, args): + return self.__class__(self.data % args) + + def __rmod__(self, template): + return self.__class__(str(template) % self) + + # the following methods are defined in alphabetical order: + def capitalize(self): + return self.__class__(self.data.capitalize()) + + def casefold(self): + return self.__class__(self.data.casefold()) + + def center(self, width, *args): + return self.__class__(self.data.center(width, *args)) + + def count(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.count(sub, start, end) + + def removeprefix(self, prefix, /): + if isinstance(prefix, UserString): + prefix = prefix.data + return self.__class__(self.data.removeprefix(prefix)) + + def removesuffix(self, suffix, /): + if isinstance(suffix, UserString): + suffix = suffix.data + return self.__class__(self.data.removesuffix(suffix)) + + def encode(self, encoding='utf-8', errors='strict'): + encoding = 'utf-8' if encoding is None else encoding + errors = 'strict' if errors is None else errors + return self.data.encode(encoding, errors) + + def endswith(self, suffix, start=0, end=_sys.maxsize): + return self.data.endswith(suffix, start, end) + + def expandtabs(self, tabsize=8): + return self.__class__(self.data.expandtabs(tabsize)) + + def find(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.find(sub, start, end) + + def format(self, /, *args, **kwds): + return self.data.format(*args, **kwds) + + def format_map(self, mapping): + return self.data.format_map(mapping) + + def index(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.index(sub, start, end) + + def isalpha(self): + return self.data.isalpha() + + def isalnum(self): + return self.data.isalnum() + + def isascii(self): + return self.data.isascii() + + def isdecimal(self): + return self.data.isdecimal() + + def isdigit(self): + return self.data.isdigit() + + def isidentifier(self): + return self.data.isidentifier() + + def islower(self): + return self.data.islower() + + def isnumeric(self): + return self.data.isnumeric() + + def isprintable(self): + return self.data.isprintable() + + def isspace(self): + return self.data.isspace() + + def istitle(self): + return self.data.istitle() + + def isupper(self): + return self.data.isupper() + + def join(self, seq): + return self.data.join(seq) + + def ljust(self, width, *args): + return self.__class__(self.data.ljust(width, *args)) + + def lower(self): + return self.__class__(self.data.lower()) + + def lstrip(self, chars=None): + return self.__class__(self.data.lstrip(chars)) + + maketrans = str.maketrans + + def partition(self, sep): + return self.data.partition(sep) + + def replace(self, old, new, maxsplit=-1): + if isinstance(old, UserString): + old = old.data + if isinstance(new, UserString): + new = new.data + return self.__class__(self.data.replace(old, new, maxsplit)) + + def rfind(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.rfind(sub, start, end) + + def rindex(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.rindex(sub, start, end) + + def rjust(self, width, *args): + return self.__class__(self.data.rjust(width, *args)) + + def rpartition(self, sep): + return self.data.rpartition(sep) + + def rstrip(self, chars=None): + return self.__class__(self.data.rstrip(chars)) + + def split(self, sep=None, maxsplit=-1): + return self.data.split(sep, maxsplit) + + def rsplit(self, sep=None, maxsplit=-1): + return self.data.rsplit(sep, maxsplit) + + def splitlines(self, keepends=False): + return self.data.splitlines(keepends) + + def startswith(self, prefix, start=0, end=_sys.maxsize): + return self.data.startswith(prefix, start, end) + + def strip(self, chars=None): + return self.__class__(self.data.strip(chars)) + + def swapcase(self): + return self.__class__(self.data.swapcase()) + + def title(self): + return self.__class__(self.data.title()) - return _NT + def translate(self, *args): + return self.__class__(self.data.translate(*args)) + def upper(self): + return self.__class__(self.data.upper()) -# Pull in the abc-backed user wrappers last (see note near `__all__`). -from _collections_user import UserDict, UserList, UserString + def zfill(self, width): + return self.__class__(self.data.zfill(width)) diff --git a/crates/weavepy-vm/src/stdlib/python/inspect.py b/crates/weavepy-vm/src/stdlib/python/inspect.py index ed2cb33..53e6fdd 100644 --- a/crates/weavepy-vm/src/stdlib/python/inspect.py +++ b/crates/weavepy-vm/src/stdlib/python/inspect.py @@ -296,6 +296,20 @@ def isasyncgen(obj): return type(obj).__name__ == "async_generator" +def isawaitable(obj): + """True for coroutines, iterable-coroutine generators, and objects + with a `__await__` method (CPython `inspect.isawaitable`).""" + if iscoroutine(obj): + return True + tn = type(obj).__name__ + if tn == "generator": + code = getattr(obj, "gi_code", None) + return bool(getattr(code, "co_flags", 0) & CO_ITERABLE_COROUTINE) + # CPython also accepts any object implementing __await__ (including + # its own async_generator_asend/athrow awaitables, which expose it). + return hasattr(type(obj), "__await__") or hasattr(obj, "__await__") + + def isasyncgenfunction(obj): code = getattr(obj, "__code__", None) if code is None: diff --git a/crates/weavepy-vm/src/stdlib/python/tempfile.py b/crates/weavepy-vm/src/stdlib/python/tempfile.py index 44f960a..100e852 100644 --- a/crates/weavepy-vm/src/stdlib/python/tempfile.py +++ b/crates/weavepy-vm/src/stdlib/python/tempfile.py @@ -32,6 +32,23 @@ def mkdtemp(suffix=None, prefix=None, dir=None): return _tempfile.mkdtemp(suffix, prefix, dir) +def mktemp(suffix="", prefix=None, dir=None): + """Return a unique pathname that did not exist at call time + (deprecated in CPython, but still exercised by tests).""" + if dir is None: + dir = gettempdir() + if prefix is None: + prefix = gettempprefix() + import random + letters = "abcdefghijklmnopqrstuvwxyz0123456789_" + for _ in range(10000): + name = "".join(random.choice(letters) for _ in range(8)) + path = os.path.join(dir, prefix + name + suffix) + if not os.path.exists(path): + return path + raise FileExistsError("No usable temporary filename found") + + class _NamedTempFile: """Wraps an `open()`ed file with a `name` attribute and (optional) delete-on-close semantics. Returned by `NamedTemporaryFile`.""" @@ -102,6 +119,6 @@ def cleanup(self): __all__ = [ - "gettempdir", "gettempprefix", "mkstemp", "mkdtemp", + "gettempdir", "gettempprefix", "mkstemp", "mkdtemp", "mktemp", "NamedTemporaryFile", "TemporaryDirectory", ] diff --git a/crates/weavepy-vm/src/stdlib/python/types_mod.py b/crates/weavepy-vm/src/stdlib/python/types_mod.py index 53b6404..b76eca1 100644 --- a/crates/weavepy-vm/src/stdlib/python/types_mod.py +++ b/crates/weavepy-vm/src/stdlib/python/types_mod.py @@ -80,13 +80,11 @@ async def _ag(): _a = _ag() -try: - AsyncGeneratorType = type(_a) -finally: - try: - _a.aclose() - except Exception: - pass +AsyncGeneratorType = type(_a) +# The never-started bootstrap agen needs no aclose() — calling it would +# create (and discard) an aclose awaitable, tripping the gh-113753 +# "was never awaited" RuntimeWarning during interpreter startup. +del _a, _ag class _C: diff --git a/crates/weavepy-vm/src/stdlib/python/warnings.py b/crates/weavepy-vm/src/stdlib/python/warnings.py index b662b81..396e8d6 100644 --- a/crates/weavepy-vm/src/stdlib/python/warnings.py +++ b/crates/weavepy-vm/src/stdlib/python/warnings.py @@ -156,6 +156,29 @@ def warn(message, category=UserWarning, stacklevel=1, source=None): registry=registry, module_globals=globals_, source=source) +def _warn_unawaited_coroutine(coro): + """Called by the VM when a coroutine is finalized without ever + being awaited (CPython's identically-named hook in Lib/warnings.py). + Appends the cr_origin creation traceback when origin tracking is on. + """ + msg_lines = [ + f"coroutine '{coro.__qualname__}' was never awaited\n" + ] + if getattr(coro, "cr_origin", None) is not None: + import linecache + import traceback + + def extract(): + for filename, lineno, funcname in reversed(coro.cr_origin): + line = linecache.getline(filename, lineno).strip() + yield (filename, lineno, funcname, line) + + msg_lines.append("Coroutine created at (most recent call last)\n") + msg_lines += traceback.format_list(list(extract())) + msg = "".join(msg_lines).rstrip("\n") + warn(msg, category=RuntimeWarning, stacklevel=2, source=coro) + + def warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None): if registry is None: diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index a212205..8fe0481 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -302,6 +302,31 @@ pub fn build_with_state( DictKey(Object::from_static("getrefcount")), builtin("getrefcount", sys_getrefcount), ); + d.insert( + DictKey(Object::from_static("get_coroutine_origin_tracking_depth")), + builtin("get_coroutine_origin_tracking_depth", |_| { + Ok(Object::Int(coroutine_origin_tracking_depth())) + }), + ); + d.insert( + DictKey(Object::from_static("set_coroutine_origin_tracking_depth")), + builtin( + "set_coroutine_origin_tracking_depth", + sys_set_coroutine_origin_tracking_depth, + ), + ); + d.insert( + DictKey(Object::from_static("get_asyncgen_hooks")), + builtin("get_asyncgen_hooks", sys_get_asyncgen_hooks), + ); + d.insert( + DictKey(Object::from_static("set_asyncgen_hooks")), + Object::Builtin(Rc::new(BuiltinFn { + name: "set_asyncgen_hooks", + call: Box::new(|args| sys_set_asyncgen_hooks(args, &[])), + call_kw: Some(Box::new(sys_set_asyncgen_hooks)), + })), + ); // `displayhook` — invoked by the REPL after every // evaluated expression. Default writes `repr(value)` to // stdout and stashes the value in `builtins._`. The hook @@ -1379,6 +1404,107 @@ fn sys_getrefcount(args: &[Object]) -> Result { Ok(Object::Int(visible.max(1) as i64)) } +thread_local! { + /// PEP 565-era coroutine origin tracking depth + /// (`sys.set_coroutine_origin_tracking_depth`). Per-thread in + /// CPython (a `PyThreadState` field). + static CORO_ORIGIN_DEPTH: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Current `sys.get_coroutine_origin_tracking_depth()` value; read by +/// the interpreter when constructing coroutine objects. +pub fn coroutine_origin_tracking_depth() -> i64 { + CORO_ORIGIN_DEPTH.with(std::cell::Cell::get) +} + +fn sys_set_coroutine_origin_tracking_depth(args: &[Object]) -> Result { + let depth = match args.first() { + Some(Object::Int(i)) => *i, + Some(Object::Bool(b)) => i64::from(*b), + _ => { + return Err(type_error( + "set_coroutine_origin_tracking_depth() takes an integer", + )) + } + }; + if depth < 0 { + return Err(crate::error::value_error("depth must be >= 0")); + } + CORO_ORIGIN_DEPTH.with(|c| c.set(depth)); + Ok(Object::None) +} + +thread_local! { + /// PEP 525 `sys.set_asyncgen_hooks` — `(firstiter, finalizer)`. + /// Per-thread in CPython (a `PyThreadState` field). + static ASYNCGEN_HOOKS: std::cell::RefCell<(Object, Object)> = + std::cell::RefCell::new((Object::None, Object::None)); +} + +/// The currently-installed `(firstiter, finalizer)` asyncgen hooks. +pub fn asyncgen_hooks() -> (Object, Object) { + ASYNCGEN_HOOKS.with(|h| h.borrow().clone()) +} + +fn check_asyncgen_hook(v: &Object, which: &str) -> Result<(), RuntimeError> { + let callable = matches!( + v, + Object::Function(_) + | Object::Builtin(_) + | Object::BoundMethod(_) + | Object::Type(_) + | Object::StaticMethod(_) + ) || matches!(v, Object::Instance(inst) if inst.cls().lookup("__call__").is_some()); + if matches!(v, Object::None) || callable { + Ok(()) + } else { + Err(type_error(format!( + "callable {which} expected, got {}", + v.type_name() + ))) + } +} + +fn sys_set_asyncgen_hooks( + args: &[Object], + kwargs: &[(String, Object)], +) -> Result { + let mut firstiter = args.first().cloned(); + let mut finalizer = args.get(1).cloned(); + for (k, v) in kwargs { + match k.as_str() { + "firstiter" => firstiter = Some(v.clone()), + "finalizer" => finalizer = Some(v.clone()), + other => { + return Err(type_error(format!( + "set_asyncgen_hooks() got an unexpected keyword argument '{other}'" + ))) + } + } + } + if let Some(f) = &firstiter { + check_asyncgen_hook(f, "firstiter")?; + } + if let Some(f) = &finalizer { + check_asyncgen_hook(f, "finalizer")?; + } + ASYNCGEN_HOOKS.with(|h| { + let mut h = h.borrow_mut(); + if let Some(f) = firstiter { + h.0 = f; + } + if let Some(f) = finalizer { + h.1 = f; + } + }); + Ok(Object::None) +} + +fn sys_get_asyncgen_hooks(_args: &[Object]) -> Result { + let (firstiter, finalizer) = asyncgen_hooks(); + Ok(Object::new_tuple(vec![firstiter, finalizer])) +} + /// Default `sys.displayhook`: if the value is None do nothing, /// otherwise print `repr(value)` and stash on /// `builtins._`. Matches CPython's reference implementation. diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs index 05515cb..55bafc5 100644 --- a/crates/weavepy-vm/src/stdlib/weakref_real.rs +++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs @@ -241,11 +241,43 @@ fn install_proxy_forwarding(td: &mut DictData) { let target = proxy_target(args.first().ok_or_else(|| type_error("missing self"))?)?; proxy_forward_via_builtin("str", &target) } + fn fwd_setattr(args: &[Object]) -> Result { + let target = proxy_target(args.first().ok_or_else(|| type_error("missing self"))?)?; + let name = match args.get(1) { + Some(Object::Str(s)) => s.to_string(), + _ => return Err(type_error("attribute name must be string")), + }; + let value = args + .get(2) + .cloned() + .ok_or_else(|| type_error("__setattr__ expected 2 arguments"))?; + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| type_error("no running interpreter"))?; + // SAFETY: published by an enclosing VM frame on this thread. + let interp = unsafe { &mut *ptr }; + interp.store_attr_public(&target, &name, value)?; + Ok(Object::None) + } + fn fwd_delattr(args: &[Object]) -> Result { + let target = proxy_target(args.first().ok_or_else(|| type_error("missing self"))?)?; + let name = match args.get(1) { + Some(Object::Str(s)) => s.to_string(), + _ => return Err(type_error("attribute name must be string")), + }; + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| type_error("no running interpreter"))?; + // SAFETY: published by an enclosing VM frame on this thread. + let interp = unsafe { &mut *ptr }; + interp.delete_attr_public(&target, &name)?; + Ok(Object::None) + } for (name, f) in [ ( "__getattr__", fwd_getattr as fn(&[Object]) -> Result, ), + ("__setattr__", fwd_setattr), + ("__delattr__", fwd_delattr), ("__iter__", fwd_iter), ("__next__", fwd_next), ("__len__", fwd_len), diff --git a/tests/regrtest/expectations.toml b/tests/regrtest/expectations.toml index c02a718..ce66ea9 100644 --- a/tests/regrtest/expectations.toml +++ b/tests/regrtest/expectations.toml @@ -112,8 +112,8 @@ status = "skip" reason = "measured: past the sys.getsizeof/interning probes it reaches the same gc.collect() reachable-hang as test_set (tuple/iterator reference cycles). Marked skip so CI doesn't stall 30s per run; tracked with test_set as a GC reachable-hang to revisit in wave 3." [tests."cpython/Lib/test/test_bytes.py"] -status = "fail" -reason = "measured: first remaining failure is 'SkipTest: No module named _testlimitedcapi' raised at import of a C-API test helper WeavePy doesn't provide; the CAPI-dependent bytes subtests can't run." +status = "timeout" +reason = "measured: exceeds the 30s budget. RFC 0037 WS4 rebuilt bytes/bytearray (CPython-exact constructors with source/encoding/errors + __bytes__/__index__/__buffer__, find/index/count search windows (ADJUST_INDICES), fromhex/hex with sep, %-formatting incl. b'%(key)b' mappings and __rmod__, BufferError reentrancy guards, live bytearray iterators), so the suite now runs its full matrix — wall time is the blocker (completes with failures at larger budgets); needs the substring fast path." [tests."cpython/Lib/test/test_string.py"] status = "pass" @@ -124,8 +124,8 @@ status = "fail" reason = "depends on full unicodedata DB + width handling" [tests."cpython/Lib/test/test_math.py"] -status = "fail" -reason = "math module: domain edge cases (gamma, lgamma) differ from libm path" +status = "timeout" +reason = "measured: exceeds the 30s budget even sequentially (was a completes-and-fails row; the heavy numeric loops plus interpreter overhead push it over). Underlying gaps when it ran: domain edge cases (gamma, lgamma) vs the libm path." [tests."cpython/Lib/test/test_int.py"] status = "pass" @@ -149,7 +149,7 @@ reason = "RFC 0037 WS3/WS8: passes end-to-end. cmath is ported (pure-Python over [tests."cpython/Lib/test/test_collections.py"] status = "fail" -reason = "measured: dict(**kwargs)/dict(mapping,**kwargs) + collections.abc now work (RFC 0037); first remaining failure is a builtin '__new__' rejecting keyword arguments (namedtuple/typed-collection construction path)." +reason = "measured: completes in ~24s sequentially and fails 9F/15E (was 21F/24E). RFC 0037 WS8 swapped the collections shim for CPython's verbatim collections/__init__.py over a new _collections accelerator (pure-Python deque/defaultdict/_count_elements; OrderedDict/namedtuple run the reference pure-Python paths via dict.__missing__, weakproxy __setattr__ forwarding, tuple.__new__-from-iterable, PEP 584 dict |/|=). Remaining clusters: namedtuple __defaults__ assignment, ChainMap __or__ over ABC views, abc-view equality, set.__sub__ type-level dunders. NB: flips to 'timeout' under parallel contention (-j8) — the pure-Python OrderedDict linked list is the hot path; needs a native OrderedDict/_tuplegetter." [tests."cpython/Lib/test/test_array.py"] status = "fail" @@ -193,7 +193,7 @@ reason = "marshal ships (RFC 0033) and imports cleanly; full test_marshal.py con [tests."cpython/Lib/test/test_re.py"] status = "fail" -reason = "re engine: Unicode property classes + atomic groups" +reason = "measured: completes in ~23s sequentially with a mixed F/E run (re engine gaps: Unicode property classes + atomic groups among them). NB: flips to 'timeout' under parallel contention (-j4+) — borderline on the 30s budget." [tests."cpython/Lib/test/test_json.py"] status = "fail" @@ -288,8 +288,8 @@ status = "fail" reason = "measured (WS2): 90 tests run, 51 pass / 36 fail / 3 error (was 44/25/6). The lexer's f-string *extent* scanner now emits CPython's exact PEP 701 wording (unterminated f-string literal, unterminated triple-quoted f-string literal, f-string: expecting '}', f-string: expecting '}', or format specs, newlines-not-allowed-in-format-specifiers-for-single-quoted) and uses an explicit bracket *stack* (not a depth counter) so it reports closing-paren-does-not-match-opening, f-string: unmatched ')', and '{'/'(' was-never-closed (comment-to-EOF) — distinguishing a same-quote terminator (f'{3') from a real nested string (f'{3 + 'a'}'); test_not_closing_quotes/test_unterminated_string/test_newlines_in_format_specifiers/test_mismatched_parens/test_comments pass. The FORMAT_VALUE opcode now routes through __format__ like format(): !s/!r/!a convert-then-format-as-string (test_conversions), custom __format__ objects are honoured, object.__format__ rejects a non-empty spec with TypeError 'unsupported format string passed to T.__format__' (test_errors), and the int spec parser rejects duplicate ,/_ grouping with CPython's two messages (the four test_with_*_in_format_specifier pass). Remaining: AST source positions/lineno + compile(), decimal.Decimal.__format__, \\N{...} escape decoding, backslash SyntaxWarnings, lambda-without-parens message, and the lexer-finds-extent-first ordering that pre-empts the parser's 'valid expression required before X' on empty fields." [tests."cpython/Lib/test/test_class.py"] -status = "fail" -reason = "measured: the compiler now accepts **kwargs (and *bases) in a class header via the CallEx lowering (RFC 0037 WS2); the first remaining blocker is the module-level 'from _testinternalcapi import has_inline_values' — a CPython-internal managed-dict test helper WeavePy doesn't ship (the TestInlineValues subtests probe a CPython-specific layout detail)." +status = "pass" +reason = "measured (RFC 0037 WS6): passes end-to-end (skipped=1 — the _testinternalcapi inline-values probe skips cleanly)." [tests."cpython/Lib/test/test_dataclasses.py"] status = "fail" @@ -297,11 +297,11 @@ reason = "dataclass: __init_subclass__ + slots=True + kw_only" [tests."cpython/Lib/test/test_enum.py"] status = "fail" -reason = "measured: first remaining failure imports the not-yet-ported 'pydoc' module at setup — blocked on WS8 pydoc port (large; pulls in inspect/text-wrapping)." +reason = "measured: runs end-to-end in ~10s (the pydoc import blocker is gone) and fails on a broad WS6 cluster (~60F: doc/repr subtests plus enum corner semantics). NB: can flip to 'timeout' under parallel contention (-j8)." [tests."cpython/Lib/test/test_inspect.py"] -status = "fail" -reason = "inspect: Signature from C callables + getsource for frozen" +status = "pass" +reason = "measured but VACUOUS: test_inspect is a package whose __init__.py only defines a lazy load_tests(); running it directly executes zero tests and exits 0. Real inspect conformance (Signature from C callables, getsource for frozen) is still unverified — needs a load_package_tests-aware runner." [tests."cpython/Lib/test/test_typing.py"] status = "fail" @@ -320,20 +320,20 @@ status = "pass" reason = "RFC 0037 WS7: full iterator-protocol fidelity. The prior gc-reachable hang is gone — the legacy __getitem__ sequence protocol and iter(callable, sentinel) now build *lazy* iterators (frozen _seqtools _SeqIter/_CallableIter) instead of eagerly materialising, so an unbounded sequence iterates on demand. Built-in iterators gained a faithful __reduce__ ((iter, (remaining,)) / (reversed, (fwd,), idx)) that resolves the iter/reversed builtin through the live builtins module dict so a hash-colliding custom __eq__ exhausts the iterator before its state is snapshotted (gh-101765); a PEP 585 generic alias iterates by yielding typing.Unpack[self] once (matching CPython ga_iternext). Plus: file objects are iterable (for line in f / x in f / list(f)) and writelines accepts any iterable; pickle gained memoisation so co-referenced instances unpickle shared; closures over enclosing-function locals resolve in nested methods; and tracebacks carry PEP 657 column offsets (co_positions/f_lasti/tb_lasti translated to CPython byte offsets). (skipped=2 are @cpython_only refcount subtests.)" [tests."cpython/Lib/test/test_generators.py"] -status = "fail" -reason = "measured: first failure is 'InternalError: bad cell index' — closure-cell indexing bug in a generator frame; VM cell-resolution gap." +status = "pass" +reason = "measured (RFC 0037 WS7): passes end-to-end (skipped=1). Generator finalization (del-site/close-site frame reap, gc tracking), real sys.getrefcount, PEP 667 live f_locals on generator frames, frame chaining through yield-from throws, and CPython-exact yield-assignment SyntaxError messages." [tests."cpython/Lib/test/test_coroutines.py"] -status = "fail" -reason = "measured: the compiler now lowers nested async comprehensions via PEP 530 implicit-async propagation (an async comprehension nested in another comprehension's element makes the outer one a coroutine too) (RFC 0037 WS2/WS7); the suite now runs but most coroutine send/throw/await subtests error and the run ends in a VM 'stack underflow' — coroutine-driver fidelity gaps remain." +status = "pass" +reason = "measured (RFC 0037 WS7): passes end-to-end (skipped=3). Coroutine send/throw/close fidelity, PEP 530 implicit-async comprehension lowering, unawaited-coroutine RuntimeWarnings, and CPython-exact coroutine doctest SyntaxErrors." [tests."cpython/Lib/test/test_asyncgen.py"] -status = "fail" -reason = "measured: the test.support helpers + implicit-return fix let the suite run (RFC 0037 WS5/WS9); it now reports a broad run of errors ending in an unhandled GeneratorExit — async-generator aclose()/athrow() finalization semantics aren't matched." +status = "pass" +reason = "measured (RFC 0037 WS7): passes end-to-end. PEP 525 finalization hooks (sys.set_asyncgen_hooks / firstiter+finalizer), aclose()/athrow() semantics incl. GeneratorExit propagation, asyncio.sleep(0) yield points, and shutdown_asyncgens." [tests."cpython/Lib/test/test_with.py"] -status = "fail" -reason = "with: PEP 617 parenthesized context managers" +status = "pass" +reason = "measured (RFC 0037): passes end-to-end — parenthesized context managers and the with-protocol error paths all hold." [tests."cpython/Lib/test/test_exceptions.py"] status = "fail" @@ -348,8 +348,8 @@ status = "fail" reason = "warnings: filter precedence + showwarning override" [tests."cpython/Lib/test/test_contextlib.py"] -status = "fail" -reason = "contextlib: ExitStack.callback + contextmanager generator semantics" +status = "pass" +reason = "measured (RFC 0037 WS7): passes end-to-end — the verbatim contextlib port plus generator close()/throw() fidelity covers ExitStack/contextmanager semantics." [tests."cpython/Lib/test/test_contextvars.py"] status = "fail" @@ -436,8 +436,8 @@ status = "fail" reason = "random: Mersenne Twister state save/load + SystemRandom" [tests."cpython/Lib/test/test_statistics.py"] -status = "fail" -reason = "measured: runs all 374 cases and fails on missing numeric-tower features (NormalDist + harmonic_mean weighted forms, Fraction/Decimal interop). The former 'ran-further-now-slow' timeout cleared once RFC 0037 gave set/dict the Python __hash__/__eq__ key hook (no more single-bucket O(n^2) on custom-hashable keys); revisit the remaining failures once the numeric tower (WS3) lands." +status = "timeout" +reason = "measured: exceeds the 30s budget even sequentially. Previously ran all 374 cases and failed on missing numeric-tower features (NormalDist + harmonic_mean weighted forms, Fraction/Decimal interop); the verbatim-collections swap (pure-Python Counter/OrderedDict paths) plus those suites' heavy loops push wall time past the budget. Revisit with the WS8 perf pass (native _collections accelerators)." [tests."cpython/Lib/test/test_unicodedata.py"] status = "skip" @@ -586,8 +586,8 @@ status = "pass" reason = "RFC 0037 WS9: passes end-to-end (36 tests). Required faithful sys.settrace event fidelity in the VM dispatch loop: the frame-entry RESUME no longer emits a spurious 'line' event; an exception raised *inside* a trace callback propagates into the traced program (and disables the offending hook) for non-'exception' events while being swallowed on 'exception' events, matching CPython's call_trace_protected; Object::Frame compares by identity so bdb's `frame == self.stopframe`/returnframe checks work; generator/coroutine frames reuse one cached PyFrame snapshot across suspensions so frame identity is stable for set_next/until/return; FOR_ITER and SEND (`yield from`) surface a generator's terminating StopIteration to the 'exception' hook before swallowing it; END_FOR is attributed to the `for` line (not the loop body) so an exhausted loop emits no spurious body-line event; a frame popped by a propagating exception fires a 'return' event with arg None (sys.monitoring PY_UNWIND); and f_trace_lines / f_trace_opcodes are real per-frame flags driving 'opcode' events for bdb/pdb instruction stepping (set_stepinstr)." [tests."cpython/Lib/test/test_contextlib_async.py"] -status = "fail" -reason = "measured: contextlib.asynccontextmanager/AsyncExitStack now exist (RFC 0037 WS7 verbatim swap); first remaining failure is 'No module named test.test_contextlib' — this file imports its sibling test module, which isn't bundled." +status = "pass" +reason = "measured (RFC 0037 WS7): passes end-to-end — verbatim contextlib (asynccontextmanager/AsyncExitStack) over the WS7 coroutine machinery, with the sibling test.test_contextlib import resolving from the vendored tree." [tests."cpython/Lib/test/test_descrtut.py"] status = "fail" @@ -648,6 +648,14 @@ reason = "http.cookiejar not shipped yet" status = "skip" reason = "needs network + test.support.requires_subprocess; urllib2-style handlers unverified in the sandbox" +[tests."cpython/Lib/test/test_sqlite3.py"] +status = "fail" +reason = "measured: the package __init__ does 'from test.support import load_package_tests' at import, which WeavePy's test.support shim doesn't provide (same gap as test_zoneinfo; test_inspect only references it lazily)." + +[tests."cpython/Lib/test/test_zoneinfo.py"] +status = "fail" +reason = "measured: the package __init__ does 'from test.support import load_package_tests' at import, which WeavePy's test.support shim doesn't provide (same gap as test_sqlite3)." + [tests."cpython/Lib/test/test_xml_etree.py"] status = "skip" reason = "pyexpat C accelerator not shipped; ElementTree C-parser paths unavailable" From 8dd36c20722800ea5a0ef817be84f651fa5bf826 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 11 Jun 2026 00:46:08 -0700 Subject: [PATCH 9/9] feat: advance CPython Lib/test conformance wave 2 --- crates/weavepy-compiler/src/lib.rs | 132 ++- crates/weavepy-parser/src/ast.rs | 31 +- crates/weavepy-parser/src/parser.rs | 39 +- crates/weavepy-vm/src/builtin_types.rs | 29 + crates/weavepy-vm/src/builtins.rs | 259 ++++- crates/weavepy-vm/src/gc_trace.rs | 10 +- crates/weavepy-vm/src/lib.rs | 476 ++++++-- crates/weavepy-vm/src/object.rs | 123 +- crates/weavepy-vm/src/stdlib/ast_mod.rs | 15 +- crates/weavepy-vm/src/stdlib/python/heapq.py | 684 +++++++++-- crates/weavepy-vm/src/stdlib/python/pickle.py | 19 +- .../python/test_support_import_helper.py | 11 +- .../src/stdlib/python/test_support_init.py | 18 +- crates/weavepy-vm/src/stdlib/python/typing.py | 70 ++ .../weavepy-vm/src/stdlib/python/warnings.py | 22 + crates/weavepy-vm/src/stdlib/symtable_mod.rs | 8 + crates/weavepy-vm/src/stdlib/thread_real.rs | 2 + crates/weavepy-vm/src/stdlib/weakref_real.rs | 1 + crates/weavepy-vm/src/type_surface.rs | 1029 +++++++++++++++++ crates/weavepy-vm/src/types.rs | 67 ++ .../weavepy/tests/fixtures/run/36_slots.out | 4 +- 21 files changed, 2788 insertions(+), 261 deletions(-) create mode 100644 crates/weavepy-vm/src/type_surface.rs diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index e38ec9e..ec76b08 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -1132,16 +1132,30 @@ impl Compiler { args, body, decorator_list, + type_params, + returns, } => { - self.compile_function_def(name, args, body, decorator_list)?; + self.compile_pep695_prologue(type_params, stmt.span)?; + self.compile_function_def(name, args, body, decorator_list, returns.as_deref())?; + self.compile_pep695_epilogue(name, type_params, stmt.span)?; } StmtKind::AsyncFunctionDef { name, args, body, decorator_list, + type_params, + returns, } => { - self.compile_async_function_def(name, args, body, decorator_list)?; + self.compile_pep695_prologue(type_params, stmt.span)?; + self.compile_async_function_def( + name, + args, + body, + decorator_list, + returns.as_deref(), + )?; + self.compile_pep695_epilogue(name, type_params, stmt.span)?; } StmtKind::ClassDef { name, @@ -1149,8 +1163,11 @@ impl Compiler { keywords, body, decorator_list, + type_params, } => { + self.compile_pep695_prologue(type_params, stmt.span)?; self.compile_class_def(name, bases, keywords, body, decorator_list)?; + self.compile_pep695_epilogue(name, type_params, stmt.span)?; } StmtKind::Try { body, @@ -1711,14 +1728,108 @@ impl Compiler { /// Compile a function definition statement: builds the function /// object, threads it through any decorators, and binds the result /// to `name` in the enclosing scope. + /// PEP 695 lowering, part 1: bind each type parameter as a + /// `TypeVar` *before* the `def`/`class` compiles, so parameter and + /// return annotations referencing `T` resolve at definition time: + /// + /// ```text + /// T = __weavepy_typevar__('T') + /// def f(a: T): ... + /// f.__type_params__ = (T,) + /// f.__annotations__['return'] = R + /// del T + /// ``` + /// + /// CPython gives the parameters a dedicated lexical scope; the + /// flat-block approximation is observably equivalent here because + /// nothing reads the names after the epilogue's `del`. + fn compile_pep695_prologue( + &mut self, + type_params: &[String], + span: weavepy_lexer::Span, + ) -> Result<(), CompileError> { + if type_params.is_empty() { + return Ok(()); + } + let name_expr = |n: &str| Expr { + kind: ExprKind::Name(n.to_owned()), + span, + }; + for tp in type_params { + let assign = Stmt { + kind: StmtKind::Assign { + targets: vec![name_expr(tp)], + value: Expr { + kind: ExprKind::Call { + func: Box::new(name_expr("__weavepy_typevar__")), + args: vec![Expr { + kind: ExprKind::Constant(AstConstant::Str(tp.clone())), + span, + }], + keywords: Vec::new(), + }, + span, + }, + }, + span, + }; + self.compile_stmt(&assign)?; + } + Ok(()) + } + + /// PEP 695 lowering, part 2: after the `def`/`class` statement has + /// bound its name, stamp `__type_params__`, then drop the temporary + /// bindings. (The return annotation is *not* handled here — it goes + /// into the annotations dict at MakeFunction time, before + /// decorators wrap the function, exactly like CPython.) + fn compile_pep695_epilogue( + &mut self, + name: &str, + type_params: &[String], + span: weavepy_lexer::Span, + ) -> Result<(), CompileError> { + if type_params.is_empty() { + return Ok(()); + } + let name_expr = |n: &str| Expr { + kind: ExprKind::Name(n.to_owned()), + span, + }; + let set_params = Stmt { + kind: StmtKind::Assign { + targets: vec![Expr { + kind: ExprKind::Attribute { + value: Box::new(name_expr(name)), + attr: "__type_params__".to_owned(), + }, + span, + }], + value: Expr { + kind: ExprKind::Tuple(type_params.iter().map(|t| name_expr(t)).collect()), + span, + }, + }, + span, + }; + self.compile_stmt(&set_params)?; + let del = Stmt { + kind: StmtKind::Delete(type_params.iter().map(|t| name_expr(t)).collect()), + span, + }; + self.compile_stmt(&del)?; + Ok(()) + } + fn compile_function_def( &mut self, name: &str, args: &AstArguments, body: &[Stmt], decorator_list: &[Expr], + returns: Option<&Expr>, ) -> Result<(), CompileError> { - self.compile_function_def_inner(name, args, body, decorator_list, false) + self.compile_function_def_inner(name, args, body, decorator_list, returns, false) } fn compile_async_function_def( @@ -1727,8 +1838,9 @@ impl Compiler { args: &AstArguments, body: &[Stmt], decorator_list: &[Expr], + returns: Option<&Expr>, ) -> Result<(), CompileError> { - self.compile_function_def_inner(name, args, body, decorator_list, true) + self.compile_function_def_inner(name, args, body, decorator_list, returns, true) } fn compile_function_def_inner( @@ -1737,12 +1849,13 @@ impl Compiler { args: &AstArguments, body: &[Stmt], decorator_list: &[Expr], + returns: Option<&Expr>, is_async: bool, ) -> Result<(), CompileError> { for d in decorator_list { self.compile_expr(d)?; } - self.build_function_object_inner(name, args, body, is_async)?; + self.build_function_object_inner(name, args, body, returns, is_async)?; for _ in decorator_list { self.emit(OpCode::Call, 1); } @@ -1761,7 +1874,7 @@ impl Compiler { args: &AstArguments, body: &[Stmt], ) -> Result<(), CompileError> { - self.build_function_object_inner(name, args, body, false) + self.build_function_object_inner(name, args, body, None, false) } fn build_function_object_inner( @@ -1769,6 +1882,7 @@ impl Compiler { name: &str, args: &AstArguments, body: &[Stmt], + returns: Option<&Expr>, is_async: bool, ) -> Result<(), CompileError> { // Fast-local slots follow CPython's order exactly: @@ -1915,6 +2029,12 @@ impl Compiler { annotated_params.push((kw.name.clone(), ann)); } } + // `-> R` joins the same dict under the `'return'` key — at + // MakeFunction time, *before* decorators see the function + // (CPython compiles all annotations into one dict). + if let Some(ret) = returns { + annotated_params.push(("return".to_owned(), ret)); + } if !annotated_params.is_empty() { for (pname, ann) in &annotated_params { let idx = self.co.intern_constant(Constant::Str(pname.clone())); diff --git a/crates/weavepy-parser/src/ast.rs b/crates/weavepy-parser/src/ast.rs index 4249e7d..648597a 100644 --- a/crates/weavepy-parser/src/ast.rs +++ b/crates/weavepy-parser/src/ast.rs @@ -35,12 +35,16 @@ pub struct Stmt { #[derive(Debug, Clone, PartialEq)] pub enum StmtKind { - /// `def name(args): body` + /// `def name[T](args) -> returns: body` FunctionDef { name: String, args: Arguments, body: Vec, decorator_list: Vec, + /// PEP 695 type-parameter names (`def f[T, U](…)`). + type_params: Vec, + /// `-> annotation`, evaluated at definition time. + returns: Option>, }, /// `async def name(args): body` (PEP 492, RFC 0016). Same shape /// as [`StmtKind::FunctionDef`]; the compiler routes the body @@ -50,6 +54,10 @@ pub enum StmtKind { args: Arguments, body: Vec, decorator_list: Vec, + /// PEP 695 type-parameter names. + type_params: Vec, + /// `-> annotation`, evaluated at definition time. + returns: Option>, }, /// `class name(bases, **keywords): body` ClassDef { @@ -58,6 +66,8 @@ pub enum StmtKind { keywords: Vec, body: Vec, decorator_list: Vec, + /// PEP 695 type-parameter names (`class C[T](…)`). + type_params: Vec, }, /// `return value` Return(Option), @@ -555,6 +565,8 @@ fn dump_stmt(out: &mut String, s: &Stmt, depth: usize) { args, body, decorator_list, + returns, + .. } => { out.push_str("FunctionDef(name='"); out.push_str(name); @@ -569,13 +581,20 @@ fn dump_stmt(out: &mut String, s: &Stmt, depth: usize) { } dump_expr(out, d, depth); } - out.push_str("], returns=None, type_comment=None)"); + out.push_str("], returns="); + match returns { + Some(r) => dump_expr(out, r, depth), + None => out.push_str("None"), + } + out.push_str(", type_comment=None)"); } S::AsyncFunctionDef { name, args, body, decorator_list, + returns, + .. } => { out.push_str("AsyncFunctionDef(name='"); out.push_str(name); @@ -590,7 +609,12 @@ fn dump_stmt(out: &mut String, s: &Stmt, depth: usize) { } dump_expr(out, d, depth); } - out.push_str("], returns=None, type_comment=None)"); + out.push_str("], returns="); + match returns { + Some(r) => dump_expr(out, r, depth), + None => out.push_str("None"), + } + out.push_str(", type_comment=None)"); } S::ClassDef { name, @@ -598,6 +622,7 @@ fn dump_stmt(out: &mut String, s: &Stmt, depth: usize) { keywords, body, decorator_list, + .. } => { out.push_str("ClassDef(name='"); out.push_str(name); diff --git a/crates/weavepy-parser/src/parser.rs b/crates/weavepy-parser/src/parser.rs index 18ebec0..7864dec 100644 --- a/crates/weavepy-parser/src/parser.rs +++ b/crates/weavepy-parser/src/parser.rs @@ -586,30 +586,33 @@ impl<'src> Parser<'src> { let def_tok = self.bump(); // `def` let name_tok = self.expect(&TokenKind::Name, "function name")?; let name = self.ident(name_tok.span); - // PEP 695: optional `[T, *Ts, **P]` type-parameter list. - // Consumed-and-discarded for now — the names are real `TypeVar`-shaped - // objects in CPython, but the parser tolerates the syntax so generic - // libraries that target 3.12+ load. The compiler's downstream - // dataflow analysis doesn't see them today; pretty-printer round-trip - // loses them. - self.skip_pep695_type_params()?; + // PEP 695: optional `[T, *Ts, **P]` type-parameter list. The + // captured names desugar into `TypeVar` bindings around the def + // (see `desugar_pep695_def`) so annotations referencing them + // resolve and `f.__type_params__` is populated. + let type_params = self.collect_pep695_type_params()?; self.expect(&TokenKind::LPar, "`(`")?; let args = self.parse_function_arguments()?; self.expect(&TokenKind::RPar, "`)`")?; - if self.eat(&TokenKind::RArrow) { - let _ = self.parse_expression(false)?; - } + let returns = if self.eat(&TokenKind::RArrow) { + Some(self.parse_expression(false)?) + } else { + None + }; self.expect(&TokenKind::Colon, "`:`")?; let body = self.parse_block()?; let span_end = body.last().map_or(def_tok.span, |s| s.span); + let span = def_tok.span.merge(span_end); Ok(Stmt { kind: StmtKind::FunctionDef { name, args, body, decorator_list, + type_params, + returns: returns.map(Box::new), }, - span: def_tok.span.merge(span_end), + span, }) } @@ -627,12 +630,16 @@ impl<'src> Parser<'src> { args, body, decorator_list, + type_params, + returns, } => Ok(Stmt { kind: StmtKind::AsyncFunctionDef { name, args, body, decorator_list, + type_params, + returns, }, span: async_tok.span.merge(stmt.span), }), @@ -693,9 +700,9 @@ impl<'src> Parser<'src> { let class_tok = self.bump(); // `class` let name_tok = self.expect(&TokenKind::Name, "class name")?; let name = self.ident(name_tok.span); - // PEP 695: optional `[T, *Ts, **P]` type-parameter list (same as - // function form). - self.skip_pep695_type_params()?; + // PEP 695: optional `[T, *Ts, **P]` type-parameter list — same + // desugar as the function form (TypeVar bindings around the def). + let type_params = self.collect_pep695_type_params()?; let (bases, keywords) = if self.eat(&TokenKind::LPar) { let (a, kw) = self.parse_call_args()?; self.expect(&TokenKind::RPar, "`)`")?; @@ -706,6 +713,7 @@ impl<'src> Parser<'src> { self.expect(&TokenKind::Colon, "`:`")?; let body = self.parse_block()?; let span_end = body.last().map_or(class_tok.span, |s| s.span); + let span = class_tok.span.merge(span_end); Ok(Stmt { kind: StmtKind::ClassDef { name, @@ -713,8 +721,9 @@ impl<'src> Parser<'src> { keywords, body, decorator_list, + type_params, }, - span: class_tok.span.merge(span_end), + span, }) } diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index 8464688..5e1d9ad 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -51,6 +51,10 @@ pub struct BuiltinTypes { pub ellipsis_: Rc, pub not_implemented_type_: Rc, pub simple_namespace_: Rc, + /// `types.GenericAlias` — the type of PEP 585 aliases (`list[int]`). + pub generic_alias_: Rc, + /// `types.UnionType` — the type of PEP 604 unions (`int | str`). + pub union_type_: Rc, pub function_: Rc, pub method_: Rc, /// `builtin_function_or_method` — the type of Rust-implemented @@ -191,6 +195,25 @@ impl BuiltinTypes { let ellipsis_ = mk("ellipsis", vec![object_.clone()]); let not_implemented_type_ = mk("NotImplementedType", vec![object_.clone()]); let simple_namespace_ = mk("SimpleNamespace", vec![object_.clone()]); + // PEP 585 / PEP 604 runtime types. The *instances* are + // namespace-shaped (`Object::SimpleNamespace` carrying + // `__origin__` / `__args__`), but their reported class must be + // `types.GenericAlias` / `types.UnionType` as in CPython — + // `functools` does `GenericAlias = type(list[int])` and then both + // `isinstance(typ, GenericAlias)` and + // `__class_getitem__ = classmethod(GenericAlias)`. + let generic_alias_ = mk("GenericAlias", vec![object_.clone()]); + let union_type_ = mk("UnionType", vec![object_.clone()]); + for ty in [&generic_alias_, &union_type_] { + // Not in `as_globals` (they live in `types`, not `builtins`), + // so the bulk metaclass pass below won't reach them. + ty.set_metaclass(type_.clone()); + let mut d = ty.dict.borrow_mut(); + d.insert( + crate::object::DictKey(Object::from_static("__module__")), + Object::from_static("types"), + ); + } let function_ = mk("function", vec![object_.clone()]); // `types.MethodType` — the bound-method type. Distinct from // `function` so `type(obj.meth)` is `method` (as in CPython) and @@ -366,6 +389,8 @@ impl BuiltinTypes { ellipsis_, not_implemented_type_, simple_namespace_, + generic_alias_, + union_type_, function_, method_, builtin_function_, @@ -455,6 +480,10 @@ impl BuiltinTypes { // int.__dict__` is True — `enum._find_data_type_` uses exactly this to // recognise `int`/`str`/… as the mix-in data type. install_value_type_new(&bt); + // RFC 0037 — materialize the full method/dunder surface into the + // type dicts (CPython's `tp_dict` parity: `vars(list)`, + // `bytearray.__hash__ is None`, `_check_methods`-style ABC hooks). + crate::type_surface::install(&bt); bt } diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index e4cf752..135573f 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -658,11 +658,25 @@ pub fn lookup_method(obj: &Object, name: &str) -> Option { "__exit__" => Some(method("__exit__", memoryview_exit)), _ => None, }, - Object::DictView(_) | Object::MappingProxy(_) => match name { + Object::DictView(_) => match name { "isdisjoint" => Some(method("isdisjoint", view_isdisjoint)), "mapping" => None, _ => None, }, + // `mappingproxy` (read-only `type.__dict__` view) forwards the + // read-side mapping API to the wrapped dict. + Object::MappingProxy(_) => match name { + "isdisjoint" => Some(method("isdisjoint", view_isdisjoint)), + "get" => Some(method("get", mappingproxy_get)), + "keys" => Some(method("keys", mappingproxy_keys)), + "values" => Some(method("values", mappingproxy_values)), + "items" => Some(method("items", mappingproxy_items)), + "copy" => Some(method("copy", mappingproxy_copy)), + "__getitem__" => Some(method("__getitem__", mappingproxy_getitem)), + "__len__" => Some(method("__len__", obj_len)), + "__contains__" => Some(method("__contains__", obj_contains)), + _ => None, + }, Object::SimpleNamespace(_) => match name { "__repr__" => None, _ => None, @@ -1429,15 +1443,23 @@ pub fn builtin_type_dunder(base_name: &str, name: &str) -> Option { { return Some(Object::Builtin(Rc::new(method_kw("__call__", slot_call)))); } + // `tp_str` is defined only by `object` and `str` among the value types + // (CPython: `'__str__' in vars(int)` is False, hence + // `int.__str__ is object.__str__` — identity the enum bootstrap's + // `found_method in (data_type_method, object_method)` check relies + // on). Other types fall through here so the caller's MRO walk + // resolves `__str__` at `object`; exceptions get their own `__str__` + // via type-dict entries installed at startup. + if name == "__str__" { + if matches!(base_name, "object" | "str") { + return Some(Object::Builtin(Rc::new(method("__str__", slot_str)))); + } + return None; + } let (static_name, f): (&'static str, fn(&[Object]) -> Result) = match name { "__repr__" => ("__repr__", slot_repr), "__format__" => ("__format__", slot_format), - // Every built-in value type has its own `tp_str` in CPython - // (`int.__str__ is not object.__str__` — enum's ReprEnum wiring - // tests that identity), and `slot_str` already stringifies the - // receiver's native payload per type. - "__str__" => ("__str__", slot_str), // `object`'s default rich comparisons: `==`/`!=` compare by // identity (value identity for primitives) and return // `NotImplemented` otherwise; the orderings are always @@ -1509,13 +1531,29 @@ fn slot_sizeof(args: &[Object]) -> Result { } /// `object.__getstate__(self)` — PEP 307 default pickling state: the -/// instance `__dict__` when non-empty, else `None`. +/// instance `__dict__` when non-empty, else `None`. When `__slots__` +/// values are populated, CPython returns the 2-tuple +/// `(dict_or_None, {slot: value, …})` instead. fn slot_getstate(args: &[Object]) -> Result { let o = one(args, "__getstate__")?; if let Object::Instance(inst) = o { - if !inst.dict.borrow().is_empty() { - return Ok(Object::Dict(inst.dict.clone())); + let slots = inst.slots_snapshot(); + let dict_state = if inst.dict.borrow().is_empty() { + Object::None + } else { + Object::Dict(inst.dict.clone()) + }; + if !slots.is_empty() { + let mut slot_dict = crate::object::DictData::new(); + for (name, value) in slots { + slot_dict.insert(DictKey(Object::from_str(name)), value); + } + return Ok(Object::new_tuple(vec![ + dict_state, + Object::Dict(Rc::new(RefCell::new(slot_dict))), + ])); } + return Ok(dict_state); } Ok(Object::None) } @@ -2179,7 +2217,11 @@ fn attr_get(obj: &Object, name: &str) -> Option { } } Object::Function(f) => { - if let Some(v) = f + if crate::object::is_function_slot(name) { + if let Some(v) = f.slot(name) { + return Some(v); + } + } else if let Some(v) = f .attrs .borrow() .get(&crate::object::DictKey(Object::from_str(name))) @@ -2527,9 +2569,24 @@ fn code_varname_from_oparg(args: &[Object]) -> Result { /// The compiler keeps the leading bare string expression as /// ``constants[0]``; functions / modules / classes pick it up at /// runtime via this helper. +thread_local! { + /// Docstring objects keyed by the constant's string-data address, so + /// repeated `f.__doc__` reads return the *same* `str` object (CPython + /// stores the docstring once on the function; `update_wrapper` tests + /// `assertIs(wrapper.__doc__, wrapped.__doc__)`). + static DOCSTRING_CACHE: std::cell::RefCell> = + std::cell::RefCell::new(std::collections::HashMap::new()); +} + pub(crate) fn code_docstring(c: &weavepy_compiler::CodeObject) -> Option { match c.constants.first() { - Some(weavepy_compiler::Constant::Str(s)) => Some(Object::from_str(s.as_str())), + Some(weavepy_compiler::Constant::Str(s)) => Some(DOCSTRING_CACHE.with(|cache| { + cache + .borrow_mut() + .entry(s.as_ptr() as usize) + .or_insert_with(|| Object::from_str(s.as_str())) + .clone() + })), _ => None, } } @@ -2595,9 +2652,13 @@ fn attr_set(obj: &Object, name: &str, value: Object) -> Result<(), RuntimeError> Ok(()) } Object::Function(f) => { - f.attrs - .borrow_mut() - .insert(crate::object::DictKey(Object::from_str(name)), value); + if crate::object::is_function_slot(name) { + f.set_slot(name, value); + } else { + f.attrs + .borrow_mut() + .insert(crate::object::DictKey(Object::from_str(name)), value); + } Ok(()) } _ => Err(type_error(format!( @@ -2623,9 +2684,15 @@ fn attr_delete(obj: &Object, name: &str) -> Result<(), RuntimeError> { Ok(()) } Object::Function(f) => { - f.attrs - .borrow_mut() - .shift_remove(&crate::object::DictKey(Object::from_str(name))); + if crate::object::is_function_slot(name) { + f.slots + .borrow_mut() + .shift_remove(&crate::object::DictKey(Object::from_str(name))); + } else { + f.attrs + .borrow_mut() + .shift_remove(&crate::object::DictKey(Object::from_str(name))); + } Ok(()) } _ => Err(type_error(format!("cannot delete attribute '{}'", name))), @@ -4028,6 +4095,12 @@ fn b_tuple(args: &[Object]) -> Result { if args.is_empty() { return Ok(Object::new_tuple(Vec::new())); } + // `tuple(t)` on an exact tuple returns `t` itself (CPython reuses the + // immutable object; `copy.copy(partial).args is partial.args` relies + // on the identity). + if let Object::Tuple(_) = &args[0] { + return Ok(args[0].clone()); + } let mut it = args[0].make_iter()?; let mut out = Vec::new(); while let Some(v) = it.next_value() { @@ -4778,6 +4851,7 @@ pub fn make_super(class: Rc, receiver: Object) -> Obje })), native: None, inline_values: crate::sync::Cell::new(true), + slots: crate::sync::RefCell::new(None), }; Object::Instance(Rc::new(inst)) } @@ -4911,7 +4985,28 @@ pub fn class_of(obj: &Object) -> crate::sync::Rc { crate::object::DictViewKind::Values => bt.dict_values_.clone(), crate::object::DictViewKind::Items => bt.dict_items_.clone(), }, - Object::SimpleNamespace(_) => bt.simple_namespace_.clone(), + // Namespace-shaped objects double as the PEP 585/604 runtime + // forms; their *class* must report `types.GenericAlias` / + // `types.UnionType` (CPython: `type(list[int])`, `type(int|str)`). + Object::SimpleNamespace(d) => { + let dict = d.borrow(); + if dict + .get(&DictKey(Object::from_static("__is_pep604_union__"))) + .is_some() + { + bt.union_type_.clone() + } else if dict + .get(&DictKey(Object::from_static("__origin__"))) + .is_some() + && dict + .get(&DictKey(Object::from_static("__args__"))) + .is_some() + { + bt.generic_alias_.clone() + } else { + bt.simple_namespace_.clone() + } + } Object::Type(t) => t.metaclass_or_type(), Object::Function(_) => bt.function_.clone(), // Rust-implemented callables are `builtin_function_or_method`, @@ -5652,6 +5747,7 @@ fn b_mark_iterable_coroutine(args: &[Object]) -> Result { // Shared, not copied: `func.__dict__` mutations stay visible on // both, matching CPython where the function object is the same. attrs: f.attrs.clone(), + slots: RefCell::new(f.slots.borrow().clone()), }; Ok(Object::Function(Rc::new(marked))) } @@ -6733,8 +6829,10 @@ fn list_append(args: &[Object]) -> Result { // List dunders exposed on the type so `list.__setitem__` / // `super().__getitem__` resolve for `list` subclasses (`class C(list)`). -// Integer indices only — slice subscription routes through the VM's -// dedicated subscript opcodes, not this unbound-method path. +// These mirror CPython's `mp_subscript`/`mp_ass_subscript` slots fully: +// both integer and slice keys work (`_HashedSeq.__init__` does +// `self[:] = tup` on a `list` subclass, which dispatches here now that +// the materialized `__setitem__` is in the type dict). fn list_index_arg(l_len: usize, idx: &Object, what: &str) -> Result { match idx { Object::Int(i) => { @@ -6759,6 +6857,10 @@ fn list_getitem(args: &[Object]) -> Result { let key = args .get(1) .ok_or_else(|| type_error("__getitem__ expected 1 argument"))?; + if let Object::Slice(s) = key { + let seq = l.borrow().clone(); + return Ok(Object::new_list(crate::slice_seq(&seq, s)?)); + } let l = l.borrow(); let n = list_index_arg(l.len(), key, "__getitem__")?; Ok(l[n].clone()) @@ -6772,6 +6874,17 @@ fn list_setitem(args: &[Object]) -> Result { let val = args .get(2) .ok_or_else(|| type_error("__setitem__ expected 2 arguments"))?; + if let Object::Slice(s) = key { + // Materialize the replacement *before* the mutable borrow so + // self-assignment (`l[:] = l`) can't alias the live borrow. + let mut replacement = Vec::new(); + let mut it = val.make_iter()?; + while let Some(v) = it.next_value() { + replacement.push(v); + } + crate::apply_slice_assignment(&mut l.borrow_mut(), s, replacement)?; + return Ok(Object::None); + } let mut l = l.borrow_mut(); let n = list_index_arg(l.len(), key, "__setitem__")?; l[n] = val.clone(); @@ -6783,6 +6896,15 @@ fn list_delitem(args: &[Object]) -> Result { let key = args .get(1) .ok_or_else(|| type_error("__delitem__ expected 1 argument"))?; + if let Object::Slice(s) = key { + let mut l = l.borrow_mut(); + let mut indices = crate::slice_indices(l.len(), s)?; + indices.sort_unstable(); + for i in indices.into_iter().rev() { + l.remove(i); + } + return Ok(Object::None); + } let mut l = l.borrow_mut(); let n = list_index_arg(l.len(), key, "__delitem__")?; l.remove(n); @@ -6870,11 +6992,33 @@ fn list_index(args: &[Object]) -> Result { } let l = list_self(args)?; let l = l.borrow(); - let pos = l - .iter() - .position(|x| x.eq_value(&args[1])) - .ok_or_else(|| value_error("x not in list"))?; - Ok(Object::Int(pos as i64)) + // CPython `list.index(value, start=0, stop=maxsize)`: negative + // bounds count from the end and clamp to 0 (`PySlice_AdjustIndices` + // semantics), and the comparison is identity-first + // (`PyObject_RichCompareBool`). + let len = l.len() as i64; + let adjust = |v: i64| -> i64 { + if v < 0 { + (v + len).max(0) + } else { + v.min(len) + } + }; + let start = match args.get(2) { + Some(o) => adjust(coerce_index_i64(o)?), + None => 0, + }; + let stop = match args.get(3) { + Some(o) => adjust(coerce_index_i64(o)?), + None => len, + }; + for i in start..stop { + let x = &l[i as usize]; + if x.is_same(&args[1]) || x.eq_value(&args[1]) { + return Ok(Object::Int(i)); + } + } + Err(value_error(format!("{} is not in list", args[1].repr()))) } fn list_count(args: &[Object]) -> Result { @@ -7111,11 +7255,31 @@ fn tuple_index(args: &[Object]) -> Result { Some(Object::Tuple(t)) => t.clone(), _ => return Err(type_error("expected tuple")), }; - let pos = t - .iter() - .position(|x| x.eq_value(&args[1])) - .ok_or_else(|| value_error("x not in tuple"))?; - Ok(Object::Int(pos as i64)) + // Same `(value, start=0, stop=maxsize)` window + identity-first + // comparison semantics as `list.index`. + let len = t.len() as i64; + let adjust = |v: i64| -> i64 { + if v < 0 { + (v + len).max(0) + } else { + v.min(len) + } + }; + let start = match args.get(2) { + Some(o) => adjust(coerce_index_i64(o)?), + None => 0, + }; + let stop = match args.get(3) { + Some(o) => adjust(coerce_index_i64(o)?), + None => len, + }; + for i in start..stop { + let x = &t[i as usize]; + if x.is_same(&args[1]) || x.eq_value(&args[1]) { + return Ok(Object::Int(i)); + } + } + Err(value_error("tuple.index(x): x not in tuple")) } // ---------- dict extras ---------- @@ -9155,6 +9319,41 @@ fn memoryview_exit(_args: &[Object]) -> Result { // ----- dict view + mappingproxy methods (RFC 0023) ----- +/// Re-key a `mappingproxy` receiver as the wrapped dict so the dict +/// method implementations can be reused verbatim (the proxy is a +/// read-only *view*, so the share is intentional). +fn mappingproxy_args(args: &[Object]) -> Vec { + let mut v = args.to_vec(); + if let Some(Object::MappingProxy(d)) = v.first() { + v[0] = Object::Dict(d.clone()); + } + v +} + +fn mappingproxy_get(args: &[Object]) -> Result { + dict_get(&mappingproxy_args(args)) +} + +fn mappingproxy_keys(args: &[Object]) -> Result { + dict_keys(&mappingproxy_args(args)) +} + +fn mappingproxy_values(args: &[Object]) -> Result { + dict_values(&mappingproxy_args(args)) +} + +fn mappingproxy_items(args: &[Object]) -> Result { + dict_items(&mappingproxy_args(args)) +} + +fn mappingproxy_copy(args: &[Object]) -> Result { + dict_copy(&mappingproxy_args(args)) +} + +fn mappingproxy_getitem(args: &[Object]) -> Result { + dict_getitem(&mappingproxy_args(args)) +} + fn view_isdisjoint(args: &[Object]) -> Result { let other = args .get(1) diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 417dad8..3c66c83 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -658,6 +658,13 @@ pub fn traverse_object(obj: &Object, visit: &mut dyn FnMut(&Object)) { visit(&k.0); visit(v); } + drop(m); + if let Some(slots) = i.slots.borrow().as_ref() { + for (k, v) in slots.iter() { + visit(&k.0); + visit(v); + } + } } Object::Module(m) => { let dict = m.dict.borrow(); @@ -682,7 +689,7 @@ pub fn traverse_object(obj: &Object, visit: &mut dyn FnMut(&Object)) { visit(&p.fget); visit(&p.fset); visit(&p.fdel); - visit(&p.doc); + visit(&p.doc.borrow()); } Object::StaticMethod(o) | Object::ClassMethod(o) => { visit(o); @@ -781,6 +788,7 @@ pub fn clear_object_fields(obj: &Object) { } Object::Instance(i) => { i.dict.borrow_mut().clear(); + *i.slots.borrow_mut() = None; } Object::ByteArray(b) => { b.borrow_mut().clear(); diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 697dde2..8f29383 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -45,6 +45,7 @@ pub mod thread_registry; #[cfg(feature = "jit")] mod tier2; pub mod trace; +pub mod type_surface; pub mod types; pub mod vm_singletons; pub mod weakref_registry; @@ -3134,7 +3135,10 @@ impl Interpreter { } } let name = code.name.clone(); - let attrs = Rc::new(RefCell::new(DictData::new())); + // Function getset slots live *outside* `__dict__` + // (`f.__dict__` starts empty in CPython; only genuine + // user attributes land there). + let slots = RefCell::new(DictData::new()); // Stamp __module__ from globals['__name__'] (mirrors CPython's // function dispatch). Pickle relies on this to serialise the // function by qualified name. @@ -3144,7 +3148,7 @@ impl Interpreter { .get(&DictKey(Object::from_static("__name__"))) .cloned() { - attrs + slots .borrow_mut() .insert(DictKey(Object::from_static("__module__")), name_obj); } @@ -3155,19 +3159,18 @@ impl Interpreter { // stable identity, which `assertIs(wrapper.__name__, // func.__name__)` in test_decorators relies on. let name_obj = Object::from_str(name.clone()); - attrs.borrow_mut().insert( - DictKey(Object::from_static("__name__")), - name_obj.clone(), - ); + slots + .borrow_mut() + .insert(DictKey(Object::from_static("__name__")), name_obj.clone()); // `__qualname__` is the code object's PEP 3155 dotted name // (computed at compile time from lexical nesting), not the // bare `__name__`. Pinned as a stable object like `__name__`. - attrs.borrow_mut().insert( + slots.borrow_mut().insert( DictKey(Object::from_static("__qualname__")), Object::from_str(code.qualname.clone()), ); if let Some(ann) = annotations_obj { - attrs + slots .borrow_mut() .insert(DictKey(Object::from_static("__annotations__")), ann); } @@ -3178,7 +3181,8 @@ impl Interpreter { defaults, kw_defaults, closure, - attrs, + attrs: Rc::new(RefCell::new(DictData::new())), + slots, }; frame.push(Object::Function(Rc::new(f))); } @@ -4282,7 +4286,7 @@ impl Interpreter { "fget" => Ok(p.fget.clone()), "fset" => Ok(p.fset.clone()), "fdel" => Ok(p.fdel.clone()), - "__doc__" => Ok(p.doc.clone()), + "__doc__" => Ok(p.doc()), // CPython computes `property.__isabstractmethod__` as the // OR of the wrapped accessors' flags, so the modern // `@property` / `@abstractmethod` stacking marks the @@ -4411,6 +4415,15 @@ impl Interpreter { if name == "__dict__" { return Ok(Object::Dict(d.clone())); } + // Read-side mapping API (`get`/`keys`/`items`/…) comes from + // the method table (enum's `__setattr__` does + // `cls.__dict__.get(name)` on exactly this proxy type). + if let Some(m) = self.lookup_method(obj, name) { + return Ok(Object::BoundMethod(Rc::new(BoundMethod { + receiver: obj.clone(), + function: m, + }))); + } Err(attribute_error(format!( "'mappingproxy' object has no attribute '{name}'" ))) @@ -4440,7 +4453,13 @@ impl Interpreter { } }, Object::Function(f) => { - if let Some(v) = f.attrs.borrow().get(&DictKey(Object::from_str(name))) { + // Slot dunders are data descriptors in CPython — they + // resolve *before* (and never through) `__dict__`. + if crate::object::is_function_slot(name) { + if let Some(v) = f.slot(name) { + return Ok(v); + } + } else if let Some(v) = f.attrs.borrow().get(&DictKey(Object::from_str(name))) { return Ok(v.clone()); } match name { @@ -4448,7 +4467,7 @@ impl Interpreter { // PEP 3155 qualname comes from the code object (computed // at compile time from lexical nesting), unless user code // has overridden it via `f.__qualname__ = …` (handled by - // the `f.attrs` lookup above). + // the slot lookup above). "__qualname__" => return Ok(Object::from_str(&f.code.qualname)), "__doc__" => { // CPython convention: the first statement of @@ -4459,7 +4478,7 @@ impl Interpreter { } "__module__" => { // Fall back to globals['__name__'] if the function's - // attrs dict didn't already pin a value (e.g. for + // slots didn't already pin a value (e.g. for // synthesised functions in tests / REPL). if let Some(name_obj) = f .globals @@ -4474,6 +4493,19 @@ impl Interpreter { "__dict__" => return Ok(Object::Dict(f.attrs.clone())), "__code__" => return Ok(Object::Code(f.code.clone())), "__globals__" => return Ok(Object::Dict(f.globals.clone())), + // CPython `function.__builtins__`: the builtins mapping + // the function executes against — `__globals__['__builtins__']` + // when present, else the interpreter's builtins dict. + "__builtins__" => { + if let Some(b) = f + .globals + .borrow() + .get(&DictKey(Object::from_static("__builtins__"))) + { + return Ok(b.clone()); + } + return Ok(Object::Dict(self.builtins.clone())); + } "__defaults__" => { if f.defaults.is_empty() { return Ok(Object::None); @@ -4501,14 +4533,10 @@ impl Interpreter { // function was defined without annotations, // so reads of ``__annotations__`` never raise // ``AttributeError``. Stash it on the - // function's attrs so subsequent writes mutate - // the same dict. - let key = DictKey(Object::from_static("__annotations__")); - if let Some(v) = f.attrs.borrow().get(&key) { - return Ok(v.clone()); - } + // function's slots so subsequent writes mutate + // the same dict (the slot lookup above missed). let d = Object::Dict(Rc::new(RefCell::new(DictData::new()))); - f.attrs.borrow_mut().insert(key, d.clone()); + f.set_slot("__annotations__", d.clone()); return Ok(d); } // PEP 695: every function carries `__type_params__` @@ -4609,15 +4637,17 @@ impl Interpreter { Object::BoundMethod(bm) => match name { "__func__" => Ok(bm.function.clone()), "__self__" => Ok(bm.receiver.clone()), - "__name__" => match &bm.function { - Object::Function(f) => Ok(Object::from_str(f.name.clone())), + // `method.__name__`/`__doc__` delegate to `__func__` so a + // `functools.wraps`-patched attribute (stored in the + // function's attrs dict) wins over the compile-time name — + // `classmethod(contextmanager(f))` relies on this. + "__name__" | "__qualname__" => match &bm.function { + Object::Function(_) => self.load_attr(&bm.function, name), Object::Builtin(b) => Ok(Object::from_static(builtin_display_name(b.name))), _ => Ok(Object::from_static("?")), }, "__doc__" => match &bm.function { - Object::Function(f) => { - Ok(crate::builtins::code_docstring(&f.code).unwrap_or(Object::None)) - } + Object::Function(_) => self.load_attr(&bm.function, name), Object::Builtin(b) => Ok(builtin_doc(b.name) .map(Object::from_static) .unwrap_or(Object::None)), @@ -4825,11 +4855,22 @@ impl Interpreter { // `super.__getattribute__` passes `su.__obj_type__` — the // class that originally triggered super — as the `owner` // argument to the descriptor protocol; we mirror that here. - let super_receiver = inst + // A proxy is identified by the `__obj_type__` key, which *only* + // `make_super` writes — a plain object that merely carries a + // `__self__` attribute (`partialmethod.__get__` writes one onto + // its bound `partial`) must stay on the normal path. + let super_receiver = if inst .dict .borrow() - .get(&DictKey(Object::from_static("__self__"))) - .cloned(); + .contains_key(&DictKey(Object::from_static("__obj_type__"))) + { + inst.dict + .borrow() + .get(&DictKey(Object::from_static("__self__"))) + .cloned() + } else { + None + }; if name != "__self__" { if let Some(receiver) = super_receiver { if let Some(v) = inst.cls().lookup(name) { @@ -4907,7 +4948,18 @@ impl Interpreter { // user's instance dict but Python code (e.g. // `functools.cached_property`) reaches for them anyway. match name { - "__dict__" => return Ok(Object::Dict(inst.dict.clone())), + "__dict__" => { + // A pure-`__slots__` class has no instance `__dict__` at + // all (CPython raises AttributeError; `cached_property` + // keys its slots-unsupported diagnostic off this). + if inst.cls().forbids_dict { + return Err(attribute_error(format!( + "'{}' object has no attribute '__dict__'", + inst.cls().name + ))); + } + return Ok(Object::Dict(inst.dict.clone())); + } "__class__" => return Ok(Object::Type(inst.cls())), _ => {} } @@ -5042,7 +5094,11 @@ impl Interpreter { return Ok(Object::new_tuple(mro)); } "__class__" => return Ok(Object::Type(meta)), - "__dict__" => return Ok(Object::Dict(ty.dict.clone())), + // CPython: `type.__dict__` is a read-only `mappingproxy` + // (direct `cls.__dict__[k] = v` raises TypeError; mutation + // must go through `setattr`). The proxy *shares* the dict, so + // reads stay live. + "__dict__" => return Ok(Object::MappingProxy(ty.dict.clone())), "__flags__" => return Ok(Object::Int(ty.flags_bits())), "__subclasses__" => { // `type.__subclasses__` is a bound method; the actual @@ -5200,16 +5256,13 @@ impl Interpreter { }))), Object::SlotDescriptor(slot) => match instance { Object::None => Ok(attr.clone()), - Object::Instance(inst) => { - let key = DictKey(Object::from_str(&slot.name)); - match inst.dict.borrow().get(&key) { - Some(v) => Ok(v.clone()), - None => Err(attribute_error(format!( - "'{}' object has no attribute '{}'", - inst.cls().name, slot.name - ))), - } - } + Object::Instance(inst) => match inst.slot_get(&slot.name) { + Some(v) => Ok(v), + None => Err(attribute_error(format!( + "'{}' object has no attribute '{}'", + inst.cls().name, slot.name + ))), + }, _ => Err(type_error("slot descriptor requires an instance")), }, Object::Function(_) | Object::Builtin(_) => { @@ -6491,6 +6544,14 @@ impl Interpreter { v: &Object, globals: &Rc>, ) -> Result { + // `tuple(t)` on an exact tuple returns `t` itself (CPython reuses + // the immutable object — `copy.copy(partial).args is partial.args` + // depends on it). + if name == "tuple" { + if let Object::Tuple(_) = v { + return Ok(v.clone()); + } + } let collected = self.collect_iterable(v, globals)?; if name == "list" { Ok(Object::new_list(collected)) @@ -7519,19 +7580,43 @@ impl Interpreter { reverse: bool, globals: &Rc>, ) -> Result<(), RuntimeError> { + // CPython's `reverse=True` is *tie-stable*: equal elements keep + // their original relative order (list.sort reverses the slice + // before and after sorting). A post-sort `.reverse()` alone would + // flip ties — observable in `heapq.nlargest`/`Counter.most_common`. if let Some(f) = key_fn { let mut decorated: Vec<(Object, Object)> = Vec::with_capacity(items.len()); for item in items.iter() { let k = self.call(f, std::slice::from_ref(item), &[], globals)?; decorated.push((k, item.clone())); } - decorated.sort_by(|a, b| a.0.cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); + if reverse { + decorated.reverse(); + } + if decorated.iter().any(|(k, _)| sort_key_needs_dunder_lt(k)) { + decorated = merge_sort_by_pylt(self, decorated, &|p: &(Object, Object)| &p.0, globals)?; + } else { + decorated.sort_by(|a, b| a.0.cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); + } if reverse { decorated.reverse(); } *items = decorated.into_iter().map(|(_, v)| v).collect(); } else { - items.sort_by(|a, b| a.cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + if reverse { + items.reverse(); + } + if items.iter().any(sort_key_needs_dunder_lt) { + let sorted = merge_sort_by_pylt( + self, + std::mem::take(items), + &|o: &Object| o, + globals, + )?; + *items = sorted; + } else { + items.sort_by(|a, b| a.cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + } if reverse { items.reverse(); } @@ -9540,6 +9625,20 @@ impl Interpreter { name: &str, globals: &Rc>, ) -> Option { + // Comparing *classes* dispatches through the metaclass's rich + // comparisons (CPython: `A < B` with `Meta.__lt__` — e.g. a + // `@total_ordering`-decorated metaclass). Only a real user + // metaclass method counts; `type` itself supplies none. + if let Object::Type(ty) = obj { + let meta = ty.metaclass_or_type(); + if let Some(m @ (Object::Function(_) | Object::BoundMethod(_))) = meta.lookup(name) { + return Some(Object::BoundMethod(Rc::new(BoundMethod { + receiver: obj.clone(), + function: m, + }))); + } + return None; + } let inst = match obj { Object::Instance(i) => i.clone(), _ => return None, @@ -10604,6 +10703,13 @@ impl Interpreter { fn store_attr(&mut self, obj: &Object, name: &str, value: Object) -> Result<(), RuntimeError> { match obj { Object::Instance(inst) => self.store_attr_instance(inst, obj, name, value), + // `property.__doc__` is a writable member in CPython + // (`prop.__doc__ = "…"`); the accessor triple stays + // immutable (replaced via `.getter/.setter/.deleter`). + Object::Property(p) if name == "__doc__" => { + *p.doc.borrow_mut() = value; + Ok(()) + } Object::Type(ty) => { if ty.flags.is_builtin { return Err(type_error(format!( @@ -10646,9 +10752,49 @@ impl Interpreter { Ok(()) } Object::Function(f) => { - f.attrs - .borrow_mut() - .insert(DictKey(Object::from_str(name)), value); + // CPython's function getsets validate the assigned shape + // before accepting it. + match name { + "__defaults__" + if !matches!(value, Object::Tuple(_) | Object::None) => + { + return Err(type_error("__defaults__ must be set to a tuple object")); + } + "__kwdefaults__" + if !matches!(value, Object::Dict(_) | Object::None) => + { + return Err(type_error( + "__kwdefaults__ must be set to a dict object", + )); + } + "__name__" | "__qualname__" if !matches!(value, Object::Str(_)) => { + return Err(type_error(format!( + "{name} must be set to a string object" + ))); + } + "__annotations__" + if !matches!(value, Object::Dict(_) | Object::None) => + { + return Err(type_error( + "__annotations__ must be set to a dict object", + )); + } + "__type_params__" if !matches!(value, Object::Tuple(_)) => { + return Err(type_error( + "__type_params__ must be set to a tuple object", + )); + } + _ => {} + } + // Slot dunders are data descriptors: assignment lands in + // the slot store, never `__dict__`. + if crate::object::is_function_slot(name) { + f.set_slot(name, value); + } else { + f.attrs + .borrow_mut() + .insert(DictKey(Object::from_str(name)), value); + } Ok(()) } Object::SimpleNamespace(d) => { @@ -10891,9 +11037,7 @@ impl Interpreter { return Ok(()); } Object::SlotDescriptor(_) => { - inst.dict - .borrow_mut() - .insert(DictKey(Object::from_str(name)), value); + inst.slot_set(name, value); return Ok(()); } Object::Instance(descriptor_inst) => { @@ -11008,6 +11152,33 @@ impl Interpreter { ))) } } + // `del f.attr` removes a function attribute from `__dict__` + // (CPython raises AttributeError — not TypeError — when + // absent; `functools.update_wrapper` probes with delete). + Object::Function(f) => { + if crate::object::is_function_slot(name) { + // CPython allows deleting the nullable slots + // (`__doc__`, `__annotations__`, …): the value + // resets so the computed default resurfaces. + f.slots + .borrow_mut() + .shift_remove(&DictKey(Object::from_str(name))); + return Ok(()); + } + let removed = f + .attrs + .borrow_mut() + .shift_remove(&DictKey(Object::from_str(name))) + .is_some(); + if removed { + Ok(()) + } else { + Err(attribute_error(format!( + "'function' object has no attribute '{}'", + name + ))) + } + } _ => Err(type_error(format!( "'{}' object has no attribute '{}'", obj.type_name(), @@ -11054,6 +11225,16 @@ impl Interpreter { )?; return Ok(()); } + Object::SlotDescriptor(slot) => { + if inst.slot_del(&slot.name) { + return Ok(()); + } + return Err(attribute_error(format!( + "'{}' object has no attribute '{}'", + inst.cls().name, + slot.name + ))); + } Object::Instance(descriptor_inst) => { if let Some(deleter) = descriptor_inst.cls().lookup("__delete__") { let bound = Object::BoundMethod(Rc::new(BoundMethod { @@ -12537,6 +12718,7 @@ impl Interpreter { kw_defaults: vec![], closure, attrs: Rc::new(RefCell::new(DictData::new())), + slots: RefCell::new(DictData::new()), }))) } @@ -13665,6 +13847,23 @@ impl Interpreter { "classmethod" => { return builtins::construct_classmethod(args); } + // `types.GenericAlias(origin, args)` — also reachable as + // `__class_getitem__ = classmethod(GenericAlias)` in pure- + // Python classes following CPython's own idiom (functools). + "GenericAlias" => { + if args.len() != 2 { + return Err(type_error(format!( + "GenericAlias expected 2 arguments, got {}", + args.len() + ))); + } + return Ok(make_generic_alias(args[0].clone(), args[1].clone())); + } + "UnionType" => { + return Err(type_error( + "cannot create 'types.UnionType' instances".to_owned(), + )); + } // `types.MethodType(func, obj)` — bind `func` to `obj`, // producing a callable bound method (CPython `method`). "method" => { @@ -14249,10 +14448,23 @@ impl Interpreter { // Defaults plug remaining holes among positional args. CPython // attaches positional defaults right-aligned to the param // list (so `def f(a, b=1, c=2)` has `defaults = (1, 2)`). + // A user-assigned `__defaults__` (stored on the function's slot + // store — `namedtuple` relies on `__new__.__defaults__ = …`) + // replaces the compiled tuple wholesale; `None` clears it. if filled.iter().take(total_args).any(|x| !x) { + let def_override = f.slot("__defaults__"); + let overridden: Option> = match def_override { + Some(Object::Tuple(t)) => Some(t.iter().cloned().collect()), + Some(Object::None) => Some(Vec::new()), + _ => None, + }; + let defaults: &[Object] = match &overridden { + Some(v) => v, + None => &f.defaults, + }; let needed = total_args; - let first_default = needed.saturating_sub(f.defaults.len()); - for (i, d) in f.defaults.iter().enumerate() { + let first_default = needed.saturating_sub(defaults.len()); + for (i, d) in defaults.iter().enumerate() { let slot = first_default + i; if slot < needed && !filled[slot] { positional[slot] = d.clone(); @@ -14262,18 +14474,14 @@ impl Interpreter { } // Then plug kwonly defaults by name. Guarded on `kwonly_count` // so the overwhelmingly common no-keyword-only call skips this - // entirely (no per-call attrs probe). A user-assigned - // `__kwdefaults__` (stored on the function's attrs dict) replaces + // entirely (no per-call slot probe). A user-assigned + // `__kwdefaults__` (stored on the function's slot store) replaces // the compiled set wholesale — CPython's `func.__kwdefaults__ = // {...}` makes any keyword-only name absent from the new mapping // required again. Only the override path allocates; otherwise we // borrow the compiled `kw_defaults` directly. if kwonly_count > 0 { - let kwd_override = f - .attrs - .borrow() - .get(&DictKey(Object::from_static("__kwdefaults__"))) - .cloned(); + let kwd_override = f.slot("__kwdefaults__"); let overridden: Option> = match kwd_override { Some(Object::Dict(d)) => Some( d.borrow() @@ -14388,10 +14596,10 @@ impl Interpreter { }; // CPython snapshots the *function's* current // `__name__`/`__qualname__` (which user code may have - // reassigned; overrides live in `f.attrs`) into + // reassigned; overrides live in `f.slots`) into // `gi_name`/`gi_qualname` at call time. let attr_str = |attr: &'static str| -> Option { - match f.attrs.borrow().get(&DictKey(Object::from_static(attr))) { + match f.slot(attr) { Some(Object::Str(s)) => Some(s.to_string()), _ => None, } @@ -15766,7 +15974,7 @@ fn resolve_slice_ints(s: &PySlice) -> Result { }) } -fn apply_slice_assignment( +pub(crate) fn apply_slice_assignment( data: &mut Vec, s: &PySlice, replacement: Vec, @@ -15836,7 +16044,7 @@ fn apply_slice_assignment( /// Compute the concrete indices covered by `s` over a sequence of /// length `len` (CPython's `PySlice_Unpack` + `PySlice_AdjustIndices`), /// returned in iteration order. -fn slice_indices(len: usize, s: &PySlice) -> Result, RuntimeError> { +pub(crate) fn slice_indices(len: usize, s: &PySlice) -> Result, RuntimeError> { let len = len as i64; let step = match &s.step { Object::None => 1i64, @@ -16519,6 +16727,77 @@ fn exception_value(instance: &Object) -> Object { Object::None } +/// True when sorting must dispatch `__lt__` through the interpreter: +/// the sort key is a user instance whose class defines a real Python +/// `__lt__` (`functools.cmp_to_key`'s `K`, rich-comparable dataclasses, +/// …). Everything else keeps the native `Object::cmp` fast path. +fn sort_key_needs_dunder_lt(o: &Object) -> bool { + matches!( + o, + Object::Instance(inst) + if matches!( + inst.cls().lookup("__lt__"), + Some(Object::Function(_) | Object::BoundMethod(_)) + ) + ) +} + +/// Stable merge sort that orders by Python `<` (full rich-comparison +/// dispatch, reflected operands included). `key` projects the comparison +/// object out of each element (the decorated sort key, or the element +/// itself). Comparison errors (unorderable types, raising `__lt__`) +/// propagate exactly as CPython's `list.sort` does. +fn merge_sort_by_pylt( + interp: &mut Interpreter, + mut v: Vec, + key: &impl Fn(&T) -> &Object, + globals: &Rc>, +) -> Result, RuntimeError> { + if v.len() <= 1 { + return Ok(v); + } + let right = v.split_off(v.len() / 2); + let left = merge_sort_by_pylt(interp, v, key, globals)?; + let right = merge_sort_by_pylt(interp, right, key, globals)?; + let mut out = Vec::with_capacity(left.len() + right.len()); + let (mut li, mut ri) = (0, 0); + while li < left.len() && ri < right.len() { + // Stability: take from the right run only when strictly smaller + // (timsort's `b < a` merge test). + if interp.dispatch_compare_op( + key(&right[ri]), + key(&left[li]), + CompareKind::Lt, + globals, + )? { + out.push(right[ri].clone()); + ri += 1; + } else { + out.push(left[li].clone()); + li += 1; + } + } + out.extend_from_slice(&left[li..]); + out.extend_from_slice(&right[ri..]); + Ok(out) +} + +/// Convert a freshly built `set` result into a `frozenset` — used when +/// the left operand of a set operator is frozen (result kind follows the +/// left operand in CPython). +fn freeze_set_result(o: Object) -> Object { + match o { + Object::Set(s) => { + let data = match Rc::try_unwrap(s) { + Ok(cell) => cell.into_inner(), + Err(rc) => rc.borrow().clone(), + }; + Object::FrozenSet(Rc::new(data)) + } + other => other, + } +} + fn union_sets(a: &crate::object::SetData, b: &crate::object::SetData) -> Object { let mut out = a.clone(); for k in b.iter() { @@ -17624,6 +17903,14 @@ fn builtin_doc(name: &str) -> Option<&'static str> { "throw(typ[,val[,tb]]) -> raise exception in coroutine,\nreturn next iterated value or raise StopIteration.", ), ".cor_close" => Some("close() -> raise GeneratorExit inside coroutine."), + // CPython's signature-style docstrings (`functools.wraps(max)` + // copies one; test_functools asserts the `max(` prefix). + "max" => Some( + "max(iterable, *[, default=obj, key=func]) -> value\nmax(arg1, arg2, *args, *[, key=func]) -> value\n\nWith a single iterable argument, return its biggest item. The\ndefault keyword-only argument specifies an object to return if\nthe provided iterable is empty.\nWith two or more positional arguments, return the largest argument.", + ), + "min" => Some( + "min(iterable, *[, default=obj, key=func]) -> value\nmin(arg1, arg2, *args, *[, key=func]) -> value\n\nWith a single iterable argument, return its smallest item. The\ndefault keyword-only argument specifies an object to return if\nthe provided iterable is empty.\nWith two or more positional arguments, return the smallest argument.", + ), _ => None, } } @@ -18917,10 +19204,34 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result Ok(union_sets(&a.borrow(), &b.borrow())), - (O::Set(a), O::Set(b), B::BitAnd) => Ok(intersect_sets(&a.borrow(), &b.borrow())), - (O::Set(a), O::Set(b), B::Sub) => Ok(difference_sets(&a.borrow(), &b.borrow())), - (O::Set(a), O::Set(b), B::BitXor) => Ok(symmetric_diff_sets(&a.borrow(), &b.borrow())), + // Set operators accept any mix of `set`/`frozenset` operands; the + // result kind follows the *left* operand (CPython's `set_and` & + // co. use `PyAnySet_Check` on the other operand and build a result + // of `Py_TYPE(self)`). + (O::Set(x), O::Set(y), B::BitOr) => Ok(union_sets(&x.borrow(), &y.borrow())), + (O::Set(x), O::FrozenSet(y), B::BitOr) => Ok(union_sets(&x.borrow(), y)), + (O::FrozenSet(x), O::Set(y), B::BitOr) => Ok(freeze_set_result(union_sets(x, &y.borrow()))), + (O::FrozenSet(x), O::FrozenSet(y), B::BitOr) => Ok(freeze_set_result(union_sets(x, y))), + (O::Set(x), O::Set(y), B::BitAnd) => Ok(intersect_sets(&x.borrow(), &y.borrow())), + (O::Set(x), O::FrozenSet(y), B::BitAnd) => Ok(intersect_sets(&x.borrow(), y)), + (O::FrozenSet(x), O::Set(y), B::BitAnd) => { + Ok(freeze_set_result(intersect_sets(x, &y.borrow()))) + } + (O::FrozenSet(x), O::FrozenSet(y), B::BitAnd) => Ok(freeze_set_result(intersect_sets(x, y))), + (O::Set(x), O::Set(y), B::Sub) => Ok(difference_sets(&x.borrow(), &y.borrow())), + (O::Set(x), O::FrozenSet(y), B::Sub) => Ok(difference_sets(&x.borrow(), y)), + (O::FrozenSet(x), O::Set(y), B::Sub) => { + Ok(freeze_set_result(difference_sets(x, &y.borrow()))) + } + (O::FrozenSet(x), O::FrozenSet(y), B::Sub) => Ok(freeze_set_result(difference_sets(x, y))), + (O::Set(x), O::Set(y), B::BitXor) => Ok(symmetric_diff_sets(&x.borrow(), &y.borrow())), + (O::Set(x), O::FrozenSet(y), B::BitXor) => Ok(symmetric_diff_sets(&x.borrow(), y)), + (O::FrozenSet(x), O::Set(y), B::BitXor) => { + Ok(freeze_set_result(symmetric_diff_sets(x, &y.borrow()))) + } + (O::FrozenSet(x), O::FrozenSet(y), B::BitXor) => { + Ok(freeze_set_result(symmetric_diff_sets(x, y))) + } // PEP 584 — `dict | dict` merges left-to-right into a new dict. (O::Dict(x), O::Dict(y), B::BitOr) => { @@ -18930,10 +19241,6 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result Ok(union_sets(a, b)), - (O::FrozenSet(a), O::FrozenSet(b), B::BitAnd) => Ok(intersect_sets(a, b)), - (O::FrozenSet(a), O::FrozenSet(b), B::Sub) => Ok(difference_sets(a, b)), - (O::FrozenSet(a), O::FrozenSet(b), B::BitXor) => Ok(symmetric_diff_sets(a, b)), (O::List(x), O::List(y), B::Add) => { let mut out = x.borrow().clone(); @@ -19115,7 +19422,13 @@ pub fn make_pep604_union(a: &Object, b: &Object) -> Object { /// keep types as types; keep `None` as `None` (downstream /// `isinstance` recognises both). fn normalize_union_arg(x: Object) -> Object { - x + // `int | None` stores `type(None)` (CPython's `_Py_union_args` never + // contains the bare `None` object; singledispatch's "all arguments + // are classes" check depends on it). + match x { + Object::None => Object::Type(crate::builtin_types::builtin_types().none_type.clone()), + other => other, + } } /// Python `float % float`. Unlike Rust's `%` (C `fmod`, sign of the @@ -19448,14 +19761,35 @@ fn i64_op(x: i64, y: i64, op: BinOpKind) -> Result, RuntimeError> /// `__class_getitem__`. The result is a `SimpleNamespace`-shaped /// object with `__origin__` and `__args__` attributes; `isinstance` /// unwraps it via `__origin__` before walking the MRO. +/// Crate-visible alias builder for [`type_surface`]'s materialized +/// `__class_getitem__` entries. +pub(crate) fn make_generic_alias_public(origin: Object, params: Object) -> Object { + make_generic_alias(origin, params) +} + fn make_generic_alias(origin: Object, params: Object) -> Object { let mut d = DictData::new(); let args_tuple = match ¶ms { Object::Tuple(_) => params.clone(), other => Object::new_tuple(vec![other.clone()]), }; + // `__parameters__` collects the TypeVar-ish entries of `__args__` + // (objects exposing `__typing_subst__`/named-TypeVar shape). Plain + // classes yield the empty tuple CPython reports for `list[int]`. + let params_tuple = match &args_tuple { + Object::Tuple(items) => { + let vars: Vec = items + .iter() + .filter(|o| matches!(o, Object::Instance(inst) if inst.cls().name == "TypeVar")) + .cloned() + .collect(); + Object::new_tuple(vars) + } + _ => Object::new_tuple(Vec::new()), + }; d.insert(DictKey(Object::from_static("__origin__")), origin); d.insert(DictKey(Object::from_static("__args__")), args_tuple); + d.insert(DictKey(Object::from_static("__parameters__")), params_tuple); Object::SimpleNamespace(Rc::new(RefCell::new(d))) } @@ -19593,7 +19927,7 @@ fn complex_arith( } } -fn compare_op(a: &Object, b: &Object, op: CompareKind) -> Result { +pub(crate) fn compare_op(a: &Object, b: &Object, op: CompareKind) -> Result { // CPython lifts ``<``, ``<=``, ``>``, ``>=`` to subset/superset // tests on the set family. They are *not* total orderings, so we // intercept this before falling through to ``Object::cmp``. diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index fd1e3d2..7dffc31 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -472,7 +472,10 @@ pub struct PyProperty { pub fget: Object, pub fset: Object, pub fdel: Object, - pub doc: Object, + /// Interior-mutable: CPython's `property.__doc__` is a writable + /// member (`namedtuple` field docs are patched in place: + /// `Point.x.__doc__ = …`). + pub doc: RefCell, } impl PyProperty { @@ -481,10 +484,15 @@ impl PyProperty { fget, fset, fdel, - doc, + doc: RefCell::new(doc), } } + /// Current `__doc__` value. + pub fn doc(&self) -> Object { + self.doc.borrow().clone() + } + /// Return a clone of `self` with the given attribute replaced. Used /// by `property.getter`/`setter`/`deleter` (which CPython models as /// methods that return a *new* property carrying the patched @@ -494,7 +502,7 @@ impl PyProperty { fget: self.fget.clone(), fset: self.fset.clone(), fdel: self.fdel.clone(), - doc: self.doc.clone(), + doc: RefCell::new(self.doc()), }; match which { PropertyAttr::Get => next.fget = fn_, @@ -687,6 +695,48 @@ pub struct PyFunction { /// `__isabstractmethod__`, or any decorator that stashes /// per-callable metadata. pub attrs: Rc>, + /// CPython function *getset/member slots* (`__name__`, + /// `__qualname__`, `__doc__`, `__module__`, `__annotations__`, + /// `__type_params__`, …). These live outside `__dict__`: they're + /// data descriptors on the `function` type, so `f.__name__ = x` + /// must never appear in `f.__dict__` (functools.update_wrapper + /// copies `__dict__` and asserts the wrapper's annotations are + /// untouched by the wrapped function's slots). + pub slots: RefCell, +} + +/// Attribute names backed by function slots rather than `__dict__`. +pub fn is_function_slot(name: &str) -> bool { + matches!( + name, + "__name__" + | "__qualname__" + | "__doc__" + | "__module__" + | "__annotations__" + | "__type_params__" + | "__defaults__" + | "__kwdefaults__" + | "__code__" + ) +} + +impl PyFunction { + /// Read a slot value if one has been stored (explicitly assigned or + /// stamped at definition time). Computed fallbacks live at the + /// attribute-access sites. + pub fn slot(&self, name: &str) -> Option { + self.slots + .borrow() + .get(&DictKey(Object::from_str(name))) + .cloned() + } + + pub fn set_slot(&self, name: &str, value: Object) { + self.slots + .borrow_mut() + .insert(DictKey(Object::from_str(name)), value); + } } impl fmt::Debug for PyFunction { @@ -2007,6 +2057,16 @@ impl Object { "a bytes-like object is required for memoryview membership", )), }, + // A built-in-subclass instance (`class C(dict)`, …) contains + // through its wrapped native payload — the receiver-side + // analogue of CPython dispatching `sq_contains` on the base. + Object::Instance(inst) => match &inst.native { + Some(native) => native.contains(item), + None => Err(type_error(format!( + "argument of type '{}' is not iterable", + self.type_name() + ))), + }, _ => Err(type_error(format!( "argument of type '{}' is not iterable", self.type_name() @@ -2289,7 +2349,14 @@ impl Object { } } Object::Function(f) => { - format!("", f.name, Rc::as_ptr(f) as usize) + // CPython shows the *qualname* (with any user override + // via `f.__qualname__ = …` taking priority). + let qual = f + .slot("__qualname__") + .as_ref() + .map(Object::to_str) + .unwrap_or_else(|| f.code.qualname.clone()); + format!("", qual, Rc::as_ptr(f) as usize) } Object::Builtin(b) => format!("", b.name), Object::BoundMethod(_) => "".to_owned(), @@ -2302,7 +2369,7 @@ impl Object { s.step.repr() ), Object::Cell(inner) => format!("", inner.borrow().repr()), - Object::Type(t) => format!("", t.name), + Object::Type(t) => format!("", t.qualified_display_name()), Object::Module(m) => match &m.filename { Some(path) => format!("", m.name, path), None => format!("", m.name), @@ -2409,8 +2476,35 @@ impl Object { format!("{}([{}])", v.kind.type_name(), body.join(", ")) } Object::SimpleNamespace(d) => { - let d = d.borrow(); - let parts: Vec = d + let dict = d.borrow(); + // PEP 585/604 runtime forms repr as type expressions + // (CPython: `repr(list[int])` is "list[int]", `repr(int | + // str)` is "int | str"), not as namespace literals. + let type_param_repr = |o: &Object| -> String { + match o { + Object::Type(t) => t.qualified_display_name(), + Object::None => "None".to_owned(), + other => other.repr(), + } + }; + let args = dict.get(&DictKey(Object::from_static("__args__"))).cloned(); + if dict + .get(&DictKey(Object::from_static("__is_pep604_union__"))) + .is_some() + { + if let Some(Object::Tuple(items)) = &args { + let parts: Vec = items.iter().map(type_param_repr).collect(); + return parts.join(" | "); + } + } + if let (Some(origin), Some(Object::Tuple(items))) = ( + dict.get(&DictKey(Object::from_static("__origin__"))), + &args, + ) { + let parts: Vec = items.iter().map(type_param_repr).collect(); + return format!("{}[{}]", type_param_repr(origin), parts.join(", ")); + } + let parts: Vec = dict .iter() .map(|(k, v)| format!("{}={}", k.0.to_str(), v.repr())) .collect(); @@ -2839,6 +2933,12 @@ pub(crate) fn py_hash_value(obj: &Object) -> Option { Some(if v == -1 { -2 } else { v }) } Object::Instance(inst) => { + // A user-defined `__hash__` outranks the wrapped value's hash — + // e.g. functools' `_HashedSeq(list)` caches its hash precisely so + // the (unhashable) list payload is never consulted. + if instance_has_custom_dunder(obj, "__hash__") { + return current_interp_hash(obj); + } if let Some(native) = &inst.native { // int/str/… subclass instance hashes as the wrapped value. return py_hash_value(native); @@ -3121,6 +3221,15 @@ impl Object { } pub fn new_tuple(items: Vec) -> Self { + if items.is_empty() { + // CPython interns the empty tuple (`() is ()`); + // `functools.update_wrapper` asserts identity on copied + // `__type_params__` and similar empty-tuple attributes. + thread_local! { + static EMPTY_TUPLE: Rc<[Object]> = Rc::from(Vec::new().into_boxed_slice()); + } + return Object::Tuple(EMPTY_TUPLE.with(Clone::clone)); + } Object::Tuple(Rc::from(items.into_boxed_slice())) } diff --git a/crates/weavepy-vm/src/stdlib/ast_mod.rs b/crates/weavepy-vm/src/stdlib/ast_mod.rs index 05688fb..d7a9838 100644 --- a/crates/weavepy-vm/src/stdlib/ast_mod.rs +++ b/crates/weavepy-vm/src/stdlib/ast_mod.rs @@ -202,6 +202,8 @@ impl Builder<'_> { args, body, decorator_list, + returns, + .. } => node( "FunctionDef", vec![ @@ -209,7 +211,10 @@ impl Builder<'_> { ("args", self.arguments(args)), ("body", list_of(body, |x| self.stmt(x))), ("decorator_list", list_of(decorator_list, |x| self.expr(x))), - ("returns", Object::None), + ( + "returns", + returns.as_deref().map_or(Object::None, |r| self.expr(r)), + ), ("type_comment", Object::None), ("type_params", Object::new_list(vec![])), ], @@ -221,6 +226,8 @@ impl Builder<'_> { args, body, decorator_list, + returns, + .. } => node( "AsyncFunctionDef", vec![ @@ -228,7 +235,10 @@ impl Builder<'_> { ("args", self.arguments(args)), ("body", list_of(body, |x| self.stmt(x))), ("decorator_list", list_of(decorator_list, |x| self.expr(x))), - ("returns", Object::None), + ( + "returns", + returns.as_deref().map_or(Object::None, |r| self.expr(r)), + ), ("type_comment", Object::None), ("type_params", Object::new_list(vec![])), ], @@ -241,6 +251,7 @@ impl Builder<'_> { keywords, body, decorator_list, + .. } => node( "ClassDef", vec![ diff --git a/crates/weavepy-vm/src/stdlib/python/heapq.py b/crates/weavepy-vm/src/stdlib/python/heapq.py index 8040d71..2fd9d1f 100644 --- a/crates/weavepy-vm/src/stdlib/python/heapq.py +++ b/crates/weavepy-vm/src/stdlib/python/heapq.py @@ -1,151 +1,603 @@ -"""Min-heap algorithms (`heapq`). +"""Heap queue algorithm (a.k.a. priority queue). -A pure-Python heap. Implements the core CPython API: `heappush`, -`heappop`, `heappushpop`, `heapreplace`, `heapify`, `nlargest`, -`nsmallest`, `merge`. +Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for +all k, counting elements from 0. For the sake of comparison, +non-existing elements are considered to be infinite. The interesting +property of a heap is that a[0] is always its smallest element. + +Usage: + +heap = [] # creates an empty heap +heappush(heap, item) # pushes a new item on the heap +item = heappop(heap) # pops the smallest item from the heap +item = heap[0] # smallest item on the heap without popping it +heapify(x) # transforms list into a heap, in-place, in linear time +item = heappushpop(heap, item) # pushes a new item and then returns + # the smallest item; the heap size is unchanged +item = heapreplace(heap, item) # pops and returns smallest item, and adds + # new item; the heap size is unchanged + +Our API differs from textbook heap algorithms as follows: + +- We use 0-based indexing. This makes the relationship between the + index for a node and the indexes for its children slightly less + obvious, but is more suitable since Python uses 0-based indexing. + +- Our heappop() method returns the smallest item, not the largest. + +These two make it possible to view the heap as a regular Python list +without surprises: heap[0] is the smallest item, and heap.sort() +maintains the heap invariant! +""" + +# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger + +__about__ = """Heap queues + +[explanation by François Pinard] + +Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for +all k, counting elements from 0. For the sake of comparison, +non-existing elements are considered to be infinite. The interesting +property of a heap is that a[0] is always its smallest element. + +The strange invariant above is meant to be an efficient memory +representation for a tournament. The numbers below are `k', not a[k]: + + 0 + + 1 2 + + 3 4 5 6 + + 7 8 9 10 11 12 13 14 + + 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 + + +In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In +a usual binary tournament we see in sports, each cell is the winner +over the two cells it tops, and we can trace the winner down the tree +to see all opponents s/he had. However, in many computer applications +of such tournaments, we do not need to trace the history of a winner. +To be more memory efficient, when a winner is promoted, we try to +replace it by something else at a lower level, and the rule becomes +that a cell and the two cells it tops contain three different items, +but the top cell "wins" over the two topped cells. + +If this heap invariant is protected at all time, index 0 is clearly +the overall winner. The simplest algorithmic way to remove it and +find the "next" winner is to move some loser (let's say cell 30 in the +diagram above) into the 0 position, and then percolate this new 0 down +the tree, exchanging values, until the invariant is re-established. +This is clearly logarithmic on the total number of items in the tree. +By iterating over all items, you get an O(n ln n) sort. + +A nice feature of this sort is that you can efficiently insert new +items while the sort is going on, provided that the inserted items are +not "better" than the last 0'th element you extracted. This is +especially useful in simulation contexts, where the tree holds all +incoming events, and the "win" condition means the smallest scheduled +time. When an event schedule other events for execution, they are +scheduled into the future, so they can easily go into the heap. So, a +heap is a good structure for implementing schedulers (this is what I +used for my MIDI sequencer :-). + +Various structures for implementing schedulers have been extensively +studied, and heaps are good for this, as they are reasonably speedy, +the speed is almost constant, and the worst case is not much different +than the average case. However, there are other representations which +are more efficient overall, yet the worst cases might be terrible. + +Heaps are also very useful in big disk sorts. You most probably all +know that a big sort implies producing "runs" (which are pre-sorted +sequences, which size is usually related to the amount of CPU memory), +followed by a merging passes for these runs, which merging is often +very cleverly organised[1]. It is very important that the initial +sort produces the longest runs possible. Tournaments are a good way +to that. If, using all the memory available to hold a tournament, you +replace and percolate items that happen to fit the current run, you'll +produce runs which are twice the size of the memory for random input, +and much better for input fuzzily ordered. + +Moreover, if you output the 0'th item on disk and get an input which +may not fit in the current tournament (because the value "wins" over +the last output value), it cannot fit in the heap, so the size of the +heap decreases. The freed memory could be cleverly reused immediately +for progressively building a second heap, which grows at exactly the +same rate the first heap is melting. When the first heap completely +vanishes, you switch heaps and start a new run. Clever and quite +effective! + +In a word, heaps are useful memory structures to know. I use them in +a few applications, and I think it is good to keep a `heap' module +around. :-) + +-------------------- +[1] The disk balancing algorithms which are current, nowadays, are +more annoying than clever, and this is a consequence of the seeking +capabilities of the disks. On devices which cannot seek, like big +tape drives, the story was quite different, and one had to be very +clever to ensure (far in advance) that each tape movement will be the +most effective possible (that is, will best participate at +"progressing" the merge). Some tapes were even able to read +backwards, and this was also used to avoid the rewinding time. +Believe me, real good tape sorts were quite spectacular to watch! +From all times, sorting has always been a Great Art! :-) """ +__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', + 'nlargest', 'nsmallest', 'heappushpop'] def heappush(heap, item): - """Push `item` onto `heap`, maintaining the heap invariant.""" + """Push item onto heap, maintaining the heap invariant.""" heap.append(item) - _siftdown(heap, 0, len(heap) - 1) - + _siftdown(heap, 0, len(heap)-1) def heappop(heap): - """Pop the smallest item from `heap`, maintaining the invariant.""" - last = heap.pop() + """Pop the smallest item off the heap, maintaining the heap invariant.""" + lastelt = heap.pop() # raises appropriate IndexError if heap is empty if heap: - ret = heap[0] - heap[0] = last + returnitem = heap[0] + heap[0] = lastelt _siftup(heap, 0) - return ret - return last + return returnitem + return lastelt +def heapreplace(heap, item): + """Pop and return the current smallest value, and add the new item. + + This is more efficient than heappop() followed by heappush(), and can be + more appropriate when using a fixed-size heap. Note that the value + returned may be larger than item! That constrains reasonable uses of + this routine unless written as part of a conditional replacement: + + if item > heap[0]: + item = heapreplace(heap, item) + """ + returnitem = heap[0] # raises appropriate IndexError if heap is empty + heap[0] = item + _siftup(heap, 0) + return returnitem def heappushpop(heap, item): - """Push then pop in one operation — faster than separate calls.""" + """Fast version of a heappush followed by a heappop.""" if heap and heap[0] < item: item, heap[0] = heap[0], item _siftup(heap, 0) return item +def heapify(x): + """Transform list into a heap, in-place, in O(len(x)) time.""" + n = len(x) + # Transform bottom-up. The largest index there's any point to looking at + # is the largest with a child index in-range, so must have 2*i + 1 < n, + # or i < (n-1)/2. If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so + # j-1 is the largest, which is n//2 - 1. If n is odd = 2*j+1, this is + # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1. + for i in reversed(range(n//2)): + _siftup(x, i) + +def _heappop_max(heap): + """Maxheap version of a heappop.""" + lastelt = heap.pop() # raises appropriate IndexError if heap is empty + if heap: + returnitem = heap[0] + heap[0] = lastelt + _siftup_max(heap, 0) + return returnitem + return lastelt -def heapreplace(heap, item): - """Pop and return the smallest, then push `item`.""" - ret = heap[0] +def _heapreplace_max(heap, item): + """Maxheap version of a heappop followed by a heappush.""" + returnitem = heap[0] # raises appropriate IndexError if heap is empty heap[0] = item - _siftup(heap, 0) - return ret + _siftup_max(heap, 0) + return returnitem - -def heapify(x): - """In-place transform `x` into a heap.""" +def _heapify_max(x): + """Transform list into a maxheap, in-place, in O(len(x)) time.""" n = len(x) - for i in reversed(range(n // 2)): - _siftup(x, i) + for i in reversed(range(n//2)): + _siftup_max(x, i) + +# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos +# is the index of a leaf with a possibly out-of-order value. Restore the +# heap invariant. +def _siftdown(heap, startpos, pos): + newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + parentpos = (pos - 1) >> 1 + parent = heap[parentpos] + if newitem < parent: + heap[pos] = parent + pos = parentpos + continue + break + heap[pos] = newitem +# The child indices of heap index pos are already heaps, and we want to make +# a heap at index pos too. We do this by bubbling the smaller child of +# pos up (and so on with that child's children, etc) until hitting a leaf, +# then using _siftdown to move the oddball originally at index pos into place. +# +# We *could* break out of the loop as soon as we find a pos where newitem <= +# both its children, but turns out that's not a good idea, and despite that +# many books write the algorithm that way. During a heap pop, the last array +# element is sifted in, and that tends to be large, so that comparing it +# against values starting from the root usually doesn't pay (= usually doesn't +# get us out of the loop early). See Knuth, Volume 3, where this is +# explained and quantified in an exercise. +# +# Cutting the # of comparisons is important, since these routines have no +# way to extract "the priority" from an array element, so that intelligence +# is likely to be hiding in custom comparison methods, or in array elements +# storing (priority, record) tuples. Comparisons are thus potentially +# expensive. +# +# On random arrays of length 1000, making this change cut the number of +# comparisons made by heapify() a little, and those made by exhaustive +# heappop() a lot, in accord with theory. Here are typical results from 3 +# runs (3 just to demonstrate how small the variance is): +# +# Compares needed by heapify Compares needed by 1000 heappops +# -------------------------- -------------------------------- +# 1837 cut to 1663 14996 cut to 8680 +# 1855 cut to 1659 14966 cut to 8678 +# 1847 cut to 1660 15024 cut to 8703 +# +# Building the heap by using heappush() 1000 times instead required +# 2198, 2148, and 2219 compares: heapify() is more efficient, when +# you can use it. +# +# The total compares needed by list.sort() on the same lists were 8627, +# 8627, and 8632 (this should be compared to the sum of heapify() and +# heappop() compares): list.sort() is (unsurprisingly!) more efficient +# for sorting. -def nlargest(n, iterable, key=None): - """Return the `n` largest items from `iterable`.""" - items = list(iterable) - if key is not None: - items.sort(key=key, reverse=True) - else: - items.sort(reverse=True) - return items[:n] +def _siftup(heap, pos): + endpos = len(heap) + startpos = pos + newitem = heap[pos] + # Bubble up the smaller child until hitting a leaf. + childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of smaller child. + rightpos = childpos + 1 + if rightpos < endpos and not heap[childpos] < heap[rightpos]: + childpos = rightpos + # Move the smaller child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + _siftdown(heap, startpos, pos) +def _siftdown_max(heap, startpos, pos): + 'Maxheap variant of _siftdown' + newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + parentpos = (pos - 1) >> 1 + parent = heap[parentpos] + if parent < newitem: + heap[pos] = parent + pos = parentpos + continue + break + heap[pos] = newitem -def nsmallest(n, iterable, key=None): - """Return the `n` smallest items from `iterable`.""" - items = list(iterable) - if key is not None: - items.sort(key=key) +def _siftup_max(heap, pos): + 'Maxheap variant of _siftup' + endpos = len(heap) + startpos = pos + newitem = heap[pos] + # Bubble up the larger child until hitting a leaf. + childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of larger child. + rightpos = childpos + 1 + if rightpos < endpos and not heap[rightpos] < heap[childpos]: + childpos = rightpos + # Move the larger child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + _siftdown_max(heap, startpos, pos) + +def merge(*iterables, key=None, reverse=False): + '''Merge multiple sorted inputs into a single sorted output. + + Similar to sorted(itertools.chain(*iterables)) but returns a generator, + does not pull the data into memory all at once, and assumes that each of + the input streams is already sorted (smallest to largest). + + >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25])) + [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25] + + If *key* is not None, applies a key function to each element to determine + its sort order. + + >>> list(merge(['dog', 'horse'], ['cat', 'fish', 'kangaroo'], key=len)) + ['dog', 'cat', 'fish', 'horse', 'kangaroo'] + + ''' + + h = [] + h_append = h.append + + if reverse: + _heapify = _heapify_max + _heappop = _heappop_max + _heapreplace = _heapreplace_max + direction = -1 else: - items.sort() - return items[:n] + _heapify = heapify + _heappop = heappop + _heapreplace = heapreplace + direction = 1 + if key is None: + for order, it in enumerate(map(iter, iterables)): + try: + next = it.__next__ + h_append([next(), order * direction, next]) + except StopIteration: + pass + _heapify(h) + while len(h) > 1: + try: + while True: + value, order, next = s = h[0] + yield value + s[0] = next() # raises StopIteration when exhausted + _heapreplace(h, s) # restore heap condition + except StopIteration: + _heappop(h) # remove empty iterator + if h: + # fast case when only a single iterator remains + value, order, next = h[0] + yield value + yield from next.__self__ + return -def merge(*iterables, key=None, reverse=False): - """Merge sorted iterables into a single sorted iterator.""" - its = [iter(it) for it in iterables] - heap = [] - for idx, it in enumerate(its): + for order, it in enumerate(map(iter, iterables)): try: - v = next(it) + next = it.__next__ + value = next() + h_append([key(value), order * direction, value, next]) except StopIteration: - continue - k = key(v) if key is not None else v - heap.append((k, idx, v)) - if reverse: - heap.sort(reverse=True) - else: - heap.sort() - while heap: - k, idx, v = heap.pop(0) - yield v + pass + _heapify(h) + while len(h) > 1: try: - v = next(its[idx]) + while True: + key_value, order, value, next = s = h[0] + yield value + value = next() + s[0] = key(value) + s[2] = value + _heapreplace(h, s) except StopIteration: - continue - k = key(v) if key is not None else v - new = (k, idx, v) - # Insert in sorted order. - lo, hi = 0, len(heap) - while lo < hi: - mid = (lo + hi) // 2 - if reverse: - if heap[mid] < new: - hi = mid - else: - lo = mid + 1 - else: - if heap[mid] < new: - lo = mid + 1 - else: - hi = mid - heap.insert(lo, new) - - -# ---- internal ----------------------------------------------------- - - -def _siftdown(heap, start, pos): - item = heap[pos] - while pos > start: - parent_pos = (pos - 1) >> 1 - parent = heap[parent_pos] - if item < parent: - heap[pos] = parent - pos = parent_pos - else: - break - heap[pos] = item + _heappop(h) + if h: + key_value, order, value, next = h[0] + yield value + yield from next.__self__ -def _siftup(heap, pos): - end = len(heap) - start = pos - item = heap[pos] - child = 2 * pos + 1 - while child < end: - right = child + 1 - if right < end and not heap[child] < heap[right]: - child = right - heap[pos] = heap[child] - pos = child - child = 2 * pos + 1 - heap[pos] = item - _siftdown(heap, start, pos) - - -__all__ = [ - "heappush", - "heappop", - "heappushpop", - "heapreplace", - "heapify", - "nlargest", - "nsmallest", - "merge", -] +# Algorithm notes for nlargest() and nsmallest() +# ============================================== +# +# Make a single pass over the data while keeping the k most extreme values +# in a heap. Memory consumption is limited to keeping k values in a list. +# +# Measured performance for random inputs: +# +# number of comparisons +# n inputs k-extreme values (average of 5 trials) % more than min() +# ------------- ---------------- --------------------- ----------------- +# 1,000 100 3,317 231.7% +# 10,000 100 14,046 40.5% +# 100,000 100 105,749 5.7% +# 1,000,000 100 1,007,751 0.8% +# 10,000,000 100 10,009,401 0.1% +# +# Theoretical number of comparisons for k smallest of n random inputs: +# +# Step Comparisons Action +# ---- -------------------------- --------------------------- +# 1 1.66 * k heapify the first k-inputs +# 2 n - k compare remaining elements to top of heap +# 3 k * (1 + lg2(k)) * ln(n/k) replace the topmost value on the heap +# 4 k * lg2(k) - (k/2) final sort of the k most extreme values +# +# Combining and simplifying for a rough estimate gives: +# +# comparisons = n + k * (log(k, 2) * log(n/k) + log(k, 2) + log(n/k)) +# +# Computing the number of comparisons for step 3: +# ----------------------------------------------- +# * For the i-th new value from the iterable, the probability of being in the +# k most extreme values is k/i. For example, the probability of the 101st +# value seen being in the 100 most extreme values is 100/101. +# * If the value is a new extreme value, the cost of inserting it into the +# heap is 1 + log(k, 2). +# * The probability times the cost gives: +# (k/i) * (1 + log(k, 2)) +# * Summing across the remaining n-k elements gives: +# sum((k/i) * (1 + log(k, 2)) for i in range(k+1, n+1)) +# * This reduces to: +# (H(n) - H(k)) * k * (1 + log(k, 2)) +# * Where H(n) is the n-th harmonic number estimated by: +# gamma = 0.5772156649 +# H(n) = log(n, e) + gamma + 1 / (2 * n) +# http://en.wikipedia.org/wiki/Harmonic_series_(mathematics)#Rate_of_divergence +# * Substituting the H(n) formula: +# comparisons = k * (1 + log(k, 2)) * (log(n/k, e) + (1/n - 1/k) / 2) +# +# Worst-case for step 3: +# ---------------------- +# In the worst case, the input data is reversed sorted so that every new element +# must be inserted in the heap: +# +# comparisons = 1.66 * k + log(k, 2) * (n - k) +# +# Alternative Algorithms +# ---------------------- +# Other algorithms were not used because they: +# 1) Took much more auxiliary memory, +# 2) Made multiple passes over the data. +# 3) Made more comparisons in common cases (small k, large n, semi-random input). +# See the more detailed comparison of approach at: +# http://code.activestate.com/recipes/577573-compare-algorithms-for-heapqsmallest + +def nsmallest(n, iterable, key=None): + """Find the n smallest elements in a dataset. + + Equivalent to: sorted(iterable, key=key)[:n] + """ + + # Short-cut for n==1 is to use min() + if n == 1: + it = iter(iterable) + sentinel = object() + result = min(it, default=sentinel, key=key) + return [] if result is sentinel else [result] + + # When n>=size, it's faster to use sorted() + try: + size = len(iterable) + except (TypeError, AttributeError): + pass + else: + if n >= size: + return sorted(iterable, key=key)[:n] + + # When key is none, use simpler decoration + if key is None: + it = iter(iterable) + # put the range(n) first so that zip() doesn't + # consume one too many elements from the iterator + result = [(elem, i) for i, elem in zip(range(n), it)] + if not result: + return result + _heapify_max(result) + top = result[0][0] + order = n + _heapreplace = _heapreplace_max + for elem in it: + if elem < top: + _heapreplace(result, (elem, order)) + top, _order = result[0] + order += 1 + result.sort() + return [elem for (elem, order) in result] + + # General case, slowest method + it = iter(iterable) + result = [(key(elem), i, elem) for i, elem in zip(range(n), it)] + if not result: + return result + _heapify_max(result) + top = result[0][0] + order = n + _heapreplace = _heapreplace_max + for elem in it: + k = key(elem) + if k < top: + _heapreplace(result, (k, order, elem)) + top, _order, _elem = result[0] + order += 1 + result.sort() + return [elem for (k, order, elem) in result] + +def nlargest(n, iterable, key=None): + """Find the n largest elements in a dataset. + + Equivalent to: sorted(iterable, key=key, reverse=True)[:n] + """ + + # Short-cut for n==1 is to use max() + if n == 1: + it = iter(iterable) + sentinel = object() + result = max(it, default=sentinel, key=key) + return [] if result is sentinel else [result] + + # When n>=size, it's faster to use sorted() + try: + size = len(iterable) + except (TypeError, AttributeError): + pass + else: + if n >= size: + return sorted(iterable, key=key, reverse=True)[:n] + + # When key is none, use simpler decoration + if key is None: + it = iter(iterable) + result = [(elem, i) for i, elem in zip(range(0, -n, -1), it)] + if not result: + return result + heapify(result) + top = result[0][0] + order = -n + _heapreplace = heapreplace + for elem in it: + if top < elem: + _heapreplace(result, (elem, order)) + top, _order = result[0] + order -= 1 + result.sort(reverse=True) + return [elem for (elem, order) in result] + + # General case, slowest method + it = iter(iterable) + result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)] + if not result: + return result + heapify(result) + top = result[0][0] + order = -n + _heapreplace = heapreplace + for elem in it: + k = key(elem) + if top < k: + _heapreplace(result, (k, order, elem)) + top, _order, _elem = result[0] + order -= 1 + result.sort(reverse=True) + return [elem for (k, order, elem) in result] + +# If available, use C implementation +try: + from _heapq import * +except ImportError: + pass +try: + from _heapq import _heapreplace_max +except ImportError: + pass +try: + from _heapq import _heapify_max +except ImportError: + pass +try: + from _heapq import _heappop_max +except ImportError: + pass + + +if __name__ == "__main__": + + import doctest # pragma: no cover + print(doctest.testmod()) # pragma: no cover diff --git a/crates/weavepy-vm/src/stdlib/python/pickle.py b/crates/weavepy-vm/src/stdlib/python/pickle.py index 8afc822..6c16d2d 100644 --- a/crates/weavepy-vm/src/stdlib/python/pickle.py +++ b/crates/weavepy-vm/src/stdlib/python/pickle.py @@ -93,6 +93,10 @@ class UnpicklingError(PickleError): def dumps(obj, protocol=None, *, fix_imports=True, buffer_callback=None): if protocol is None: protocol = DEFAULT_PROTOCOL + # CPython `Pickler.__init__`: a negative protocol selects + # HIGHEST_PROTOCOL. + if protocol < 0: + protocol = HIGHEST_PROTOCOL if not 0 <= protocol <= HIGHEST_PROTOCOL: raise ValueError("unsupported pickle protocol: %d" % protocol) pickler = _Pickler(io.BytesIO(), protocol) @@ -137,6 +141,10 @@ def _resolves_to_self(module, qualname, obj): class _Pickler: def __init__(self, buf, protocol): self._buf = buf + if protocol is None: + protocol = DEFAULT_PROTOCOL + elif protocol < 0: + protocol = HIGHEST_PROTOCOL self.protocol = protocol self.bin = protocol >= 1 self.fast = False @@ -745,9 +753,18 @@ def _build(u): setstate = getattr(obj, "__setstate__", None) if setstate is not None: setstate(state) - elif isinstance(state, dict): + return + # CPython `load_build`: a 2-tuple state is (__dict__ state, slot + # state); apply the dict half directly and the slots via setattr. + slotstate = None + if isinstance(state, tuple) and len(state) == 2: + state, slotstate = state + if state: for k, v in state.items(): setattr(obj, k, v) + if slotstate: + for k, v in slotstate.items(): + setattr(obj, k, v) def _newobj(u): diff --git a/crates/weavepy-vm/src/stdlib/python/test_support_import_helper.py b/crates/weavepy-vm/src/stdlib/python/test_support_import_helper.py index 4373e38..0e2e538 100644 --- a/crates/weavepy-vm/src/stdlib/python/test_support_import_helper.py +++ b/crates/weavepy-vm/src/stdlib/python/test_support_import_helper.py @@ -88,12 +88,13 @@ def find_spec(self, fullname, path=None, target=None): if blocker is not None: sys.meta_path.insert(0, blocker) try: - for modname in fresh: - try: - importlib.import_module(modname) - except ImportError: - pass + # CPython contract: if any *fresh* module can't be imported the + # whole call answers None — `import_fresh_module('functools', + # fresh=['_functools'])` is how test files probe for a missing + # C accelerator (the C-variant test classes then skip). try: + for modname in fresh: + importlib.import_module(modname) mod = importlib.import_module(name) except ImportError: return None diff --git a/crates/weavepy-vm/src/stdlib/python/test_support_init.py b/crates/weavepy-vm/src/stdlib/python/test_support_init.py index e8cdd53..c40f079 100644 --- a/crates/weavepy-vm/src/stdlib/python/test_support_init.py +++ b/crates/weavepy-vm/src/stdlib/python/test_support_init.py @@ -266,8 +266,14 @@ def cpython_only(test): # docstrings, so this is False. MISSING_C_DOCSTRINGS = False -# True when the build keeps docstrings (it does). -HAVE_DOCSTRINGS = True + +def _check_docstrings(): + """Just used to check if docstrings are enabled""" + + +# Probe-derived, exactly as CPython's test.support computes them. +HAVE_PY_DOCSTRINGS = _check_docstrings.__doc__ is not None +HAVE_DOCSTRINGS = (HAVE_PY_DOCSTRINGS and not MISSING_C_DOCSTRINGS) def requires_docstrings(func): @@ -486,6 +492,14 @@ def check__all__(test_case, module, name_of_module=None, extra=(), test_case.assertCountEqual(module.__all__, expected) +def setswitchinterval(interval): + """Set the bytecode switch interval, clamped to a sane minimum + (CPython clamps harder on Android; we keep the plain floor).""" + minimum_interval = 1e-9 + interval = max(interval, minimum_interval) + sys.setswitchinterval(interval) + + # --------------------------------------------------------------------------- # GC helpers # --------------------------------------------------------------------------- diff --git a/crates/weavepy-vm/src/stdlib/python/typing.py b/crates/weavepy-vm/src/stdlib/python/typing.py index 7ffc923..b7d2aea 100644 --- a/crates/weavepy-vm/src/stdlib/python/typing.py +++ b/crates/weavepy-vm/src/stdlib/python/typing.py @@ -42,6 +42,26 @@ def __repr__(self): def __getitem__(self, params): if not isinstance(params, tuple): params = (params,) + if self._name == "Union": + # CPython normalizes at construction: `None` becomes + # `type(None)`, nested unions flatten, duplicates collapse + # (`Union[str, None]` == `Union[str, NoneType]`; singledispatch + # requires every arg to be a real class). + flat = [] + for p in params: + if p is None: + p = type(None) + if getattr(p, "__origin__", None) is Union: + flat.extend(p.__args__) + else: + flat.append(p) + deduped = [] + for p in flat: + if not any(p is q for q in deduped): + deduped.append(p) + if len(deduped) == 1: + return deduped[0] + params = tuple(deduped) return _GenericAlias(self, params) def __call__(self, *args, **kwargs): @@ -563,6 +583,14 @@ def get_origin(tp): """Return the unsubscripted version of a generic alias.""" if isinstance(tp, _GenericAlias): return tp.__origin__ + # PEP 604 unions (`int | str`) — CPython answers `types.UnionType`. + if getattr(tp, "__is_pep604_union__", False): + import types + return types.UnionType + # PEP 585 aliases (`list[int]`) carry a real `__origin__`. + origin = getattr(tp, "__origin__", None) + if origin is not None and getattr(tp, "__args__", None) is not None: + return origin return None @@ -570,6 +598,12 @@ def get_args(tp): """Return the type arguments of a generic alias.""" if isinstance(tp, _GenericAlias): return tp.__args__ + if getattr(tp, "__is_pep604_union__", False): + return tuple(tp.__args__) + if getattr(tp, "__origin__", None) is not None: + args = getattr(tp, "__args__", None) + if args is not None: + return tuple(args) return () @@ -772,6 +806,35 @@ def _namedtuple_mro_entries(bases): "ChainMap": "ChainMap", } +# Aliases backed by ``collections.abc`` (``typing.Iterable[str]`` and +# friends). Same lazy PEP 562 treatment as the container aliases above. +_LAZY_ABC_ALIASES = { + "AbstractSet": "Set", + "AsyncGenerator": "AsyncGenerator", + "AsyncIterable": "AsyncIterable", + "AsyncIterator": "AsyncIterator", + "Awaitable": "Awaitable", + "ByteString": "ByteString", + "Collection": "Collection", + "Container": "Container", + "Coroutine": "Coroutine", + "Generator": "Generator", + "Hashable": "Hashable", + "ItemsView": "ItemsView", + "Iterable": "Iterable", + "Iterator": "Iterator", + "KeysView": "KeysView", + "Mapping": "Mapping", + "MappingView": "MappingView", + "MutableMapping": "MutableMapping", + "MutableSequence": "MutableSequence", + "MutableSet": "MutableSet", + "Reversible": "Reversible", + "Sequence": "Sequence", + "Sized": "Sized", + "ValuesView": "ValuesView", +} + def __getattr__(name): target = _LAZY_COLLECTION_ALIASES.get(name) @@ -781,4 +844,11 @@ def __getattr__(name): alias = _OriginAlias(name, getattr(collections, target)) globals()[name] = alias return alias + target = _LAZY_ABC_ALIASES.get(name) + if target is not None: + import collections.abc + + alias = _OriginAlias(name, getattr(collections.abc, target)) + globals()[name] = alias + return alias raise AttributeError(f"module 'typing' has no attribute {name!r}") diff --git a/crates/weavepy-vm/src/stdlib/python/warnings.py b/crates/weavepy-vm/src/stdlib/python/warnings.py index 396e8d6..02014b2 100644 --- a/crates/weavepy-vm/src/stdlib/python/warnings.py +++ b/crates/weavepy-vm/src/stdlib/python/warnings.py @@ -303,5 +303,27 @@ def __exit__(self, *exc): return False +_DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" + + +def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): + """Warn that *name* is deprecated or should be removed. + + RuntimeError is raised if *remove* specifies a major/minor tuple older than + the current Python version or the same version but past the alpha. + + The *message* argument is formatted with *name* and *remove* as a Python + version tuple (e.g. (3, 11)). + + """ + remove_formatted = f"{remove[0]}.{remove[1]}" + if (_version[:2] > remove) or (_version[:2] == remove and _version[3] != "alpha"): + msg = f"{name!r} was slated for removal after Python {remove_formatted} alpha" + raise RuntimeError(msg) + else: + msg = message.format(name=name, remove=remove_formatted) + warn(msg, DeprecationWarning, stacklevel=3) + + # Install a sane default filter set on import. simplefilter("default") diff --git a/crates/weavepy-vm/src/stdlib/symtable_mod.rs b/crates/weavepy-vm/src/stdlib/symtable_mod.rs index bdb7855..83a514f 100644 --- a/crates/weavepy-vm/src/stdlib/symtable_mod.rs +++ b/crates/weavepy-vm/src/stdlib/symtable_mod.rs @@ -322,15 +322,22 @@ impl Builder { args, body, decorator_list, + returns, + .. } | S::AsyncFunctionDef { name, args, body, decorator_list, + returns, + .. } => { self.add_def(name, DEF_LOCAL); self.visit_defaults_and_annotations(args, true); + if let Some(r) = returns { + self.visit_expr(r); + } for d in decorator_list { self.visit_expr(d); } @@ -347,6 +354,7 @@ impl Builder { keywords, body, decorator_list, + .. } => { self.add_def(name, DEF_LOCAL); for b in bases { diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs index b8fcbbf..5546a9d 100644 --- a/crates/weavepy-vm/src/stdlib/thread_real.rs +++ b/crates/weavepy-vm/src/stdlib/thread_real.rs @@ -279,6 +279,7 @@ fn make_lock_object(lock: Arc) -> Object { dict, native: None, inline_values: crate::sync::Cell::new(true), + slots: crate::sync::RefCell::new(None), }); Object::Instance(inst) } @@ -380,6 +381,7 @@ fn make_rlock_object(rlock: Arc) -> Object { dict, native: None, inline_values: crate::sync::Cell::new(true), + slots: crate::sync::RefCell::new(None), }); Object::Instance(inst) } diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs index 55bafc5..b42dd28 100644 --- a/crates/weavepy-vm/src/stdlib/weakref_real.rs +++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs @@ -465,6 +465,7 @@ fn make_ref_object(target: Object, callback: Option, kind_tag: u8) -> Ob dict, native: None, inline_values: crate::sync::Cell::new(true), + slots: crate::sync::RefCell::new(None), })) } diff --git a/crates/weavepy-vm/src/type_surface.rs b/crates/weavepy-vm/src/type_surface.rs new file mode 100644 index 0000000..a24341e --- /dev/null +++ b/crates/weavepy-vm/src/type_surface.rs @@ -0,0 +1,1029 @@ +//! Materialized method/dunder surface for the built-in types. +//! +//! CPython stores every method and slot wrapper of a built-in type in +//! that type's `tp_dict`; `vars(list)`, `'__hash__' in bytearray.__dict__` +//! and `_collections_abc._check_methods` all introspect those dicts +//! directly. WeavePy historically synthesized built-in methods *on +//! demand* (variant match tables in `builtins.rs`), which kept the type +//! dicts almost empty and made structural ABC checks +//! (`Hashable`/`Callable`/`Reversible`/`Buffer`, …) misreport. +//! +//! This module fills the built-in type dicts at interpreter start with +//! real entries that delegate to the existing method machinery: +//! +//! - regular methods (`list.append`, `dict.keys`, …) reuse the +//! [`crate::builtins::lookup_method`] tables, wrapped in a shim that +//! unwraps a built-in-subclass receiver to its native payload (the +//! binding CPython's descriptors do via the C `self` slot); +//! - protocol dunders missing from those tables (`set.__sub__`, +//! `dict.__or__`, `list.__reversed__`, `__class_getitem__`, +//! `__buffer__`, …) are implemented here with CPython's exact +//! semantics (strict operand types, `NotImplemented` declines); +//! - unhashable container types get the literal `__hash__ = None` +//! marker CPython stores (what `Hashable`'s subclass hook keys on). +//! +//! Entries are only inserted when absent so the specialized dunders +//! installed by `builtin_types.rs` (`__new__`, `__init__`, exception +//! `__str__`, …) keep priority. + +use crate::sync::Rc; +use crate::sync::RefCell; + +use crate::builtin_types::BuiltinTypes; +use crate::error::{type_error, value_error, RuntimeError}; +use crate::object::{BuiltinFn, DictData, DictKey, Object, PyIterator, PyMemoryView}; +use crate::types::TypeObject; + +/// Entry point: called once per thread when [`BuiltinTypes`] is built. +pub fn install(bt: &BuiltinTypes) { + install_callables(bt); + install_hash_markers(bt); + install_container_protocols(bt); + install_set_operators(bt); + install_dict_operators(bt); + install_class_getitem(bt); + install_buffer_protocol(bt); + install_method_tables(bt); + install_object_compare(bt); + install_value_richcmp(bt); +} + +// --------------------------------------------------------------------------- +// Value-type rich comparisons +// --------------------------------------------------------------------------- + +/// Materialize `tp_richcompare` for the value types that define one in +/// CPython (`'__lt__' in vars(int)` is True there). Each slot is +/// type-strict: a foreign right operand *declines* with +/// `NotImplemented` so the reflected dunder gets its turn — e.g. +/// `(3).__eq__(3.0)` is `NotImplemented` and `3 == 3.0` resolves via +/// `float.__eq__`. +fn install_value_richcmp(bt: &BuiltinTypes) { + use crate::CompareKind; + + fn fam_int(o: &Object) -> bool { + matches!(o, Object::Int(_) | Object::Long(_) | Object::Bool(_)) + } + fn fam_float(o: &Object) -> bool { + matches!( + o, + Object::Int(_) | Object::Long(_) | Object::Bool(_) | Object::Float(_) + ) + } + fn fam_complex(o: &Object) -> bool { + matches!( + o, + Object::Complex(_) | Object::Int(_) | Object::Long(_) | Object::Bool(_) | Object::Float(_) + ) + } + fn fam_str(o: &Object) -> bool { + matches!(o, Object::Str(_)) + } + fn fam_bytes(o: &Object) -> bool { + matches!(o, Object::Bytes(_) | Object::ByteArray(_)) + } + fn fam_tuple(o: &Object) -> bool { + matches!(o, Object::Tuple(_)) + } + fn fam_list(o: &Object) -> bool { + matches!(o, Object::List(_)) + } + fn fam_none(_: &Object) -> bool { + false + } + + fn richcmp_builtin( + name: &'static str, + op: CompareKind, + family: fn(&Object) -> bool, + ) -> Object { + Object::Builtin(Rc::new(BuiltinFn { + name, + call: Box::new(move |args: &[Object]| { + let (a, b) = match args { + [a, b] => (as_native(a), as_native(b)), + _ => { + return Err(type_error(format!( + "{name} expected 2 arguments, got {}", + args.len() + ))) + } + }; + if !family(&a) || !family(&b) { + return Ok(crate::vm_singletons::not_implemented()); + } + match op { + CompareKind::Eq => Ok(Object::Bool(a.eq_value(&b))), + CompareKind::NotEq => Ok(Object::Bool(!a.eq_value(&b))), + _ => match crate::compare_op(&a, &b, op) { + Ok(v) => Ok(Object::Bool(v)), + // An unordered pair declines rather than raising; + // the dispatcher produces the final TypeError. + Err(_) => Ok(crate::vm_singletons::not_implemented()), + }, + } + }), + call_kw: None, + })) + } + + const ORDERED: &[(&str, CompareKind)] = &[ + ("__lt__", CompareKind::Lt), + ("__le__", CompareKind::LtE), + ("__gt__", CompareKind::Gt), + ("__ge__", CompareKind::GtE), + ("__eq__", CompareKind::Eq), + ("__ne__", CompareKind::NotEq), + ]; + let totally_ordered: &[(&Rc, fn(&Object) -> bool)] = &[ + (&bt.int_, fam_int), + (&bt.float_, fam_float), + (&bt.str_, fam_str), + (&bt.bytes_, fam_bytes), + (&bt.bytearray_, fam_bytes), + (&bt.tuple_, fam_tuple), + (&bt.list_, fam_list), + ]; + for (ty, fam) in totally_ordered { + for (name, op) in ORDERED { + insert_if_absent(ty, name, richcmp_builtin(name, *op, *fam)); + } + } + // complex: equality across the numeric tower, ordering always + // declines (CPython's `complex_richcompare` returns NotImplemented + // for Py_LT/…, but the slots still exist in `vars(complex)`). + for (name, op) in ORDERED { + let fam = if matches!(op, CompareKind::Eq | CompareKind::NotEq) { + fam_complex + } else { + fam_none + }; + insert_if_absent(&bt.complex_, name, richcmp_builtin(name, *op, fam)); + } + // dict / range: equality only. + fn fam_dict(o: &Object) -> bool { + matches!(o, Object::Dict(_)) + } + fn fam_range(o: &Object) -> bool { + matches!(o, Object::Range(_)) + } + for (name, op) in &[("__eq__", CompareKind::Eq), ("__ne__", CompareKind::NotEq)] { + insert_if_absent(&bt.dict_, name, richcmp_builtin(name, *op, fam_dict)); + insert_if_absent(&bt.range_, name, richcmp_builtin(name, *op, fam_range)); + } +} + +// --------------------------------------------------------------------------- +// object's rich comparisons +// --------------------------------------------------------------------------- + +/// `object` owns the six rich-comparison dunders in CPython +/// (`'__lt__' in vars(object)` is True and `type.__gt__ is +/// object.__gt__` — `functools.total_ordering`'s root detection does +/// exactly that identity test). Materialize them once so every MRO +/// lookup returns the *same* object. +fn install_object_compare(bt: &BuiltinTypes) { + fn not_implemented(_args: &[Object]) -> Result { + Ok(crate::vm_singletons::not_implemented()) + } + fn obj_eq(args: &[Object]) -> Result { + // Default `__eq__` is identity; non-identical operands *decline* + // (NotImplemented) so the reflected dunder gets its turn. + match args { + [a, b] if a.is_same(b) => Ok(Object::Bool(true)), + _ => Ok(crate::vm_singletons::not_implemented()), + } + } + fn obj_ne(args: &[Object]) -> Result { + match args { + [a, b] if a.is_same(b) => Ok(Object::Bool(false)), + _ => Ok(crate::vm_singletons::not_implemented()), + } + } + insert_if_absent(&bt.object_, "__eq__", builtin("__eq__", obj_eq)); + insert_if_absent(&bt.object_, "__ne__", builtin("__ne__", obj_ne)); + for name in ["__lt__", "__le__", "__gt__", "__ge__"] { + // One shared entry per name; the closure ignores its arguments. + let f = match name { + "__lt__" => builtin("__lt__", not_implemented), + "__le__" => builtin("__le__", not_implemented), + "__gt__" => builtin("__gt__", not_implemented), + _ => builtin("__ge__", not_implemented), + }; + insert_if_absent(&bt.object_, name, f); + } +} + +/// Insert `name` into `ty`'s dict unless already present. +fn insert_if_absent(ty: &Rc, name: &str, value: Object) { + let key = DictKey(Object::from_str(name)); + let mut dict = ty.dict.borrow_mut(); + if !dict.contains_key(&key) { + dict.insert(key, value); + } +} + +/// Replace an `Instance` receiver/operand with its wrapped native +/// payload (`class C(dict)` instances act as their payload for the +/// base type's methods, like CPython's C-level `self`). +fn as_native(o: &Object) -> Object { + if let Object::Instance(inst) = o { + if let Some(native) = &inst.native { + return native.clone(); + } + } + o.clone() +} + +/// Wrap a `lookup_method`-table builtin so an `Instance` receiver +/// (built-in subclass) is unwrapped to its native payload before the +/// underlying implementation runs. +fn unwrap_shim(inner: Rc) -> Object { + let inner_pos = inner.clone(); + let has_kw = inner.call_kw.is_some(); + let mut shim = BuiltinFn { + name: inner.name, + call: Box::new(move |args| { + if let Some(first) = args.first() { + let unwrapped = as_native(first); + if !unwrapped.is_same(first) { + let mut v = args.to_vec(); + v[0] = unwrapped; + return (inner_pos.call)(&v); + } + } + (inner_pos.call)(args) + }), + call_kw: None, + }; + if has_kw { + let inner_kw = inner.clone(); + shim.call_kw = Some(Box::new(move |args, kwargs| { + let kw = inner_kw + .call_kw + .as_ref() + .expect("call_kw checked at shim construction"); + if let Some(first) = args.first() { + let unwrapped = as_native(first); + if !unwrapped.is_same(first) { + let mut v = args.to_vec(); + v[0] = unwrapped; + return kw(&v, kwargs); + } + } + kw(args, kwargs) + })); + } + Object::Builtin(Rc::new(shim)) +} + +fn builtin(name: &'static str, f: fn(&[Object]) -> Result) -> Object { + Object::Builtin(Rc::new(BuiltinFn { + name, + call: Box::new(f), + call_kw: None, + })) +} + +// --------------------------------------------------------------------------- +// callables: `__call__` (what `collections.abc.Callable`'s hook checks) +// --------------------------------------------------------------------------- + +fn install_callables(bt: &BuiltinTypes) { + for ty in [ + &bt.function_, + &bt.builtin_function_, + &bt.method_, + &bt.method_wrapper_, + &bt.type_, + ] { + if let Some(w) = crate::builtins::builtin_type_dunder(&ty.name, "__call__") { + insert_if_absent(ty, "__call__", w); + } + } +} + +// --------------------------------------------------------------------------- +// hashability markers +// --------------------------------------------------------------------------- + +fn obj_hash_builtin(args: &[Object]) -> Result { + let o = args + .first() + .ok_or_else(|| type_error("__hash__() takes exactly one argument (0 given)"))?; + crate::builtins::hash_object(&as_native(o)) +} + +fn install_hash_markers(bt: &BuiltinTypes) { + // CPython stores the literal `None` in the type dict of every + // unhashable built-in; `_check_methods` (Hashable's hook) and + // user-visible `bytearray.__hash__ is None` both rely on it. It also + // makes subclass instances correctly unhashable through the MRO. + for ty in [&bt.list_, &bt.dict_, &bt.set_, &bt.bytearray_] { + insert_if_absent(ty, "__hash__", Object::None); + } + // Hashable value types expose a real `__hash__` slot. + for ty in [ + &bt.str_, + &bt.bytes_, + &bt.int_, + &bt.bool_, + &bt.float_, + &bt.complex_, + &bt.tuple_, + &bt.frozenset_, + &bt.range_, + &bt.slice_, + ] { + insert_if_absent(ty, "__hash__", builtin("__hash__", obj_hash_builtin)); + } +} + +// --------------------------------------------------------------------------- +// container protocol dunders +// --------------------------------------------------------------------------- + +fn obj_iter_builtin(args: &[Object]) -> Result { + let recv = as_native( + args.first() + .ok_or_else(|| type_error("__iter__() missing self"))?, + ); + Ok(Object::Iter(Rc::new(RefCell::new(recv.make_iter()?)))) +} + +fn obj_len_builtin(args: &[Object]) -> Result { + let recv = as_native( + args.first() + .ok_or_else(|| type_error("__len__() missing self"))?, + ); + Ok(Object::Int(recv.len()? as i64)) +} + +fn obj_contains_builtin(args: &[Object]) -> Result { + let recv = as_native( + args.first() + .ok_or_else(|| type_error("__contains__() missing self"))?, + ); + let item = args + .get(1) + .ok_or_else(|| type_error("__contains__() takes exactly one argument (0 given)"))?; + Ok(Object::Bool(recv.contains(item)?)) +} + +fn list_reversed_builtin(args: &[Object]) -> Result { + let recv = as_native( + args.first() + .ok_or_else(|| type_error("__reversed__() missing self"))?, + ); + let Object::List(items) = &recv else { + return Err(type_error("descriptor '__reversed__' requires a 'list' object")); + }; + let reversed: Vec = items.borrow().iter().rev().cloned().collect(); + Ok(Object::Iter(Rc::new(RefCell::new(PyIterator::Tuple { + items: Rc::from(reversed.as_slice()), + index: 0, + })))) +} + +fn dict_reversed_builtin(args: &[Object]) -> Result { + let recv = as_native( + args.first() + .ok_or_else(|| type_error("__reversed__() missing self"))?, + ); + let Object::Dict(d) = &recv else { + return Err(type_error("descriptor '__reversed__' requires a 'dict' object")); + }; + let keys: Vec = d.borrow().keys().rev().map(|k| k.0.clone()).collect(); + Ok(Object::Iter(Rc::new(RefCell::new(PyIterator::Tuple { + items: Rc::from(keys.as_slice()), + index: 0, + })))) +} + +fn iter_next_builtin(args: &[Object]) -> Result { + let recv = args + .first() + .ok_or_else(|| type_error("__next__() missing self"))?; + let Object::Iter(it) = recv else { + return Err(type_error("descriptor '__next__' requires an iterator")); + }; + match it.borrow_mut().next_value() { + Some(v) => Ok(v), + None => Err(RuntimeError::PyException( + crate::error::PyException::from_builtin("StopIteration", ""), + )), + } +} + +fn iter_self_builtin(args: &[Object]) -> Result { + args.first() + .cloned() + .ok_or_else(|| type_error("__iter__() missing self")) +} + +fn install_container_protocols(bt: &BuiltinTypes) { + let iterable: [&Rc; 9] = [ + &bt.list_, + &bt.tuple_, + &bt.str_, + &bt.dict_, + &bt.set_, + &bt.frozenset_, + &bt.bytes_, + &bt.bytearray_, + &bt.range_, + ]; + for ty in iterable { + insert_if_absent(ty, "__iter__", builtin("__iter__", obj_iter_builtin)); + insert_if_absent(ty, "__len__", builtin("__len__", obj_len_builtin)); + insert_if_absent( + ty, + "__contains__", + builtin("__contains__", obj_contains_builtin), + ); + } + for ty in [&bt.dict_keys_, &bt.dict_values_, &bt.dict_items_] { + insert_if_absent(ty, "__iter__", builtin("__iter__", obj_iter_builtin)); + insert_if_absent(ty, "__len__", builtin("__len__", obj_len_builtin)); + insert_if_absent( + ty, + "__contains__", + builtin("__contains__", obj_contains_builtin), + ); + } + insert_if_absent(&bt.list_, "__reversed__", builtin("__reversed__", list_reversed_builtin)); + insert_if_absent(&bt.dict_, "__reversed__", builtin("__reversed__", dict_reversed_builtin)); + insert_if_absent(&bt.iterator_, "__iter__", builtin("__iter__", iter_self_builtin)); + insert_if_absent(&bt.iterator_, "__next__", builtin("__next__", iter_next_builtin)); +} + +// --------------------------------------------------------------------------- +// set operators — strict operand types, `NotImplemented` declines +// --------------------------------------------------------------------------- + +/// Both operands as set payloads, or `None` to signal a decline. +fn two_sets(args: &[Object]) -> Option<(Object, Object)> { + let a = as_native(args.first()?); + let b = as_native(args.get(1)?); + let is_set = |o: &Object| matches!(o, Object::Set(_) | Object::FrozenSet(_)); + if is_set(&a) && is_set(&b) { + Some((a, b)) + } else { + None + } +} + +fn set_items(o: &Object) -> Vec { + match o { + Object::Set(s) => s.borrow().iter().cloned().collect(), + Object::FrozenSet(s) => s.iter().cloned().collect(), + _ => Vec::new(), + } +} + +/// Build the result with the *left* operand's storage kind (CPython: +/// `set | frozenset` → `set`, `frozenset | set` → `frozenset`). +fn set_like(model: &Object, items: Vec) -> Object { + match model { + Object::FrozenSet(_) => { + Object::new_frozenset_from(items.into_iter().map(|k| k.0)) + } + _ => { + let mut out = indexmap::IndexSet::new(); + for k in items { + out.insert(k); + } + Object::Set(Rc::new(RefCell::new(out))) + } + } +} + +fn contains_key(o: &Object, k: &DictKey) -> bool { + match o { + Object::Set(s) => s.borrow().contains(k), + Object::FrozenSet(s) => s.contains(k), + _ => false, + } +} + +macro_rules! set_binop { + ($fname:ident, $f:expr) => { + fn $fname(args: &[Object]) -> Result { + match two_sets(args) { + Some((a, b)) => { + let op: fn(&Object, &Object) -> Vec = $f; + Ok(set_like(&a, op(&a, &b))) + } + None => Ok(crate::vm_singletons::not_implemented()), + } + } + }; +} + +set_binop!(set_sub_builtin, |a, b| { + set_items(a).into_iter().filter(|k| !contains_key(b, k)).collect() +}); +set_binop!(set_and_builtin, |a, b| { + set_items(a).into_iter().filter(|k| contains_key(b, k)).collect() +}); +set_binop!(set_or_builtin, |a, b| { + let mut items = set_items(a); + for k in set_items(b) { + if !contains_key(a, &k) { + items.push(k); + } + } + items +}); +set_binop!(set_xor_builtin, |a, b| { + let mut items: Vec = set_items(a) + .into_iter() + .filter(|k| !contains_key(b, k)) + .collect(); + for k in set_items(b) { + if !contains_key(a, &k) { + items.push(k); + } + } + items +}); + +/// Reflected forms: `__rsub__(self, other)` computes `other - self` with +/// the result kind following `other` (the left operand of the original +/// expression). +set_binop!(set_rsub_builtin, |a, b| { + set_items(b).into_iter().filter(|k| !contains_key(a, k)).collect() +}); + +fn set_rsub_outer(args: &[Object]) -> Result { + match two_sets(args) { + Some((_, b)) => { + let r = set_rsub_builtin(args)?; + // Re-key the result on the *other* operand's storage kind. + match r { + Object::Set(_) | Object::FrozenSet(_) => { + let items = set_items(&r); + Ok(set_like(&b, items)) + } + other => Ok(other), + } + } + None => Ok(crate::vm_singletons::not_implemented()), + } +} + +fn set_rand_outer(args: &[Object]) -> Result { + match two_sets(args) { + Some((a, b)) => { + let items = set_items(&b) + .into_iter() + .filter(|k| contains_key(&a, k)) + .collect(); + Ok(set_like(&b, items)) + } + None => Ok(crate::vm_singletons::not_implemented()), + } +} + +fn set_ror_outer(args: &[Object]) -> Result { + match two_sets(args) { + Some((a, b)) => { + let mut items = set_items(&b); + for k in set_items(&a) { + if !contains_key(&b, &k) { + items.push(k); + } + } + Ok(set_like(&b, items)) + } + None => Ok(crate::vm_singletons::not_implemented()), + } +} + +fn set_rxor_outer(args: &[Object]) -> Result { + match two_sets(args) { + Some((a, b)) => { + let mut items: Vec = set_items(&b) + .into_iter() + .filter(|k| !contains_key(&a, k)) + .collect(); + for k in set_items(&a) { + if !contains_key(&b, &k) { + items.push(k); + } + } + Ok(set_like(&b, items)) + } + None => Ok(crate::vm_singletons::not_implemented()), + } +} + +fn set_subset(a: &Object, b: &Object) -> bool { + set_items(a).iter().all(|k| contains_key(b, k)) +} + +macro_rules! set_cmp { + ($fname:ident, $f:expr) => { + fn $fname(args: &[Object]) -> Result { + match two_sets(args) { + Some((a, b)) => { + let op: fn(&Object, &Object) -> bool = $f; + Ok(Object::Bool(op(&a, &b))) + } + None => Ok(crate::vm_singletons::not_implemented()), + } + } + }; +} + +fn set_len_of(o: &Object) -> usize { + match o { + Object::Set(s) => s.borrow().len(), + Object::FrozenSet(s) => s.len(), + _ => 0, + } +} + +set_cmp!(set_le_builtin, |a, b| set_subset(a, b)); +set_cmp!(set_lt_builtin, |a, b| { + set_len_of(a) < set_len_of(b) && set_subset(a, b) +}); +set_cmp!(set_ge_builtin, |a, b| set_subset(b, a)); +set_cmp!(set_gt_builtin, |a, b| { + set_len_of(a) > set_len_of(b) && set_subset(b, a) +}); +set_cmp!(set_eq_builtin, |a, b| { + set_len_of(a) == set_len_of(b) && set_subset(a, b) +}); +set_cmp!(set_ne_builtin, |a, b| { + !(set_len_of(a) == set_len_of(b) && set_subset(a, b)) +}); + +/// In-place ops (`__isub__`, …) — mutate a real `set` receiver in place +/// and return it; decline for frozenset/non-set operands so the VM +/// falls back to the binary form (CPython only defines these on `set`). +macro_rules! set_iop { + ($fname:ident, $apply:expr) => { + fn $fname(args: &[Object]) -> Result { + let recv = as_native( + args.first() + .ok_or_else(|| type_error("set operation missing self"))?, + ); + let other = as_native( + args.get(1) + .ok_or_else(|| type_error("set operation missing operand"))?, + ); + let Object::Set(target) = &recv else { + return Ok(crate::vm_singletons::not_implemented()); + }; + if !matches!(other, Object::Set(_) | Object::FrozenSet(_)) { + return Ok(crate::vm_singletons::not_implemented()); + } + let apply: fn(&mut indexmap::IndexSet, &Object) = $apply; + apply(&mut target.borrow_mut(), &other); + // Return the original receiver (subclass instance included) + // so `s -= t` preserves identity. + Ok(args[0].clone()) + } + }; +} + +set_iop!(set_isub_builtin, |t, o| { + for k in set_items(o) { + t.shift_remove(&k); + } +}); +set_iop!(set_iand_builtin, |t, o| { + t.retain(|k| contains_key(o, k)); +}); +set_iop!(set_ior_builtin, |t, o| { + for k in set_items(o) { + t.insert(k); + } +}); +set_iop!(set_ixor_builtin, |t, o| { + for k in set_items(o) { + if t.contains(&k) { + t.shift_remove(&k); + } else { + t.insert(k); + } + } +}); + +fn install_set_operators(bt: &BuiltinTypes) { + for ty in [&bt.set_, &bt.frozenset_] { + insert_if_absent(ty, "__sub__", builtin("__sub__", set_sub_builtin)); + insert_if_absent(ty, "__rsub__", builtin("__rsub__", set_rsub_outer)); + insert_if_absent(ty, "__and__", builtin("__and__", set_and_builtin)); + insert_if_absent(ty, "__rand__", builtin("__rand__", set_rand_outer)); + insert_if_absent(ty, "__or__", builtin("__or__", set_or_builtin)); + insert_if_absent(ty, "__ror__", builtin("__ror__", set_ror_outer)); + insert_if_absent(ty, "__xor__", builtin("__xor__", set_xor_builtin)); + insert_if_absent(ty, "__rxor__", builtin("__rxor__", set_rxor_outer)); + insert_if_absent(ty, "__le__", builtin("__le__", set_le_builtin)); + insert_if_absent(ty, "__lt__", builtin("__lt__", set_lt_builtin)); + insert_if_absent(ty, "__ge__", builtin("__ge__", set_ge_builtin)); + insert_if_absent(ty, "__gt__", builtin("__gt__", set_gt_builtin)); + insert_if_absent(ty, "__eq__", builtin("__eq__", set_eq_builtin)); + insert_if_absent(ty, "__ne__", builtin("__ne__", set_ne_builtin)); + } + insert_if_absent(&bt.set_, "__isub__", builtin("__isub__", set_isub_builtin)); + insert_if_absent(&bt.set_, "__iand__", builtin("__iand__", set_iand_builtin)); + insert_if_absent(&bt.set_, "__ior__", builtin("__ior__", set_ior_builtin)); + insert_if_absent(&bt.set_, "__ixor__", builtin("__ixor__", set_ixor_builtin)); +} + +// --------------------------------------------------------------------------- +// dict operators (PEP 584 + equality) +// --------------------------------------------------------------------------- + +fn dict_pairs(o: &Object) -> Option>> { + match o { + Object::Dict(d) => Some(d.clone()), + _ => None, + } +} + +fn dict_or_builtin(args: &[Object]) -> Result { + let (a, b) = ( + as_native(args.first().unwrap_or(&Object::None)), + as_native(args.get(1).unwrap_or(&Object::None)), + ); + match (dict_pairs(&a), dict_pairs(&b)) { + (Some(da), Some(db)) => { + let mut merged = da.borrow().clone(); + for (k, v) in db.borrow().iter() { + merged.insert(k.clone(), v.clone()); + } + Ok(Object::Dict(Rc::new(RefCell::new(merged)))) + } + _ => Ok(crate::vm_singletons::not_implemented()), + } +} + +fn dict_ror_builtin(args: &[Object]) -> Result { + let (a, b) = ( + as_native(args.first().unwrap_or(&Object::None)), + as_native(args.get(1).unwrap_or(&Object::None)), + ); + match (dict_pairs(&a), dict_pairs(&b)) { + (Some(da), Some(db)) => { + let mut merged = db.borrow().clone(); + for (k, v) in da.borrow().iter() { + merged.insert(k.clone(), v.clone()); + } + Ok(Object::Dict(Rc::new(RefCell::new(merged)))) + } + _ => Ok(crate::vm_singletons::not_implemented()), + } +} + +fn dict_eq_builtin(args: &[Object]) -> Result { + let (a, b) = ( + as_native(args.first().unwrap_or(&Object::None)), + as_native(args.get(1).unwrap_or(&Object::None)), + ); + match (&a, &b) { + (Object::Dict(_), Object::Dict(_)) => Ok(Object::Bool(a.eq_value(&b))), + _ => Ok(crate::vm_singletons::not_implemented()), + } +} + +fn dict_ne_builtin(args: &[Object]) -> Result { + match dict_eq_builtin(args)? { + Object::Bool(v) => Ok(Object::Bool(!v)), + other => Ok(other), + } +} + +fn install_dict_operators(bt: &BuiltinTypes) { + insert_if_absent(&bt.dict_, "__or__", builtin("__or__", dict_or_builtin)); + insert_if_absent(&bt.dict_, "__ror__", builtin("__ror__", dict_ror_builtin)); + insert_if_absent(&bt.dict_, "__eq__", builtin("__eq__", dict_eq_builtin)); + insert_if_absent(&bt.dict_, "__ne__", builtin("__ne__", dict_ne_builtin)); +} + +// --------------------------------------------------------------------------- +// PEP 585 `__class_getitem__` +// --------------------------------------------------------------------------- + +fn class_getitem_builtin(args: &[Object]) -> Result { + let origin = args + .first() + .cloned() + .ok_or_else(|| type_error("__class_getitem__() missing cls"))?; + let params = args.get(1).cloned().unwrap_or(Object::None); + Ok(crate::make_generic_alias_public(origin, params)) +} + +fn install_class_getitem(bt: &BuiltinTypes) { + for ty in [ + &bt.list_, + &bt.tuple_, + &bt.dict_, + &bt.set_, + &bt.frozenset_, + &bt.type_, + ] { + insert_if_absent( + ty, + "__class_getitem__", + Object::ClassMethod(Rc::new(builtin( + "__class_getitem__", + class_getitem_builtin, + ))), + ); + } +} + +// --------------------------------------------------------------------------- +// PEP 688 `__buffer__` +// --------------------------------------------------------------------------- + +fn buffer_builtin(args: &[Object]) -> Result { + let recv = as_native( + args.first() + .ok_or_else(|| type_error("__buffer__() missing self"))?, + ); + match &recv { + Object::Bytes(b) => Ok(Object::MemoryView(Rc::new(PyMemoryView::from_bytes( + b.clone(), + )))), + Object::ByteArray(b) => Ok(Object::MemoryView(Rc::new( + PyMemoryView::from_bytearray(b.clone()), + ))), + Object::MemoryView(_) => Ok(recv.clone()), + other => Err(value_error(format!( + "__buffer__ not supported for '{}'", + other.type_name() + ))), + } +} + +fn install_buffer_protocol(bt: &BuiltinTypes) { + for ty in [&bt.bytes_, &bt.bytearray_, &bt.memoryview_] { + insert_if_absent(ty, "__buffer__", builtin("__buffer__", buffer_builtin)); + } +} + +// --------------------------------------------------------------------------- +// regular method tables (reusing `lookup_method` via a representative) +// --------------------------------------------------------------------------- + +fn install_named_methods(ty: &Rc, type_name: &str, names: &[&str]) { + for name in names { + if let Some(Object::Builtin(inner)) = crate::builtins::unbound_method(type_name, name) { + insert_if_absent(ty, name, unwrap_shim(inner)); + } + } +} + +fn install_method_tables(bt: &BuiltinTypes) { + install_named_methods( + &bt.str_, + "str", + &[ + "upper", "lower", "title", "capitalize", "casefold", "swapcase", "strip", "lstrip", + "rstrip", "split", "rsplit", "splitlines", "join", "startswith", "endswith", + "replace", "find", "rfind", "index", "rindex", "count", "partition", "rpartition", + "isdigit", "isalpha", "isalnum", "isspace", "isupper", "islower", "isascii", + "isnumeric", "isdecimal", "isidentifier", "isprintable", "istitle", "zfill", + "ljust", "rjust", "center", "expandtabs", "encode", "removeprefix", "removesuffix", + "translate", "maketrans", "__getitem__", + ], + ); + install_named_methods( + &bt.list_, + "list", + &[ + "append", + "pop", + "extend", + "insert", + "remove", + "index", + "count", + "sort", + "reverse", + "clear", + "copy", + "__getitem__", + "__setitem__", + "__delitem__", + ], + ); + install_named_methods( + &bt.dict_, + "dict", + &[ + "get", + "keys", + "values", + "items", + "pop", + "update", + "clear", + "setdefault", + "copy", + "fromkeys", + "popitem", + "__getitem__", + "__setitem__", + "__delitem__", + ], + ); + install_named_methods(&bt.tuple_, "tuple", &["count", "index", "__getitem__"]); + install_named_methods( + &bt.set_, + "set", + &[ + "add", + "discard", + "remove", + "pop", + "clear", + "copy", + "update", + "union", + "intersection", + "difference", + "symmetric_difference", + "issubset", + "issuperset", + "isdisjoint", + "intersection_update", + "difference_update", + "symmetric_difference_update", + ], + ); + install_named_methods( + &bt.frozenset_, + "frozenset", + &[ + "copy", + "union", + "intersection", + "difference", + "symmetric_difference", + "issubset", + "issuperset", + "isdisjoint", + ], + ); + for (ty, name) in [(&bt.bytes_, "bytes"), (&bt.bytearray_, "bytearray")] { + install_named_methods( + ty, + name, + &[ + "decode", + "hex", + "fromhex", + "startswith", + "endswith", + "find", + "rfind", + "index", + "rindex", + "count", + "lower", + "upper", + "strip", + "lstrip", + "rstrip", + "split", + "rsplit", + "splitlines", + "join", + "replace", + "translate", + "maketrans", + "partition", + "rpartition", + "removeprefix", + "removesuffix", + "expandtabs", + "center", + "ljust", + "rjust", + "zfill", + "capitalize", + "title", + "swapcase", + "isalnum", + "isalpha", + "isdigit", + "isspace", + "islower", + "isupper", + "istitle", + "isascii", + ], + ); + } + install_named_methods( + &bt.bytearray_, + "bytearray", + &["append", "extend", "clear", "pop", "reverse", "insert"], + ); +} diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index d8ed42b..1a5881a 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -338,6 +338,23 @@ impl TypeObject { pub fn class_name(&self) -> &str { &self.name } + + /// CPython `type_repr` name: `__module__.__qualname__`, with the + /// module prefix omitted for `builtins` (so `` but + /// `` / ``). + pub fn qualified_display_name(&self) -> String { + let dict = self.dict.borrow(); + let module = dict + .get(&DictKey(Object::from_static("__module__"))) + .map(Object::to_str); + let qual = dict + .get(&DictKey(Object::from_static("__qualname__"))) + .map_or_else(|| self.name.clone(), Object::to_str); + match module.as_deref() { + None | Some("builtins") | Some("") => qual, + Some(m) => format!("{m}.{qual}"), + } + } } fn compute_c3( @@ -410,6 +427,12 @@ pub struct PyInstance { /// The capacity-overflow half of the state (too many attributes) /// is computed at query time from the dict size. pub inline_values: Cell, + /// `__slots__` storage. CPython lays slot values out as C struct + /// members *outside* the instance `__dict__`; we mirror that + /// separation with a side table so `vars(obj)` never exposes slot + /// values and `object.__getstate__` can report them separately. + /// `None` until the first slot write (most instances have none). + pub slots: RefCell>, } impl PyInstance { @@ -419,6 +442,7 @@ impl PyInstance { dict: Rc::new(RefCell::new(DictData::new())), native: None, inline_values: Cell::new(true), + slots: RefCell::new(None), } } @@ -430,6 +454,7 @@ impl PyInstance { dict: Rc::new(RefCell::new(DictData::new())), native: Some(native), inline_values: Cell::new(true), + slots: RefCell::new(None), } } @@ -443,4 +468,46 @@ impl PyInstance { pub fn set_cls(&self, class: Rc) { *self.class.borrow_mut() = class; } + + /// Read slot `name` from the side table (a `__slots__` member). + pub fn slot_get(&self, name: &str) -> Option { + self.slots + .borrow() + .as_ref() + .and_then(|s| s.get(&DictKey(Object::from_str(name))).cloned()) + } + + /// Write slot `name` into the side table. + pub fn slot_set(&self, name: &str, value: Object) { + self.slots + .borrow_mut() + .get_or_insert_with(DictData::new) + .insert(DictKey(Object::from_str(name)), value); + } + + /// Delete slot `name` from the side table; `false` when unset. + pub fn slot_del(&self, name: &str) -> bool { + self.slots + .borrow_mut() + .as_mut() + .map(|s| s.shift_remove(&DictKey(Object::from_str(name))).is_some()) + .unwrap_or(false) + } + + /// Snapshot of the populated slot values (for `__getstate__`, + /// `copy`, and GC tracing). + pub fn slots_snapshot(&self) -> Vec<(String, Object)> { + self.slots + .borrow() + .as_ref() + .map(|s| { + s.iter() + .filter_map(|(k, v)| match &k.0 { + Object::Str(name) => Some((name.to_string(), v.clone())), + _ => None, + }) + .collect() + }) + .unwrap_or_default() + } } diff --git a/crates/weavepy/tests/fixtures/run/36_slots.out b/crates/weavepy/tests/fixtures/run/36_slots.out index cbc1ba4..e0fa24f 100644 --- a/crates/weavepy/tests/fixtures/run/36_slots.out +++ b/crates/weavepy/tests/fixtures/run/36_slots.out @@ -1,6 +1,6 @@ 3 4 10 4 -slot reject: 'Point' object has no attribute 'z' +slot reject: 'Point' object has no attribute 'z' and no __dict__ for setting new attributes 1 2 3 -nested slot reject: 'Vec3' object has no attribute 'w' +nested slot reject: 'Vec3' object has no attribute 'w' and no __dict__ for setting new attributes 7 8 fine