From 573e6508431989baa9ae67af650713a1feedfad0 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Fri, 18 Sep 2026 11:03:10 -0700 Subject: [PATCH 1/2] chore(deps): Bump openjd-* Rust crates to 0.9.0 openjd-rs released on 2026-09-18: openjd-expr 0.9.0, openjd-model 0.9.0, openjd-sessions 0.7.0 (OpenJobDescription/openjd-rs#401). The release is one feature plumbed through all three crates -- opt-in resolved-value caps and evaluation budgets (#399) -- plus a job-creation re-check on carried-forward fields (#404) and an MSRV/dependency sweep (#403). Four breaking signature changes reach this package's bindings: FormatString::validate_expressions now takes a FormatStringOptions, decode_environment_template takes CallerLimits, evaluate_let_bindings takes the two budgets, and CallerLimits / SessionConfig gained fields. CallerLimits grows from six fields to ten, and the four new ones are threaded to every entry point upstream threads them to: the environment template decoders, evaluate_let_bindings, and Session, which is the run-time enforcement boundary for the resolved-value caps. Separately, max_template_size was inert in this package. It is checked only inside document_string_to_object, and the parse_string helper passed a default CallerLimits, so a caller asking for a byte ceiling on decode_job_template_str got none. parse_string now takes the caller's own limits. Found while documenting the field. Cargo.lock moved the three crates plus the #403 sweep; THIRD-PARTY-LICENSES regenerated to match. Verified: 6139 passed / 24 skipped / 3 xfailed, coverage 94.16%; ruff, black, mypy, cargo fmt and clippy clean. Six mutants covering each piece of new plumbing were each caught by the new tests. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- Cargo.lock | 148 ++++++++--- THIRD-PARTY-LICENSES.txt | 259 ++++++++++++++++++- rust-bindings/Cargo.toml | 6 +- rust-bindings/src/expr/format_string.rs | 12 +- rust-bindings/src/model/create_job_fns.rs | 15 +- rust-bindings/src/model/decode.rs | 55 ++-- rust-bindings/src/model/profile.rs | 72 +++++- rust-bindings/src/sessions/session.rs | 14 +- specs/python-expr-interface.md | 8 +- specs/python-model-interface.md | 62 ++++- specs/python-sessions-interface.md | 11 + src/openjd/_openjd_rs.pyi | 93 ++++++- test/openjd/model_v1/test_create_job.py | 156 ++++++++++++ test/openjd/model_v1/test_let_bindings.py | 50 +++- test/openjd/model_v1/test_parse.py | 280 +++++++++++++++++++++ test/openjd/model_v1/test_pickle.py | 13 + test/openjd/sessions/test_caller_limits.py | 79 ++++++ 17 files changed, 1235 insertions(+), 98 deletions(-) create mode 100644 test/openjd/sessions/test_caller_limits.py diff --git a/Cargo.lock b/Cargo.lock index c96fed54..57679de1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,6 +43,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + [[package]] name = "arc-swap" version = "1.9.2" @@ -58,6 +67,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "attribute-derive" version = "0.10.5" @@ -160,6 +175,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "char_str" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576ba56f6ca18ebb069d0d07260407171712aff4dfe15ee3f8982dc16a455bcf" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", +] + [[package]] name = "chrono" version = "0.4.45" @@ -181,16 +208,15 @@ checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" [[package]] name = "compact_str" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" dependencies = [ "castaway", "cfg-if", "itoa", - "rustversion", - "ryu", "static_assertions", + "zmij", ] [[package]] @@ -222,6 +248,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "drop_bomb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" + [[package]] name = "either" version = "1.18.0" @@ -323,9 +355,9 @@ dependencies = [ [[package]] name = "get-size-derive2" -version = "0.7.4" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b6d1e2f75c16bfbcd0f95d84f99858a6e2f885c2287d1f5c3a96e8444a34b4" +checksum = "c736d226c32e496b8377813b52269e11ad3a48d8373b68862d0364f04fd1229d" dependencies = [ "attribute-derive", "quote", @@ -334,15 +366,16 @@ dependencies = [ [[package]] name = "get-size2" -version = "0.7.4" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cf31a6d70300cf81461098f7797571362387ef4bf85d32ac47eaa59b3a5a1a" +checksum = "b411f34418305908ab15a82ff78958c2a9aee9a272b2a2e663836b20a4e4b9d3" dependencies = [ "compact_str", "get-size-derive2", - "hashbrown 0.16.1", + "hashbrown", "ordermap", "smallvec", + "thin-vec", ] [[package]] @@ -386,12 +419,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" - [[package]] name = "hashbrown" version = "0.17.1" @@ -435,7 +462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" dependencies = [ "equivalent", - "hashbrown 0.17.1", + "hashbrown", "serde", "serde_core", ] @@ -485,6 +512,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -671,6 +707,15 @@ dependencies = [ "rustc-hash 2.1.3", ] +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -679,9 +724,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openjd-expr" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5396e716811e785c9a81ad276f0b74b37db297ec2fb830ff5a002715787578c" +checksum = "2b016bd852c294b6a55a632e794bb150a305cf04529ba98c0255f57edefbda9c" dependencies = [ "regex", "regex-syntax", @@ -696,9 +741,9 @@ dependencies = [ [[package]] name = "openjd-model" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10cc7ef51861ddc5170c9a03eaa261161a5819c007cdbd9b0ed6c8da23fbc1bb" +checksum = "655c4a740ee46f502c993fe0dcd334f442721dfbc2aa52803b576f5370ea69d4" dependencies = [ "indexmap", "openjd-expr", @@ -728,9 +773,9 @@ dependencies = [ [[package]] name = "openjd-sessions" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c52db94b553d39934d6f6c13d16a31fe692215f4917fad4b4d695460ef3c8b22" +checksum = "1c4f7afee55eaf0cd1b7dafa55203386283e614088a94379aca213a6efa68e25" dependencies = [ "bitflags", "futures-util", @@ -863,6 +908,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "pyo3" version = "0.29.2" @@ -1144,12 +1199,14 @@ dependencies = [ [[package]] name = "rustpython-ruff_python_ast" -version = "0.15.8" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f021ff72cabf5e2cd6d8ec8813d376a8445a228dc610ab56c27bd9054cda70d4" +checksum = "bc19eeb4cfa2c35428143693eadf85ee78816f22a829298593581a56700c0a21" dependencies = [ "aho-corasick", + "arrayvec", "bitflags", + "char_str", "compact_str", "get-size2", "is-macro", @@ -1158,25 +1215,28 @@ dependencies = [ "rustpython-ruff_python_trivia", "rustpython-ruff_source_file", "rustpython-ruff_text_size", + "thin-vec", "thiserror", ] [[package]] name = "rustpython-ruff_python_parser" -version = "0.15.8" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01e6ee78bd9671fb5766664b2695fe1f2a92a961f4d9101646c570d8acdb1e0b" +checksum = "480eec9ac4dda77b6724f0c9a54a81d3c57868d6a707c3d7c5133fd83c933f66" dependencies = [ "bitflags", "bstr", - "compact_str", + "drop_bomb", "get-size2", "memchr", "rustc-hash 2.1.3", "rustpython-ruff_python_ast", "rustpython-ruff_python_trivia", "rustpython-ruff_text_size", + "stacker", "static_assertions", + "thin-vec", "unicode-ident", "unicode-normalization", "unicode_names2", @@ -1184,11 +1244,12 @@ dependencies = [ [[package]] name = "rustpython-ruff_python_trivia" -version = "0.15.8" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e7cfd1056f3a02ff0d2d0e4474286ca963260782f878b7b81c1dd87432e682" +checksum = "6d69a5d9bb610a76bee05d23adce0de55f0fa06f251c0e1c7f7a0c625a366a58" dependencies = [ - "itertools 0.14.0", + "itertools 0.15.0", + "rustc-hash 2.1.3", "rustpython-ruff_source_file", "rustpython-ruff_text_size", "unicode-ident", @@ -1196,9 +1257,9 @@ dependencies = [ [[package]] name = "rustpython-ruff_source_file" -version = "0.15.8" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "948107aad62ddb12a11fc7bf68a49e52a0b0a3737d415a2505e54f5a9edac737" +checksum = "d3aa9c91c8a4e86bf0fa8a67aec284fa6c0c533cdefb900b08584be15748c606" dependencies = [ "memchr", "rustpython-ruff_text_size", @@ -1206,9 +1267,9 @@ dependencies = [ [[package]] name = "rustpython-ruff_text_size" -version = "0.15.8" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8291ee0f5a779e54ccd4e0151a0c426f8b49a123f99b5b6545db17ccdd4277aa" +checksum = "8fedbd1af4f91cd828185123d95119662db9c4a0b21c02b695783d74360d4f46" dependencies = [ "get-size2", ] @@ -1337,6 +1398,19 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -1450,6 +1524,12 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "thin-vec" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568d7e143ec86d2021c338bae2afa88699e84b8e0af523626654fe6f03a1748" + [[package]] name = "thiserror" version = "2.0.20" diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index b9e27ea0..8e53a0ca 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -2089,19 +2089,20 @@ limitations under the License. ------ ** arc-swap; version 1.9.2 -- https://crates.io/crates/arc-swap +** arrayvec; version 0.7.8 -- https://crates.io/crates/arrayvec ** base64; version 0.23.1 -- https://crates.io/crates/base64 ** bitflags; version 2.13.1 -- https://crates.io/crates/bitflags ** bstr; version 1.13.1 -- https://crates.io/crates/bstr ** cfg-if; version 1.0.4 -- https://crates.io/crates/cfg-if +** drop_bomb; version 0.1.5 -- https://crates.io/crates/drop_bomb ** either; version 1.18.0 -- https://crates.io/crates/either ** encoding_rs_io; version 0.1.8 -- https://crates.io/crates/encoding_rs_io ** equivalent; version 1.0.2 -- https://crates.io/crates/equivalent ** errno; version 0.3.14 -- https://crates.io/crates/errno -** hashbrown; version 0.16.1 -- https://crates.io/crates/hashbrown ** hashbrown; version 0.17.1 -- https://crates.io/crates/hashbrown ** heck; version 0.5.0 -- https://crates.io/crates/heck ** indexmap; version 2.14.1 -- https://crates.io/crates/indexmap -** itertools; version 0.14.0 -- https://crates.io/crates/itertools +** itertools; version 0.15.0 -- https://crates.io/crates/itertools ** log; version 0.4.34 -- https://crates.io/crates/log ** num-traits; version 0.2.19 -- https://crates.io/crates/num-traits ** once_cell; version 1.21.4 -- https://crates.io/crates/once_cell @@ -2112,6 +2113,7 @@ limitations under the License. ** regex; version 1.13.1 -- https://crates.io/crates/regex ** signal-hook-registry; version 1.4.8 -- https://crates.io/crates/signal-hook-registry ** smallvec; version 1.15.2 -- https://crates.io/crates/smallvec +** stacker; version 0.1.25 -- https://crates.io/crates/stacker ** unicode-normalization; version 0.1.25 -- https://crates.io/crates/unicode-normalization ** unicode-width; version 0.2.2 -- https://crates.io/crates/unicode-width ** unicode_names2; version 1.3.0 -- https://crates.io/crates/unicode_names2 @@ -2526,17 +2528,222 @@ See the License for the specific language governing permissions and limitations under the License. +------ +** psm; version 0.1.32 -- https://crates.io/crates/psm + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + ------ ** arraydeque; version 0.5.1 -- https://crates.io/crates/arraydeque -** get-size-derive2; version 0.7.4 -- https://crates.io/crates/get-size-derive2 -** get-size2; version 0.7.4 -- https://crates.io/crates/get-size2 +** get-size-derive2; version 0.10.3 -- https://crates.io/crates/get-size-derive2 +** get-size2; version 0.10.3 -- https://crates.io/crates/get-size2 ** granit-parser; version 1.2.0 -- https://crates.io/crates/granit-parser ** itoa; version 1.0.18 -- https://crates.io/crates/itoa ** libc; version 0.2.189 -- https://crates.io/crates/libc ** manyhow-macros; version 0.11.4 -- https://crates.io/crates/manyhow-macros -** openjd-expr; version 0.8.0 -- https://crates.io/crates/openjd-expr -** openjd-model; version 0.8.0 -- https://crates.io/crates/openjd-model -** openjd-sessions; version 0.6.0 -- https://crates.io/crates/openjd-sessions +** openjd-expr; version 0.9.0 -- https://crates.io/crates/openjd-expr +** openjd-model; version 0.9.0 -- https://crates.io/crates/openjd-model +** openjd-sessions; version 0.7.0 -- https://crates.io/crates/openjd-sessions ** pin-project-lite; version 0.2.17 -- https://crates.io/crates/pin-project-lite ** portable-atomic; version 1.15.0 -- https://crates.io/crates/portable-atomic ** proc-macro2; version 1.0.107 -- https://crates.io/crates/proc-macro2 @@ -2558,6 +2765,7 @@ limitations under the License. ** siphasher; version 1.0.3 -- https://crates.io/crates/siphasher ** syn; version 2.0.119 -- https://crates.io/crates/syn ** syn; version 3.0.4 -- https://crates.io/crates/syn +** thin-vec; version 0.2.20 -- https://crates.io/crates/thin-vec ** thiserror-impl; version 2.0.20 -- https://crates.io/crates/thiserror-impl ** thiserror; version 2.0.20 -- https://crates.io/crates/thiserror ** unicode-ident; version 1.0.24 -- https://crates.io/crates/unicode-ident @@ -2861,7 +3069,7 @@ SOFTWARE. ------ -** compact_str; version 0.9.1 -- https://crates.io/crates/compact_str +** compact_str; version 0.10.0 -- https://crates.io/crates/compact_str MIT License Copyright (c) 2021 Parker Timmerman @@ -2935,14 +3143,39 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------ +** char_str; version 0.0.2 -- https://crates.io/crates/char_str +MIT License + +Copyright (c) 2024 ryota2357 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + ------ ** quote-use-macros; version 0.8.4 -- https://crates.io/crates/quote-use-macros ** quote-use; version 0.8.4 -- https://crates.io/crates/quote-use -** rustpython-ruff_python_ast; version 0.15.8 -- https://crates.io/crates/rustpython-ruff_python_ast -** rustpython-ruff_python_parser; version 0.15.8 -- https://crates.io/crates/rustpython-ruff_python_parser -** rustpython-ruff_python_trivia; version 0.15.8 -- https://crates.io/crates/rustpython-ruff_python_trivia -** rustpython-ruff_source_file; version 0.15.8 -- https://crates.io/crates/rustpython-ruff_source_file -** rustpython-ruff_text_size; version 0.15.8 -- https://crates.io/crates/rustpython-ruff_text_size +** rustpython-ruff_python_ast; version 0.16.5 -- https://crates.io/crates/rustpython-ruff_python_ast +** rustpython-ruff_python_parser; version 0.16.5 -- https://crates.io/crates/rustpython-ruff_python_parser +** rustpython-ruff_python_trivia; version 0.16.5 -- https://crates.io/crates/rustpython-ruff_python_trivia +** rustpython-ruff_source_file; version 0.16.5 -- https://crates.io/crates/rustpython-ruff_source_file +** rustpython-ruff_text_size; version 0.16.5 -- https://crates.io/crates/rustpython-ruff_text_size MIT License Copyright (c) diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index c27c12f5..cb094f5f 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -12,9 +12,9 @@ name = "_openjd_rs" crate-type = ["cdylib", "rlib"] [dependencies] -openjd-expr = "0.8.0" -openjd-model = "0.8.0" -openjd-sessions = "0.6.0" +openjd-expr = "0.9.0" +openjd-model = "0.9.0" +openjd-sessions = "0.7.0" tokio = { version = "1", features = ["rt-multi-thread"] } uuid = { version = "1", features = ["v4"] } serde_json = "1" diff --git a/rust-bindings/src/expr/format_string.rs b/rust-bindings/src/expr/format_string.rs index c53ee193..61b7a9b6 100644 --- a/rust-bindings/src/expr/format_string.rs +++ b/rust-bindings/src/expr/format_string.rs @@ -128,12 +128,13 @@ impl PyFormatString { /// checking through the expression tree. /// /// Mirrors the Rust crate's - /// `FormatString::validate_expressions(symtab, lib, target_type)`. + /// `FormatString::validate_expressions(symtab, opts)`. /// Returns `None` on success. /// - /// `target_type` is passed as `None` because `resolve` and - /// `resolve_string` above resolve without one; validation has to - /// observe the same values resolution will produce. The crate + /// The options carry the library only — the same ones `resolve` + /// and `resolve_string` above build, since validation has to + /// observe the values resolution will produce, including its lack + /// of a target type and its default evaluation budgets. The crate /// returns a `StaticResolution` (resolved-length bound and, when /// fully concrete, the resolved value); this binding is pass/fail /// only and discards it. @@ -145,8 +146,9 @@ impl PyFormatString { ) -> PyResult<()> { let st = extract_symtab(symtab)?; let lib = profile_for_call(profile); + let opts = FormatStringOptions::new().with_library(&lib); self.inner - .validate_expressions(&st, &lib, None) + .validate_expressions(&st, &opts) .map(|_| ()) .map_err(format_string_validation_err_to_py) } diff --git a/rust-bindings/src/model/create_job_fns.rs b/rust-bindings/src/model/create_job_fns.rs index 64a63e32..1cbea67a 100644 --- a/rust-bindings/src/model/create_job_fns.rs +++ b/rust-bindings/src/model/create_job_fns.rs @@ -316,13 +316,24 @@ fn default_to_native( feature = "stub-gen", gen_stub_pyfunction(module = "openjd._openjd_rs") )] +/// Evaluate ``let`` bindings in order, returning a symbol table with +/// each bound name added. +/// +/// ``caller_limits`` supplies the evaluation budgets: a binding +/// evaluates a parsed expression directly rather than resolving a +/// format string, so a caller enforcing ``max_eval_memory_bytes`` / +/// ``max_eval_operations`` elsewhere has to pass the same limits here +/// for the budgets to bound every evaluation uniformly. The other +/// ``CallerLimits`` fields have no meaning for a let binding and are +/// ignored. Omitting it uses the spec-recommended defaults. #[pyfunction] #[pyo3(name = "evaluate_let_bindings")] -#[pyo3(signature = (bindings, symtab, *, profile=None))] +#[pyo3(signature = (bindings, symtab, *, profile=None, caller_limits=None))] pub(crate) fn py_evaluate_let_bindings( bindings: Vec, symtab: &crate::expr::PySymbolTable, profile: Option<&crate::expr::profile::PyExprProfile>, + caller_limits: Option<&crate::model::profile::PyCallerLimits>, ) -> PyResult { let lib = crate::expr::evaluate::profile_for_call(profile); let result = openjd_model::evaluate_let_bindings( @@ -330,6 +341,8 @@ pub(crate) fn py_evaluate_let_bindings( &symtab.inner, Some(&lib), openjd_expr::path_mapping::PathFormat::host(), + caller_limits.and_then(|c| c.inner.max_eval_memory_bytes), + caller_limits.and_then(|c| c.inner.max_eval_operations), ) .map_err(super::errors::model_err_to_py)?; Ok(crate::expr::PySymbolTable { inner: result }) diff --git a/rust-bindings/src/model/decode.rs b/rust-bindings/src/model/decode.rs index a5eb3305..480b2b83 100644 --- a/rust-bindings/src/model/decode.rs +++ b/rust-bindings/src/model/decode.rs @@ -15,14 +15,19 @@ use super::template::{PyEnvironmentTemplate, PyJobTemplate}; use super::types::PyDocumentType; /// Parse a raw string into serde_json::Value based on document type. -fn parse_string(document: &str, format: PyDocumentType) -> PyResult { +/// +/// `limits` has to be the caller's own: `max_template_size` is checked here, +/// against the document's byte length before parsing, and nowhere else. Passing +/// a default here left that field inert on the `*_str` entry points even though +/// they accept it. +fn parse_string( + document: &str, + format: PyDocumentType, + limits: &CallerLimits, +) -> PyResult { let doc_type: DocumentType = format.into(); - openjd_model::template::parse::document_string_to_object( - document, - doc_type, - &CallerLimits::default(), - ) - .map_err(model_err_to_py) + openjd_model::template::parse::document_string_to_object(document, doc_type, limits) + .map_err(model_err_to_py) } /// Convert a Python dict to serde_json::Value via JSON round-trip. @@ -66,6 +71,8 @@ fn limits_or_default(c: Option<&PyCallerLimits>) -> CallerLimits { /// every extension the template requests). /// caller_limits: Optional ``CallerLimits`` to tighten /// spec-defined limits (e.g. maximum step count). +/// ``max_template_size`` is checked here, against +/// ``document``'s byte length before parsing. /// /// Returns: /// The parsed ``openjd.model._v1.template.JobTemplate``. Use @@ -84,9 +91,9 @@ pub(crate) fn decode_job_template_str( supported_extensions: Option>, caller_limits: Option<&PyCallerLimits>, ) -> PyResult { - let value = parse_string(document, format)?; - let exts = as_str_slice(&supported_extensions); let limits = limits_or_default(caller_limits); + let value = parse_string(document, format, &limits)?; + let exts = as_str_slice(&supported_extensions); let jt = openjd_model::decode_job_template(value, exts.as_deref(), &limits) .map_err(model_err_to_py)?; Ok(PyJobTemplate { inner: jt }) @@ -111,6 +118,9 @@ pub(crate) fn decode_job_template_str( /// every extension the template requests). /// caller_limits: Optional ``CallerLimits`` to tighten /// spec-defined limits (e.g. maximum step count). +/// ``max_template_size`` bounds a document's encoded byte +/// length, so it has nothing to measure on this entry point +/// and applies only to ``decode_job_template_str``. /// /// Returns: /// The parsed ``openjd.model._v1.template.JobTemplate``. Use @@ -155,24 +165,31 @@ pub(crate) fn decode_job_template( /// ``Unsupported extension names: ...``. Pass ``None`` /// (the default) for an empty allowlist (i.e., reject /// every extension the template requests). +/// caller_limits: Optional ``CallerLimits`` to tighten +/// spec-defined limits. The caps that only a job template has +/// (step and task counts) do not apply here; the +/// resolved-value caps and evaluation budgets do, and +/// ``max_template_size`` is checked against ``document``'s +/// byte length before parsing. /// /// Returns: /// The parsed ``openjd.model._v1.template.EnvironmentTemplate``. -/// Environment templates do not accept ``caller_limits``. #[cfg_attr( feature = "stub-gen", gen_stub_pyfunction(module = "openjd._openjd_rs") )] #[pyfunction] -#[pyo3(signature = (document, format=PyDocumentType::YAML, *, supported_extensions=None))] +#[pyo3(signature = (document, format=PyDocumentType::YAML, *, supported_extensions=None, caller_limits=None))] pub(crate) fn decode_environment_template_str( document: &str, format: PyDocumentType, supported_extensions: Option>, + caller_limits: Option<&PyCallerLimits>, ) -> PyResult { - let value = parse_string(document, format)?; + let limits = limits_or_default(caller_limits); + let value = parse_string(document, format, &limits)?; let exts = as_str_slice(&supported_extensions); - let et = openjd_model::decode_environment_template(value, exts.as_deref()) + let et = openjd_model::decode_environment_template(value, exts.as_deref(), &limits) .map_err(model_err_to_py)?; Ok(PyEnvironmentTemplate { inner: et }) } @@ -194,23 +211,29 @@ pub(crate) fn decode_environment_template_str( /// ``Unsupported extension names: ...``. Pass ``None`` /// (the default) for an empty allowlist (i.e., reject /// every extension the template requests). +/// caller_limits: Optional ``CallerLimits`` to tighten +/// spec-defined limits. The caps that only a job template has +/// (step and task counts) do not apply here, and +/// ``max_template_size`` has no document string to measure; +/// the resolved-value caps and evaluation budgets do apply. /// /// Returns: /// The parsed ``openjd.model._v1.template.EnvironmentTemplate``. -/// Environment templates do not accept ``caller_limits``. #[cfg_attr( feature = "stub-gen", gen_stub_pyfunction(module = "openjd._openjd_rs") )] #[pyfunction] -#[pyo3(signature = (template, *, supported_extensions=None))] +#[pyo3(signature = (template, *, supported_extensions=None, caller_limits=None))] pub(crate) fn decode_environment_template( template: &Bound<'_, PyDict>, supported_extensions: Option>, + caller_limits: Option<&PyCallerLimits>, ) -> PyResult { let value = dict_to_json_value(template)?; let exts = as_str_slice(&supported_extensions); - let et = openjd_model::decode_environment_template(value, exts.as_deref()) + let limits = limits_or_default(caller_limits); + let et = openjd_model::decode_environment_template(value, exts.as_deref(), &limits) .map_err(model_err_to_py)?; Ok(PyEnvironmentTemplate { inner: et }) } diff --git a/rust-bindings/src/model/profile.rs b/rust-bindings/src/model/profile.rs index 5946e9dd..7cdd00d6 100644 --- a/rust-bindings/src/model/profile.rs +++ b/rust-bindings/src/model/profile.rs @@ -452,6 +452,25 @@ impl PyModelProfile { /// beyond the spec-defined limit." Caller limits can only tighten /// spec-defined limits, never relax them. /// +/// Two of the fields are resolved-value caps rather than document +/// caps: ``max_resolved_arg_len`` bounds each resolved action +/// ``command`` and each argv entry an ``args`` element produces +/// (Template Schemas §5.1, §5.2), and ``max_resolved_data_len`` +/// bounds each resolved embedded-file ``data`` value (§6.1.2). The +/// spec sets no maximum for either; both count **characters**, while +/// the operating-system limits they stand in for are measured in +/// bytes or UTF-16 code units, so leave encoding headroom. Both are +/// checked at template validation against the guaranteed lower bound +/// of every possible resolution, again at job creation with +/// parameters bound, and — for a caller that mirrors them into a +/// session — at run time on the final values. +/// +/// ``max_eval_memory_bytes`` and ``max_eval_operations`` are the +/// Expression Language spec's memory-bounded-evaluation budgets, +/// applied per format-string expression. ``None`` uses the +/// spec-recommended defaults (100 MB and 10 million operations); +/// lowering them is spec-sanctioned configuration. +/// /// Mirrors `openjd_model::CallerLimits`. #[cfg_attr(feature = "stub-gen", gen_stub_pyclass(module = "openjd._openjd_rs"))] #[pyclass( @@ -477,7 +496,12 @@ impl PyCallerLimits { max_step_script_size=None, max_environment_size=None, max_template_size=None, + max_resolved_arg_len=None, + max_resolved_data_len=None, + max_eval_memory_bytes=None, + max_eval_operations=None, ))] + #[allow(clippy::too_many_arguments)] fn new( max_step_count: Option, max_env_count: Option, @@ -485,6 +509,10 @@ impl PyCallerLimits { max_step_script_size: Option, max_environment_size: Option, max_template_size: Option, + max_resolved_arg_len: Option, + max_resolved_data_len: Option, + max_eval_memory_bytes: Option, + max_eval_operations: Option, ) -> Self { Self { inner: CallerLimits { @@ -494,6 +522,10 @@ impl PyCallerLimits { max_step_script_size, max_environment_size, max_template_size, + max_resolved_arg_len, + max_resolved_data_len, + max_eval_memory_bytes, + max_eval_operations, }, } } @@ -523,20 +555,46 @@ impl PyCallerLimits { self.inner.max_template_size } + #[getter] + fn max_resolved_arg_len(&self) -> Option { + self.inner.max_resolved_arg_len + } + + #[getter] + fn max_resolved_data_len(&self) -> Option { + self.inner.max_resolved_data_len + } + + #[getter] + fn max_eval_memory_bytes(&self) -> Option { + self.inner.max_eval_memory_bytes + } + + #[getter] + fn max_eval_operations(&self) -> Option { + self.inner.max_eval_operations + } + fn __repr__(&self) -> String { format!( "CallerLimits(max_step_count={:?}, max_env_count={:?}, max_task_count={:?}, \ - max_step_script_size={:?}, max_environment_size={:?}, max_template_size={:?})", + max_step_script_size={:?}, max_environment_size={:?}, max_template_size={:?}, \ + max_resolved_arg_len={:?}, max_resolved_data_len={:?}, \ + max_eval_memory_bytes={:?}, max_eval_operations={:?})", self.inner.max_step_count, self.inner.max_env_count, self.inner.max_task_count, self.inner.max_step_script_size, self.inner.max_environment_size, self.inner.max_template_size, + self.inner.max_resolved_arg_len, + self.inner.max_resolved_data_len, + self.inner.max_eval_memory_bytes, + self.inner.max_eval_operations, ) } - /// Pickle support — round-trips through `__init__` with all six + /// Pickle support — round-trips through `__init__` with all ten /// optional fields as keyword arguments. fn __reduce__<'py>( &self, @@ -554,12 +612,16 @@ impl PyCallerLimits { kwargs.set_item("max_step_script_size", self.inner.max_step_script_size)?; kwargs.set_item("max_environment_size", self.inner.max_environment_size)?; kwargs.set_item("max_template_size", self.inner.max_template_size)?; + kwargs.set_item("max_resolved_arg_len", self.inner.max_resolved_arg_len)?; + kwargs.set_item("max_resolved_data_len", self.inner.max_resolved_data_len)?; + kwargs.set_item("max_eval_memory_bytes", self.inner.max_eval_memory_bytes)?; + kwargs.set_item("max_eval_operations", self.inner.max_eval_operations)?; let args = PyTuple::new(py, [cls.into_any(), kwargs.into_any()])?; Ok((helper, args.into())) } /// Structural equality — required by the pickle round-trip - /// contract. Compares all six fields; the underlying + /// contract. Compares all ten fields; the underlying /// `CallerLimits` upstream doesn't derive `PartialEq`, so we /// implement equality field-by-field at the binding boundary. fn __eq__(&self, other: &Self) -> bool { @@ -569,6 +631,10 @@ impl PyCallerLimits { && self.inner.max_step_script_size == other.inner.max_step_script_size && self.inner.max_environment_size == other.inner.max_environment_size && self.inner.max_template_size == other.inner.max_template_size + && self.inner.max_resolved_arg_len == other.inner.max_resolved_arg_len + && self.inner.max_resolved_data_len == other.inner.max_resolved_data_len + && self.inner.max_eval_memory_bytes == other.inner.max_eval_memory_bytes + && self.inner.max_eval_operations == other.inner.max_eval_operations } } diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index 9c13c034..61e3c492 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -13,6 +13,7 @@ use pyo3_stub_gen::derive::*; use openjd_model::types::{JobParameterType, JobParameterValue, JobParameterValues}; use openjd_model::types::{TaskParameterSet, TaskParameterType, TaskParameterValue}; use openjd_sessions::action_status::ActionStatus; +use openjd_sessions::limits::SessionLimits; use openjd_sessions::session::{Session, SessionConfig, SessionState}; use super::errors::session_err_to_py; @@ -284,8 +285,15 @@ impl PySession { #[cfg_attr(feature = "stub-gen", gen_stub_pymethods)] #[pymethods] impl PySession { + /// ``caller_limits`` carries the run-time half of a submitting + /// service's policy: the resolved-value caps and evaluation + /// budgets. A session is the enforcement boundary for them — a + /// worker can run a job that never passed through this process's + /// template validation or job creation — so pass the same + /// ``CallerLimits`` value used there. Fields with no run-time + /// meaning (document sizes, step and task counts) are ignored. #[new] - #[pyo3(signature = (*, session_id, job_parameter_values, path_mapping_rules=None, retain_working_dir=false, os_env_vars=None, session_root_directory=None, user=None, profile=None))] + #[pyo3(signature = (*, session_id, job_parameter_values, path_mapping_rules=None, retain_working_dir=false, os_env_vars=None, session_root_directory=None, user=None, profile=None, caller_limits=None))] #[allow(clippy::too_many_arguments)] // PyO3 #[new] mirrors a kwarg-rich public constructor fn new( session_id: String, @@ -296,6 +304,7 @@ impl PySession { session_root_directory: Option, user: Option<&Bound<'_, PyAny>>, profile: Option, + caller_limits: Option<&crate::model::profile::PyCallerLimits>, ) -> PyResult { let params = extract_job_parameter_values(job_parameter_values)?; @@ -356,6 +365,9 @@ impl PySession { sticky_bit_policy: Default::default(), debug_collect_stdout: false, echo_openjd_directives: true, + limits: caller_limits + .map(|c| SessionLimits::from(&c.inner)) + .unwrap_or_default(), }; let session = Session::with_config(config).map_err(session_err_to_py)?; { diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index d47d685f..b0726fa6 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -823,10 +823,10 @@ except FormatStringValidationError as e: The error message embeds the ``[start, end]`` byte offsets of the failing ``{{...}}`` pair so callers can produce structured diagnostics or syntax-highlight the failing segment. Mirrors the -Rust crate's -``FormatString::validate_expressions(symtab, lib, target_type)``, -which the binding calls with ``target_type=None`` to match ``resolve`` -and ``resolve_string``. The crate returns a ``StaticResolution``; the +Rust crate's ``FormatString::validate_expressions(symtab, opts)``, +which the binding calls with the same options ``resolve`` and +``resolve_string`` use: the library, no target type, and the default +evaluation budgets. The crate returns a ``StaticResolution``; the binding is pass/fail only and discards it. **Equality and hashability.** `FormatString` implements `__eq__` and diff --git a/specs/python-model-interface.md b/specs/python-model-interface.md index 96e587f2..f20726dd 100644 --- a/specs/python-model-interface.md +++ b/specs/python-model-interface.md @@ -151,7 +151,9 @@ Argument semantics (matching the Rust API): list is rejected with `Unsupported extension names: ...`. Pass `None` (the default) for an empty allowlist. * `caller_limits` — optional `CallerLimits` to tighten spec-defined - limits (max steps, max envs, max task count, max template size, …). + limits (max steps, max envs, max task count, max template size, …), + to cap resolved argument and embedded-file-data lengths, and to + lower the per-expression evaluation budgets. The `ModelProfile` type is used as an *output* of decoding (via `JobTemplate.profile`) and as an *input* to other functions @@ -211,9 +213,11 @@ env_template = decode_environment_template(template={ }) ``` -``decode_environment_template`` accepts ``supported_extensions`` -with the same semantics as :func:`decode_job_template`. Environment -templates do not accept ``caller_limits``. +``decode_environment_template`` accepts ``supported_extensions`` and +``caller_limits`` with the same semantics as +:func:`decode_job_template`. The document-shape caps that only a job +template has (step and task counts) do not apply here; the +resolved-value caps and evaluation budgets do. #### `decode_environment_template_str` @@ -235,8 +239,9 @@ env_template = decode_environment_template_str(yaml_str) ``` Same defaults as :func:`decode_job_template_str`: ``format`` -defaults to ``DocumentType.YAML``. Accepts ``supported_extensions``; -environment templates do not accept ``caller_limits``. +defaults to ``DocumentType.YAML``. Accepts ``supported_extensions`` +and ``caller_limits`` on the same terms as +:func:`decode_environment_template`. ### Job Creation @@ -397,11 +402,21 @@ result["a"].item(), result["b"].item(), result["c"].item() # (11, 22, 33) ``` The full signature is -``evaluate_let_bindings(bindings, symtab, *, profile=None) -> SymbolTable``. +``evaluate_let_bindings(bindings, symtab, *, profile=None, +caller_limits=None) -> SymbolTable``. ``profile`` accepts an [``ExprProfile``][profile] when the caller needs a non-default revision / extension set or a configured ``HostContext``; omitting it uses the current profile. +``caller_limits`` supplies the evaluation budgets +(``max_eval_memory_bytes``, ``max_eval_operations``). A binding +evaluates a parsed expression directly rather than resolving a format +string, so a caller enforcing those budgets elsewhere has to pass the +same ``CallerLimits`` here for them to bound every evaluation +uniformly. The remaining fields have no meaning for a let binding and +are ignored; omitting the argument uses the spec-recommended +defaults. + [profile]: ./python-expr-interface.md#exprrevision--exprextension--hostcontext--exprprofile A binding without ``=`` raises ``ExpressionError`` with the @@ -1381,6 +1396,37 @@ from openjd.expr import HostContext expr_profile = profile.to_expr_profile(HostContext.unresolved()) ``` +`CallerLimits` fields, all optional and all keyword-only: + +| Field | Bounds | Enforced | +|---|---|---| +| `max_step_count` | steps in a job template | decode | +| `max_env_count` | job + step environments | decode | +| `max_task_count` | total tasks across all steps | `create_job`, after ranges resolve | +| `max_step_script_size` | JSON-encoded step script, bytes | decode | +| `max_environment_size` | JSON-encoded environment, bytes | decode | +| `max_template_size` | whole document, bytes | the `*_str` entry points, before parsing | +| `max_resolved_arg_len` | each resolved `command` and argv entry, characters (§5.1, §5.2) | decode, `create_job`, session | +| `max_resolved_data_len` | each resolved embedded-file `data`, characters (§6.1.2) | decode, `create_job`, session | +| `max_eval_memory_bytes` | memory per expression evaluation (default 100 MB) | decode, `create_job`, session | +| `max_eval_operations` | operations per expression evaluation (default 10 million) | decode, `create_job`, session | + +`max_template_size` measures a document string, so it applies to +`decode_job_template_str` and `decode_environment_template_str` and has +nothing to measure on the dict entry points, which are handed an +already-parsed mapping. + +The last four have no spec-defined maximum to tighten — the two +resolved-value caps stand in for OS limits the spec defers to, and the +two budgets are the Expression Language spec's memory-bounded +evaluation levers, which have recommended defaults rather than limits. +A resolved-value cap is checked three times: at decode against the +guaranteed lower bound of every possible resolution, at `create_job` +with the job parameters bound, and at run time on the final value by a +session the caller passed the same limits to. The session is the +enforcement boundary, since a worker can run a job that never passed +through this process. + `ModelExtension` members: | Member | String form | Notes | @@ -1570,7 +1616,7 @@ original. | ``TaskParameterType`` | variant name (``INT``, ``CHUNK_INT``, …) | | ``ModelExtension`` | variant name (``EXPR``, ``TASK_CHUNKING``, …) | | ``ModelProfile`` | constructor arguments (``revision``, ``extensions``) | -| ``CallerLimits`` | constructor arguments (six optional fields) | +| ``CallerLimits`` | constructor arguments (ten optional fields) | | ``ValidationContext`` | constructor arguments (``profile``, ``caller_limits``) | | ``JobParameterValue`` | constructor arguments (``type``, ``value``) | | ``TaskParameterValue`` | constructor arguments (``type``, ``value``) | diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index d883bb4f..243c6182 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -107,6 +107,7 @@ class Session: user: Optional[SessionUser] = None, job_template: Optional[JobTemplate] = None, environment_templates: Optional[list[EnvironmentTemplate]] = None, + caller_limits: Optional[CallerLimits] = None, ): ... # Properties @@ -164,6 +165,16 @@ class Session: def cleanup(self) -> None: ... ``` +`caller_limits` takes the same +[`CallerLimits`](./python-model-interface.md#modelprofile--modelextension--specificationrevision--callerlimits--validationcontext) +a submitting service passes to `decode_job_template` and `create_job`. +The session applies its run-time half — the resolved-value caps and the +per-expression evaluation budgets — to the values it resolves, and +ignores the fields with no run-time meaning. This is the enforcement +boundary for those caps: a worker can run a job that never passed +through the validating process, so a session that is given no limits +enforces nothing beyond the spec. + ### SessionUser (stays in Python, passed to Rust) The `SessionUser` types involve OS-specific credential validation (sudo diff --git a/src/openjd/_openjd_rs.pyi b/src/openjd/_openjd_rs.pyi index 8586f25f..ea583441 100644 --- a/src/openjd/_openjd_rs.pyi +++ b/src/openjd/_openjd_rs.pyi @@ -305,6 +305,25 @@ class CallerLimits: beyond the spec-defined limit." Caller limits can only tighten spec-defined limits, never relax them. + Two of the fields are resolved-value caps rather than document + caps: ``max_resolved_arg_len`` bounds each resolved action + ``command`` and each argv entry an ``args`` element produces + (Template Schemas §5.1, §5.2), and ``max_resolved_data_len`` + bounds each resolved embedded-file ``data`` value (§6.1.2). The + spec sets no maximum for either; both count **characters**, while + the operating-system limits they stand in for are measured in + bytes or UTF-16 code units, so leave encoding headroom. Both are + checked at template validation against the guaranteed lower bound + of every possible resolution, again at job creation with + parameters bound, and — for a caller that mirrors them into a + session — at run time on the final values. + + ``max_eval_memory_bytes`` and ``max_eval_operations`` are the + Expression Language spec's memory-bounded-evaluation budgets, + applied per format-string expression. ``None`` uses the + spec-recommended defaults (100 MB and 10 million operations); + lowering them is spec-sanctioned configuration. + Mirrors `openjd_model::CallerLimits`. """ @@ -320,6 +339,14 @@ class CallerLimits: def max_environment_size(self) -> typing.Optional[builtins.int]: ... @property def max_template_size(self) -> typing.Optional[builtins.int]: ... + @property + def max_resolved_arg_len(self) -> typing.Optional[builtins.int]: ... + @property + def max_resolved_data_len(self) -> typing.Optional[builtins.int]: ... + @property + def max_eval_memory_bytes(self) -> typing.Optional[builtins.int]: ... + @property + def max_eval_operations(self) -> typing.Optional[builtins.int]: ... def __new__( cls, *, @@ -329,6 +356,10 @@ class CallerLimits: max_step_script_size: typing.Optional[builtins.int] = None, max_environment_size: typing.Optional[builtins.int] = None, max_template_size: typing.Optional[builtins.int] = None, + max_resolved_arg_len: typing.Optional[builtins.int] = None, + max_resolved_data_len: typing.Optional[builtins.int] = None, + max_eval_memory_bytes: typing.Optional[builtins.int] = None, + max_eval_operations: typing.Optional[builtins.int] = None, ) -> CallerLimits: ... def __repr__(self) -> builtins.str: ... def __reduce__(self) -> tuple[typing.Any, tuple]: @@ -934,12 +965,13 @@ class FormatString: checking through the expression tree. Mirrors the Rust crate's - `FormatString::validate_expressions(symtab, lib, target_type)`. + `FormatString::validate_expressions(symtab, opts)`. Returns `None` on success. - `target_type` is passed as `None` because `resolve` and - `resolve_string` above resolve without one; validation has to - observe the same values resolution will produce. The crate + The options carry the library only — the same ones `resolve` + and `resolve_string` above build, since validation has to + observe the values resolution will produce, including its lack + of a target type and its default evaluation budgets. The crate returns a `StaticResolution` (resolved-length bound and, when fully concrete, the resolved value); this binding is pass/fail only and discards it. @@ -2130,7 +2162,18 @@ class Session: session_root_directory: typing.Optional[builtins.str | os.PathLike | pathlib.Path] = None, user: typing.Optional[typing.Any] = None, profile: typing.Optional[ModelProfile] = None, - ) -> Session: ... + caller_limits: typing.Optional[CallerLimits] = None, + ) -> Session: + r""" + ``caller_limits`` carries the run-time half of a submitting + service's policy: the resolved-value caps and evaluation + budgets. A session is the enforcement boundary for them — a + worker can run a job that never passed through this process's + template validation or job creation — so pass the same + ``CallerLimits`` value used there. Fields with no run-time + meaning (document sizes, step and task counts) are ignored. + """ + def extend_path_mapping_rules(self, additional: typing.Sequence[PathMappingRule]) -> None: r""" Extend the session's path mapping rules with additional rules. @@ -3504,7 +3547,10 @@ def create_job( validation_context: typing.Optional[ValidationContext] = None, ) -> Job: ... def decode_environment_template( - template: dict, *, supported_extensions: typing.Optional[typing.Sequence[builtins.str]] = None + template: dict, + *, + supported_extensions: typing.Optional[typing.Sequence[builtins.str]] = None, + caller_limits: typing.Optional[CallerLimits] = None, ) -> EnvironmentTemplate: r""" Decode and validate an environment template from a Python dict. @@ -3524,10 +3570,14 @@ def decode_environment_template( ``Unsupported extension names: ...``. Pass ``None`` (the default) for an empty allowlist (i.e., reject every extension the template requests). + caller_limits: Optional ``CallerLimits`` to tighten + spec-defined limits. The caps that only a job template has + (step and task counts) do not apply here, and + ``max_template_size`` has no document string to measure; + the resolved-value caps and evaluation budgets do apply. Returns: The parsed ``openjd.model._v1.template.EnvironmentTemplate``. - Environment templates do not accept ``caller_limits``. """ def decode_environment_template_str( @@ -3535,6 +3585,7 @@ def decode_environment_template_str( format: DocumentType = DocumentType.YAML, *, supported_extensions: typing.Optional[typing.Sequence[builtins.str]] = None, + caller_limits: typing.Optional[CallerLimits] = None, ) -> EnvironmentTemplate: r""" Decode and validate an environment template from a YAML or JSON string. @@ -3556,10 +3607,15 @@ def decode_environment_template_str( ``Unsupported extension names: ...``. Pass ``None`` (the default) for an empty allowlist (i.e., reject every extension the template requests). + caller_limits: Optional ``CallerLimits`` to tighten + spec-defined limits. The caps that only a job template has + (step and task counts) do not apply here; the + resolved-value caps and evaluation budgets do, and + ``max_template_size`` is checked against ``document``'s + byte length before parsing. Returns: The parsed ``openjd.model._v1.template.EnvironmentTemplate``. - Environment templates do not accept ``caller_limits``. """ def decode_job_template( @@ -3588,6 +3644,9 @@ def decode_job_template( every extension the template requests). caller_limits: Optional ``CallerLimits`` to tighten spec-defined limits (e.g. maximum step count). + ``max_template_size`` bounds a document's encoded byte + length, so it has nothing to measure on this entry point + and applies only to ``decode_job_template_str``. Returns: The parsed ``openjd.model._v1.template.JobTemplate``. Use @@ -3624,6 +3683,8 @@ def decode_job_template_str( every extension the template requests). caller_limits: Optional ``CallerLimits`` to tighten spec-defined limits (e.g. maximum step count). + ``max_template_size`` is checked here, against + ``document``'s byte length before parsing. Returns: The parsed ``openjd.model._v1.template.JobTemplate``. Use @@ -3670,7 +3731,21 @@ def evaluate_let_bindings( symtab: SymbolTable, *, profile: typing.Optional[ExprProfile] = None, -) -> SymbolTable: ... + caller_limits: typing.Optional[CallerLimits] = None, +) -> SymbolTable: + r""" + Evaluate ``let`` bindings in order, returning a symbol table with + each bound name added. + + ``caller_limits`` supplies the evaluation budgets: a binding + evaluates a parsed expression directly rather than resolving a + format string, so a caller enforcing ``max_eval_memory_bytes`` / + ``max_eval_operations`` elsewhere has to pass the same limits here + for the budgets to bound every evaluation uniformly. The other + ``CallerLimits`` fields have no meaning for a let binding and are + ignored. Omitting it uses the spec-recommended defaults. + """ + def merge_job_parameter_definitions( *, job_template: JobTemplate, diff --git a/test/openjd/model_v1/test_create_job.py b/test/openjd/model_v1/test_create_job.py index a6ba7aa0..b98888af 100644 --- a/test/openjd/model_v1/test_create_job.py +++ b/test/openjd/model_v1/test_create_job.py @@ -7,6 +7,7 @@ from typing import Any from openjd.model._v1 import ( + CallerLimits, create_job, decode_environment_template, decode_job_template, @@ -16,6 +17,7 @@ from openjd.model._v1.types import ( JobParameterType, JobParameterValue, + ValidationContext, ) from openjd.model._v1.errors import ( DecodeValidationError, @@ -1978,3 +1980,157 @@ def test_all_five_fields_at_ceiling_fb1_create_job_preserved(self) -> None: len(embedded[0].filename), param_name in job.parameters, ) == (512, 512, 512, 256, True) + + +class TestResolvedValueCapsAtJobCreation: + """Companion to ``test_parse.py::TestResolvedValueCapsAtTemplateValidation``. + ``max_resolved_arg_len`` / ``max_resolved_data_len`` (openjd-model 0.9.0, + openjd-rs#399) are checked twice on this path: at template validation against + the guaranteed lower bound of every resolution, then again at job creation with + the job parameters bound. A field whose value comes from a parameter has a lower + bound of 0 at validation, so job creation is the stage that can reject it. + + Caller limits reach ``create_job`` through a ``ValidationContext``; omitting one + uses the template's default context, which carries no limits. + """ + + _LIMITS = CallerLimits(max_resolved_arg_len=10, max_resolved_data_len=10) + + @classmethod + def _decoded(cls, **script_extras: Any) -> Any: + action: dict[str, Any] = {"command": "echo", "args": ["{{Param.P}}"]} + script: dict[str, Any] = {"actions": {"onRun": action}} + script.update(script_extras) + return decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "parameterDefinitions": [{"name": "P", "type": "STRING"}], + "steps": [{"name": "S", "script": script}], + }, + supported_extensions=[], + caller_limits=cls._LIMITS, + ) + + @classmethod + def _context(cls, decoded: Any) -> Any: + return ValidationContext(decoded.profile, caller_limits=cls._LIMITS) + + def test_template_passes_validation_because_the_bound_is_unknown(self) -> None: + """Control for the deferral: the cap is 10 and the only literal run in the + argument is empty, so validation cannot reject.""" + assert self._decoded() + + def test_a_parameter_value_over_the_cap_is_rejected(self) -> None: + decoded = self._decoded() + with pytest.raises(ModelValidationError) as excinfo: + create_job( + job_template=decoded, + job_parameter_values={"P": "c" * 30}, + validation_context=self._context(decoded), + ) + message = str(excinfo.value) + assert "steps[0] -> script -> actions -> onRun -> args[0]" in message + assert "resolves to at least 30 characters, exceeding the maximum of 10" in message + + def test_a_parameter_value_under_the_cap_is_accepted(self) -> None: + """Negative control: the same template and context with a value that fits. + + The argument stays a ``FormatString`` on the created job — job creation + checks the resolved length without substituting it, because an action's + arguments also depend on task parameters and resolve in the session.""" + decoded = self._decoded() + job = create_job( + job_template=decoded, + job_parameter_values={"P": "short"}, + validation_context=self._context(decoded), + ) + args = job.steps[0].script.actions.onRun.args + assert args is not None + assert [str(a) for a in args] == ["{{Param.P}}"] + assert "P" in job.parameters + + def test_without_caller_limits_any_length_is_accepted(self) -> None: + """Negative control: with no limits in the context the value passes, which is + the only behaviour openjd-model 0.8.0 had.""" + decoded = self._decoded() + assert create_job(job_template=decoded, job_parameter_values={"P": "c" * 30}) + + def test_embedded_file_data_over_the_cap_is_rejected(self) -> None: + decoded = self._decoded( + embeddedFiles=[{"name": "F", "type": "TEXT", "data": "{{Param.P}}"}] + ) + with pytest.raises(ModelValidationError) as excinfo: + create_job( + job_template=decoded, + job_parameter_values={"P": "d" * 30}, + validation_context=self._context(decoded), + ) + message = str(excinfo.value) + assert "steps[0] -> script -> embeddedFiles[0] -> data" in message + assert "resolves to at least 30 characters, exceeding the maximum of 10" in message + + +class TestJobEnvironmentResolvedValueCapsAtJobCreation: + """openjd-model 0.9.0 (openjd-rs#404) re-runs the resolved-value checks on the + job environments that ``create_job`` carries forward into the job, against a + session-scope symbol table. A job environment's script is not part of a step, so + the step-walking re-check of openjd-rs#399 alone did not reach it: on 0.9.0 + without #404 the same template would produce a job whose ``onEnter`` argument is + 30 characters under a cap of 10. + """ + + _LIMITS = CallerLimits(max_resolved_arg_len=10) + + @classmethod + def _decoded(cls) -> Any: + return decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "parameterDefinitions": [{"name": "P", "type": "STRING"}], + "jobEnvironments": [ + { + "name": "JobEnv", + "script": { + "actions": {"onEnter": {"command": "echo", "args": ["{{Param.P}}"]}} + }, + } + ], + "steps": [{"name": "S", "script": {"actions": {"onRun": {"command": "echo"}}}}], + }, + supported_extensions=[], + caller_limits=cls._LIMITS, + ) + + def test_job_environment_argument_over_the_cap_is_rejected(self) -> None: + decoded = self._decoded() + with pytest.raises(ModelValidationError) as excinfo: + create_job( + job_template=decoded, + job_parameter_values={"P": "e" * 30}, + validation_context=ValidationContext(decoded.profile, caller_limits=self._LIMITS), + ) + message = str(excinfo.value) + assert "jobEnvironments[0] -> script -> actions -> onEnter -> args[0]" in message + assert "resolves to at least 30 characters, exceeding the maximum of 10" in message + + def test_job_environment_argument_under_the_cap_is_accepted(self) -> None: + """Negative control, and it pins that the environment is still carried + forward onto the job with its argument unsubstituted — the check reads the + resolved length, it does not rewrite the field.""" + decoded = self._decoded() + job = create_job( + job_template=decoded, + job_parameter_values={"P": "short"}, + validation_context=ValidationContext(decoded.profile, caller_limits=self._LIMITS), + ) + environments = job.jobEnvironments + assert environments is not None + script = environments[0].script + assert script is not None + on_enter = script.actions.onEnter + assert on_enter is not None + args = on_enter.args + assert args is not None + assert [str(a) for a in args] == ["{{Param.P}}"] diff --git a/test/openjd/model_v1/test_let_bindings.py b/test/openjd/model_v1/test_let_bindings.py index 541aa464..8cebafe7 100644 --- a/test/openjd/model_v1/test_let_bindings.py +++ b/test/openjd/model_v1/test_let_bindings.py @@ -23,7 +23,7 @@ import pytest from openjd.expr import ExpressionError, ExprProfile, SymbolTable -from openjd.model._v1 import evaluate_let_bindings +from openjd.model._v1 import CallerLimits, evaluate_let_bindings class TestEvaluateLetBindings: @@ -147,3 +147,51 @@ def test_with_explicit_profile(self) -> None: st = SymbolTable({"Param.X": 10}) result = evaluate_let_bindings(["y = Param.X + 5"], st, profile=ExprProfile.current()) assert result["y"].item() == 15 + + +class TestEvaluateLetBindingsCallerLimits: + """``caller_limits`` is the fourth axis, added with openjd-model 0.9.0 + (openjd-rs#399); 0.8.0's ``evaluate_let_bindings`` took no budgets at all. + + A binding evaluates a parsed expression directly rather than resolving a format + string, so without this argument a caller that lowered + ``max_eval_memory_bytes`` / ``max_eval_operations`` for template validation and + job creation would leave let bindings evaluating on the spec-recommended + defaults. The remaining ``CallerLimits`` fields have no meaning here. + """ + + def test_operation_budget_is_applied(self) -> None: + st = SymbolTable({"Param.X": 10}) + with pytest.raises(ExpressionError) as excinfo: + evaluate_let_bindings( + ["a = Param.X + 1"], st, caller_limits=CallerLimits(max_eval_operations=1) + ) + message = str(excinfo.value) + assert message.startswith("Error evaluating let binding 'a':") + assert "operation count (2) exceeded limit (1)" in message + + def test_memory_budget_is_applied(self) -> None: + st = SymbolTable({"Param.X": 10}) + with pytest.raises(ExpressionError) as excinfo: + evaluate_let_bindings( + ["a = 'z' * 10000"], st, caller_limits=CallerLimits(max_eval_memory_bytes=16) + ) + assert "memory usage (72 bytes) exceeded limit (16 bytes)" in str(excinfo.value) + + def test_unrelated_limits_do_not_restrict_a_binding(self) -> None: + """Negative control: a ``CallerLimits`` carrying only document-shape caps + leaves the binding on the spec-recommended defaults, and a binding that + exceeds neither default is evaluated.""" + st = SymbolTable({"Param.X": 10}) + result = evaluate_let_bindings( + ["a = 'z' * 10000"], + st, + caller_limits=CallerLimits(max_step_count=1, max_template_size=1), + ) + assert len(result["a"].item()) == 10000 + + def test_omitting_the_argument_uses_the_defaults(self) -> None: + """Negative control for the same expression with no limits at all — the only + behaviour available before the bump.""" + result = evaluate_let_bindings(["a = 'z' * 10000"], SymbolTable({})) + assert len(result["a"].item()) == 10000 diff --git a/test/openjd/model_v1/test_parse.py b/test/openjd/model_v1/test_parse.py index dbca11f8..917ef58d 100644 --- a/test/openjd/model_v1/test_parse.py +++ b/test/openjd/model_v1/test_parse.py @@ -663,3 +663,283 @@ def test_a_whole_field_expression_allof_element_is_deferred_to_job_creation(self resolved count, so the constraint is deferred, not dropped.""" template = self._os_family_all_of(["linux", "{{ Param.X }}"]) assert decode_job_template(template=template, supported_extensions=["EXPR"]) + + +class TestResolvedValueCapsAtTemplateValidation(object): + """``max_resolved_arg_len`` and ``max_resolved_data_len`` cap the length of a + resolved action ``command`` / argv entry (Template Schemas §5.1, §5.2) and of a + resolved embedded-file ``data`` value (§6.1.2). The spec sets no maximum for + either and defers to the operating system, so these are caller policy, not + conformance rules. openjd-model 0.8.0 had no such fields — a caller could not + express either cap — so every case here was accepted before the bump + (openjd-rs#399). + + Validation checks the guaranteed lower bound of every possible resolution: a + literal is exact, so it is rejected here, while a value that depends on a job + parameter is only bounded by its literal runs and is deferred to job creation + (see ``TestResolvedValueCapsAtJobCreation`` in ``test_create_job.py``). + """ + + @staticmethod + def _template(action: dict[str, Any], embedded: list[dict[str, Any]] | None = None) -> dict: + script: dict[str, Any] = {"actions": {"onRun": action}} + if embedded is not None: + script["embeddedFiles"] = embedded + return { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "steps": [{"name": "S", "script": script}], + } + + def test_literal_command_over_cap_is_rejected(self) -> None: + template = self._template({"command": "a" * 20}) + with pytest.raises(ModelValidationError) as excinfo: + decode_job_template( + template=template, + supported_extensions=[], + caller_limits=CallerLimits(max_resolved_arg_len=5), + ) + message = str(excinfo.value) + assert "steps[0] -> script -> actions -> onRun -> command" in message + assert "is 20 characters, exceeding the maximum of 5" in message + + def test_literal_arg_over_cap_is_rejected(self) -> None: + """The cap applies per argv entry, not to the ``args`` list as a whole.""" + template = self._template({"command": "echo", "args": ["b" * 20]}) + with pytest.raises(ModelValidationError) as excinfo: + decode_job_template( + template=template, + supported_extensions=[], + caller_limits=CallerLimits(max_resolved_arg_len=5), + ) + message = str(excinfo.value) + assert "steps[0] -> script -> actions -> onRun -> args[0]" in message + assert "is 20 characters, exceeding the maximum of 5" in message + + def test_command_under_cap_is_accepted(self) -> None: + """Negative control: the same template passes under a cap it fits.""" + template = self._template({"command": "a" * 20, "args": ["b" * 20]}) + assert decode_job_template( + template=template, + supported_extensions=[], + caller_limits=CallerLimits(max_resolved_arg_len=100), + ) + + def test_no_cap_accepts_any_length(self) -> None: + """Negative control: omitting the cap imposes no limit, which is the only + behaviour 0.8.0 had.""" + assert decode_job_template( + template=self._template({"command": "a" * 20}), supported_extensions=[] + ) + + def test_embedded_file_data_over_cap_is_rejected(self) -> None: + template = self._template( + {"command": "echo"}, + embedded=[{"name": "F", "type": "TEXT", "data": "d" * 20}], + ) + with pytest.raises(ModelValidationError) as excinfo: + decode_job_template( + template=template, + supported_extensions=[], + caller_limits=CallerLimits(max_resolved_data_len=5), + ) + message = str(excinfo.value) + assert "steps[0] -> script -> embeddedFiles[0] -> data" in message + assert "is 20 characters, exceeding the maximum of 5" in message + + def test_embedded_file_data_under_cap_is_accepted(self) -> None: + """Negative control, and it also pins that the two caps are independent: a + 20-character ``data`` passes while ``max_resolved_arg_len`` is 5.""" + template = self._template( + {"command": "echo"}, + embedded=[{"name": "F", "type": "TEXT", "data": "d" * 20}], + ) + assert decode_job_template( + template=template, + supported_extensions=[], + caller_limits=CallerLimits(max_resolved_arg_len=5, max_resolved_data_len=100), + ) + + +class TestEvaluationBudgetsAtTemplateValidation(object): + """``max_eval_memory_bytes`` and ``max_eval_operations`` are the Expression + Language spec's memory-bounded-evaluation budgets (§1.3.9, §1.3.10), applied per + format-string expression. Both have spec-recommended defaults (100 MB, 10 + million) rather than limits, so lowering them is configuration. openjd-model + 0.8.0 exposed no way to lower either (openjd-rs#399); the expression below was + accepted. + """ + + @staticmethod + def _template(expression: str) -> dict[str, Any]: + return { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "extensions": ["EXPR"], + "steps": [ + { + "name": "S", + "script": {"actions": {"onRun": {"command": "echo", "args": [expression]}}}, + } + ], + } + + def test_memory_budget_rejects_a_large_value(self) -> None: + with pytest.raises(ModelValidationError) as excinfo: + decode_job_template( + template=self._template("{{ 'a' * 100000 }}"), + supported_extensions=["EXPR"], + caller_limits=CallerLimits(max_eval_memory_bytes=1024), + ) + assert "memory usage (100136 bytes) exceeded limit (1024 bytes)" in str(excinfo.value) + + def test_operation_budget_rejects_a_long_evaluation(self) -> None: + with pytest.raises(ModelValidationError) as excinfo: + decode_job_template( + template=self._template("{{ 'a' * 100000 }}"), + supported_extensions=["EXPR"], + caller_limits=CallerLimits(max_eval_operations=5), + ) + assert "operation count (392) exceeded limit (5)" in str(excinfo.value) + + def test_default_budgets_accept_it(self) -> None: + """Negative control: the same expression is well within the spec-recommended + defaults, so omitting the budgets accepts it.""" + assert decode_job_template( + template=self._template("{{ 'a' * 100000 }}"), supported_extensions=["EXPR"] + ) + + +class TestEnvironmentTemplateCallerLimits(object): + """``decode_environment_template`` gained a ``caller_limits`` argument with + openjd-model 0.9.0 (openjd-rs#399); 0.8.0 took only the extension allowlist, and + this package's docstrings said environment templates do not accept caller limits. + The document-shape caps have no environment-template counterpart, but the + resolved-value caps and evaluation budgets apply to its script the same way. + """ + + _TEMPLATE: dict[str, Any] = { + "specificationVersion": "environment-2023-09", + "environment": { + "name": "E", + "script": {"actions": {"onEnter": {"command": "e" * 20}}}, + }, + } + + def test_dict_entry_point_applies_the_cap(self) -> None: + with pytest.raises(ModelValidationError) as excinfo: + decode_environment_template( + template=self._TEMPLATE, + supported_extensions=[], + caller_limits=CallerLimits(max_resolved_arg_len=5), + ) + message = str(excinfo.value) + assert "environment -> script -> actions -> onEnter -> command" in message + assert "is 20 characters, exceeding the maximum of 5" in message + + def test_str_entry_point_applies_the_cap(self) -> None: + document = json.dumps(self._TEMPLATE) + with pytest.raises(ModelValidationError) as excinfo: + decode_environment_template_str( + document, + DocumentType.JSON, + supported_extensions=[], + caller_limits=CallerLimits(max_resolved_arg_len=5), + ) + assert "is 20 characters, exceeding the maximum of 5" in str(excinfo.value) + + def test_cap_that_fits_is_accepted(self) -> None: + """Negative control on both entry points.""" + assert decode_environment_template( + template=self._TEMPLATE, + supported_extensions=[], + caller_limits=CallerLimits(max_resolved_arg_len=100), + ) + assert decode_environment_template_str( + json.dumps(self._TEMPLATE), DocumentType.JSON, supported_extensions=[] + ) + + +class TestMaxTemplateSizeReachesTheParser(object): + """``max_template_size`` is checked in one place only: the document-string parse + in ``openjd_model::template::parse::document_string_to_object``, against the + byte length before parsing. + + This binding's ``parse_string`` helper passed ``CallerLimits::default()`` there, + so the field was inert on both ``*_str`` entry points even though they accept it + — a caller asking for a 10-byte ceiling got no ceiling. Found while documenting + the field, and fixed by passing the caller's own limits. The dict entry points + are handed an already-parsed mapping and have no document string to measure, so + they are unaffected either way. + """ + + _JOB = ( + "specificationVersion: jobtemplate-2023-09\n" + "name: T\n" + "steps:\n" + " - name: S\n" + " script:\n" + " actions:\n" + " onRun:\n" + " command: echo\n" + ) + _ENV = ( + "specificationVersion: environment-2023-09\n" + "environment:\n" + " name: E\n" + " script:\n" + " actions:\n" + " onEnter:\n" + " command: echo\n" + ) + + def test_job_template_str_over_the_limit_is_rejected(self) -> None: + with pytest.raises(ModelValidationError) as excinfo: + decode_job_template_str( + self._JOB, + supported_extensions=[], + caller_limits=CallerLimits(max_template_size=10), + ) + assert f"Template document size ({len(self._JOB)} bytes) exceeds caller limit of 10" in str( + excinfo.value + ) + + def test_environment_template_str_over_the_limit_is_rejected(self) -> None: + with pytest.raises(ModelValidationError) as excinfo: + decode_environment_template_str( + self._ENV, + supported_extensions=[], + caller_limits=CallerLimits(max_template_size=10), + ) + assert f"Template document size ({len(self._ENV)} bytes) exceeds caller limit of 10" in str( + excinfo.value + ) + + def test_a_limit_the_document_fits_is_accepted(self) -> None: + """Negative control: the check is a ceiling, not a rejection of the field.""" + assert decode_job_template_str( + self._JOB, + supported_extensions=[], + caller_limits=CallerLimits(max_template_size=len(self._JOB)), + ) + assert decode_environment_template_str( + self._ENV, + supported_extensions=[], + caller_limits=CallerLimits(max_template_size=len(self._ENV)), + ) + + def test_the_dict_entry_points_have_nothing_to_measure(self) -> None: + """The same limit on a dict entry point is inert by construction — there is no + document string — so the template is accepted. Pins the asymmetry the + docstrings now state, so a future change that starts re-encoding the dict to + measure it has to update both.""" + template = { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "steps": [{"name": "S", "script": {"actions": {"onRun": {"command": "echo"}}}}], + } + assert decode_job_template( + template=template, + supported_extensions=[], + caller_limits=CallerLimits(max_template_size=10), + ) diff --git a/test/openjd/model_v1/test_pickle.py b/test/openjd/model_v1/test_pickle.py index 3c853099..059a51ea 100644 --- a/test/openjd/model_v1/test_pickle.py +++ b/test/openjd/model_v1/test_pickle.py @@ -131,6 +131,8 @@ def test_caller_limits_round_trip_default(): loaded = pickle.loads(pickle.dumps(limits)) assert loaded.max_step_count is None assert loaded.max_template_size is None + assert loaded.max_resolved_arg_len is None + assert loaded.max_eval_operations is None def test_caller_limits_round_trip_populated(): @@ -143,6 +145,10 @@ def test_caller_limits_round_trip_populated(): max_step_script_size=2048, max_environment_size=1024, max_template_size=4096, + max_resolved_arg_len=32768, + max_resolved_data_len=65536, + max_eval_memory_bytes=1_048_576, + max_eval_operations=100_000, ) loaded = pickle.loads(pickle.dumps(limits)) assert loaded.max_step_count == 10 @@ -151,6 +157,13 @@ def test_caller_limits_round_trip_populated(): assert loaded.max_step_script_size == 2048 assert loaded.max_environment_size == 1024 assert loaded.max_template_size == 4096 + assert loaded.max_resolved_arg_len == 32768 + assert loaded.max_resolved_data_len == 65536 + assert loaded.max_eval_memory_bytes == 1_048_576 + assert loaded.max_eval_operations == 100_000 + # Equality covers all ten fields, so this also pins that none was dropped + # from __reduce__: a field lost there would come back None and compare unequal. + assert loaded == limits def test_validation_context_round_trip(): diff --git a/test/openjd/sessions/test_caller_limits.py b/test/openjd/sessions/test_caller_limits.py new file mode 100644 index 00000000..66611e0a --- /dev/null +++ b/test/openjd/sessions/test_caller_limits.py @@ -0,0 +1,79 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``Session(caller_limits=...)`` applies the run-time half of a caller's policy. + +openjd-sessions 0.7.0 (openjd-rs#399) added ``SessionConfig.limits``, and this +binding fills it from a ``CallerLimits`` — the same value a submitting service +passes to ``decode_job_template`` and ``create_job``. A session is the +enforcement boundary for the resolved-value caps: a worker can run a job that +never passed through the validating process, and only the session sees the final +resolved value. 0.6.0 had no such field, so a session enforced nothing beyond +the spec. + +These tests run a real subprocess, which is why they live here rather than +alongside the model-layer cap tests in ``test/openjd/model_v1``. +""" + +from __future__ import annotations + +import time +from typing import Any, Optional + +from openjd._openjd_rs import ActionState, CallerLimits, Session, deserialize_step + + +def _step(arg: str) -> Any: + """A job-side ``Step`` whose ``onRun`` passes ``arg`` to ``echo``.""" + return deserialize_step( + { + "name": "S", + "script": {"actions": {"onRun": {"command": "echo", "args": [arg]}}}, + } + ) + + +def _run_task(arg: str, caller_limits: Optional[CallerLimits]) -> tuple[ActionState, str]: + """Run one task to completion; return its final state and message.""" + session = Session( + session_id=f"caller-limits-{time.time_ns()}", + job_parameter_values={}, + caller_limits=caller_limits, + ) + try: + session.run_task(step_script=_step(arg).script) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + status = session.action_status + if status is not None and status.state != ActionState.RUNNING: + break + time.sleep(0.05) + status = session.action_status + assert status is not None, "action never reported a status" + assert status.state != ActionState.RUNNING, "action did not finish within 30s" + return status.state, status.fail_message or status.status_message or "" + finally: + session.cleanup() + + +class TestSessionResolvedArgLengthCap: + def test_an_argument_over_the_cap_fails_the_action(self) -> None: + state, message = _run_task("a" * 40, CallerLimits(max_resolved_arg_len=5)) + assert state == ActionState.FAILED + assert "resolved value is 40 characters, exceeding the maximum of 5" in message + + def test_an_argument_under_the_cap_runs(self) -> None: + """Negative control: the same argument under a cap it fits.""" + state, _ = _run_task("a" * 40, CallerLimits(max_resolved_arg_len=99)) + assert state == ActionState.SUCCESS + + def test_omitting_caller_limits_enforces_nothing(self) -> None: + """Negative control, and the only behaviour openjd-sessions 0.6.0 had.""" + state, _ = _run_task("a" * 40, None) + assert state == ActionState.SUCCESS + + def test_an_unrelated_limit_does_not_affect_the_action(self) -> None: + """A ``CallerLimits`` carrying only document-shape caps has no run-time + counterpart, so the session ignores it rather than rejecting the value.""" + state, _ = _run_task("a" * 40, CallerLimits(max_step_count=1, max_template_size=1)) + assert state == ActionState.SUCCESS From 548bfe90a9393da9f3834aa33aeb67993958a41c Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:31:54 -0700 Subject: [PATCH 2/2] fix: Use Optional over PEP 604 in a test annotation for py3.9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The helper's annotation is evaluated at class-body definition time, so `list[dict[str, Any]] | None` raised TypeError on Python 3.9 and the whole module failed collection. Every local interpreter is >= 3.10, where the operator is valid, so only the 3.9 CI leg caught it — and its fail-fast cancelled the 12 macOS and Windows jobs. `Optional` never uses `|`, so it cannot reach that path. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- test/openjd/model_v1/test_parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/openjd/model_v1/test_parse.py b/test/openjd/model_v1/test_parse.py index 917ef58d..9d1b34ab 100644 --- a/test/openjd/model_v1/test_parse.py +++ b/test/openjd/model_v1/test_parse.py @@ -2,7 +2,7 @@ from enum import Enum import json -from typing import Any, Type, Union +from typing import Any, Optional, Type, Union import pytest @@ -681,7 +681,7 @@ class TestResolvedValueCapsAtTemplateValidation(object): """ @staticmethod - def _template(action: dict[str, Any], embedded: list[dict[str, Any]] | None = None) -> dict: + def _template(action: dict[str, Any], embedded: Optional[list[dict[str, Any]]] = None) -> dict: script: dict[str, Any] = {"actions": {"onRun": action}} if embedded is not None: script["embeddedFiles"] = embedded