From 017681b46bf11f201d05a54c938be74b43638fa9 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:42:56 +0000 Subject: [PATCH] bench: pin the open tool PRs' intents to their branches Each PR domain's intents now register that PR's tools, as the amazon.com, fandango.com and ipwho.is intents already do. Where the PR and main both changed a tool file since the PR branched, main is listed after the PR so its copy wins. Adds doordash.com's domain.json from #57, dates the Fandango nearest-theaters intent relative to the run, and points the tasks test's default-to-main check at an intent without a tools entry. --- bench/tasks/aa.com/aa-cheapest-fare.json | 2 +- bench/tasks/amazon.in/amazon-in-search-first-result.json | 1 + bench/tasks/booking.com/booking-cheapest-room.json | 1 + bench/tasks/booking.com/booking-cheapest-well-rated.json | 1 + bench/tasks/booking.com/booking-check-in-times.json | 1 + bench/tasks/booking.com/booking-checkout-room.json | 1 + bench/tasks/booking.com/booking-house-rules.json | 1 + .../tasks/booking.com/booking-reserve-cheapest-room.json | 1 + bench/tasks/delta.com/delta-cart-main-classic.json | 1 + bench/tasks/delta.com/delta-cheapest-fare.json | 2 +- bench/tasks/delta.com/delta-checkout-signed-in.json | 1 + bench/tasks/doordash.com/domain.json | 9 +++++++++ .../tasks/doordash.com/doordash-pizza-near-address.json | 1 + bench/tasks/ebay.com/ebay-checkout-usb-c-cable.json | 1 + bench/tasks/fandango.com/fandango-nearest-theaters.json | 2 +- bench/tasks/google.com/google-flights-cheapest-fare.json | 2 +- bench/tasks/opentable.com/opentable-available-times.json | 1 + bench/tasks/shopify.com/shopify-app-store-compare.json | 1 + bench/tasks/shopify.com/shopify-app-store-listing.json | 1 + bench/tasks/shopify.com/shopify-basic-monthly-price.json | 1 + bench/tasks/target.com/target-checkout-water-bottle.json | 1 + bench/tasks/target.com/target-search-first-result.json | 2 +- .../ticketmaster.com/ticketmaster-concerts-in-city.json | 2 +- bench/tasks/united.com/united-cheapest-fare.json | 2 +- bench/tasks/vividseats.com/vividseats-cheapest-pair.json | 1 + .../tasks/walmart.com/walmart-checkout-water-bottle.json | 1 + bench/test/tasks.test.mjs | 2 +- 27 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 bench/tasks/doordash.com/domain.json diff --git a/bench/tasks/aa.com/aa-cheapest-fare.json b/bench/tasks/aa.com/aa-cheapest-fare.json index 574d0111..d9315c5e 100644 --- a/bench/tasks/aa.com/aa-cheapest-fare.json +++ b/bench/tasks/aa.com/aa-cheapest-fare.json @@ -9,7 +9,7 @@ "date": {"days_from_start": 28} }, "start_url": "https://www.aa.com/", - "tools": [{"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_flights"]}], + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/aa-flight-status", "pr": 46}, {"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_flights"]}], "entry_tool": "search_flights", "answer_schema": "schemas/cheapest-flight.json", "rules": [{"path": "departure_time", "time": true}, {"path": "price", "range": [1, 3000]}], diff --git a/bench/tasks/amazon.in/amazon-in-search-first-result.json b/bench/tasks/amazon.in/amazon-in-search-first-result.json index c37f1ad7..57fe3375 100644 --- a/bench/tasks/amazon.in/amazon-in-search-first-result.json +++ b/bench/tasks/amazon.in/amazon-in-search-first-result.json @@ -7,6 +7,7 @@ "query": "usb c cable" }, "start_url": "https://www.amazon.in/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/amazon-in-shopping", "pr": 62}], "answer_schema": "schemas/asin-search-result.json", "rules": [ { diff --git a/bench/tasks/booking.com/booking-cheapest-room.json b/bench/tasks/booking.com/booking-cheapest-room.json index d95b1bb1..e0c5c1ed 100644 --- a/bench/tasks/booking.com/booking-cheapest-room.json +++ b/bench/tasks/booking.com/booking-cheapest-room.json @@ -14,6 +14,7 @@ } }, "start_url": "https://www.booking.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "main"}, {"repo": "kernel/xwebmcp", "ref": "hypeship/booking-select-room-party", "pr": 129}], "answer_schema": "schemas/room-rate.json", "rules": [ { diff --git a/bench/tasks/booking.com/booking-cheapest-well-rated.json b/bench/tasks/booking.com/booking-cheapest-well-rated.json index 3f40c362..8d129bd0 100644 --- a/bench/tasks/booking.com/booking-cheapest-well-rated.json +++ b/bench/tasks/booking.com/booking-cheapest-well-rated.json @@ -13,6 +13,7 @@ } }, "start_url": "https://www.booking.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "main"}, {"repo": "kernel/xwebmcp", "ref": "hypeship/booking-select-room-party", "pr": 129}], "answer_schema": "schemas/stay-price.json", "rules": [ { diff --git a/bench/tasks/booking.com/booking-check-in-times.json b/bench/tasks/booking.com/booking-check-in-times.json index 1041e183..8e06493d 100644 --- a/bench/tasks/booking.com/booking-check-in-times.json +++ b/bench/tasks/booking.com/booking-check-in-times.json @@ -8,6 +8,7 @@ "city": "Lisbon" }, "start_url": "https://www.booking.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "main"}, {"repo": "kernel/xwebmcp", "ref": "hypeship/booking-select-room-party", "pr": 129}], "answer_schema": "schemas/check-in-times.json", "rules": [ { diff --git a/bench/tasks/booking.com/booking-checkout-room.json b/bench/tasks/booking.com/booking-checkout-room.json index 5aaae9fd..83fce9d9 100644 --- a/bench/tasks/booking.com/booking-checkout-room.json +++ b/bench/tasks/booking.com/booking-checkout-room.json @@ -21,6 +21,7 @@ "zip": "94133" }, "start_url": "https://www.booking.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "main"}, {"repo": "kernel/xwebmcp", "ref": "hypeship/booking-select-room-party", "pr": 129}], "allow": [ "checkout" ], diff --git a/bench/tasks/booking.com/booking-house-rules.json b/bench/tasks/booking.com/booking-house-rules.json index bc5083d5..3e2ddee4 100644 --- a/bench/tasks/booking.com/booking-house-rules.json +++ b/bench/tasks/booking.com/booking-house-rules.json @@ -8,6 +8,7 @@ "city": "Lisbon" }, "start_url": "https://www.booking.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "main"}, {"repo": "kernel/xwebmcp", "ref": "hypeship/booking-select-room-party", "pr": 129}], "answer_schema": "schemas/house-rules.json", "rules": [ { diff --git a/bench/tasks/booking.com/booking-reserve-cheapest-room.json b/bench/tasks/booking.com/booking-reserve-cheapest-room.json index 2940ad5c..d5e82fe7 100644 --- a/bench/tasks/booking.com/booking-reserve-cheapest-room.json +++ b/bench/tasks/booking.com/booking-reserve-cheapest-room.json @@ -14,6 +14,7 @@ } }, "start_url": "https://www.booking.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "main"}, {"repo": "kernel/xwebmcp", "ref": "hypeship/booking-select-room-party", "pr": 129}], "steps": [ { "name": "search", diff --git a/bench/tasks/delta.com/delta-cart-main-classic.json b/bench/tasks/delta.com/delta-cart-main-classic.json index 5ca3d2da..3d53d404 100644 --- a/bench/tasks/delta.com/delta-cart-main-classic.json +++ b/bench/tasks/delta.com/delta-cart-main-classic.json @@ -9,6 +9,7 @@ "date": {"days_from_start": 28} }, "start_url": "https://www.delta.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/delta-flight-status", "pr": 67}, {"repo": "kernel/xwebmcp", "ref": "main"}], "steps": [ {"name": "search", "url_patterns": ["https://www.delta.com/", "https://www.delta.com/flightsearch/book-a-flight*"]}, {"name": "results", "url_patterns": ["https://www.delta.com/flightsearch/search-results*"]}, diff --git a/bench/tasks/delta.com/delta-cheapest-fare.json b/bench/tasks/delta.com/delta-cheapest-fare.json index e8421070..44f6a4cb 100644 --- a/bench/tasks/delta.com/delta-cheapest-fare.json +++ b/bench/tasks/delta.com/delta-cheapest-fare.json @@ -9,7 +9,7 @@ "date": {"days_from_start": 28} }, "start_url": "https://www.delta.com/", - "tools": [{"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_flights"]}], + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/delta-flight-status", "pr": 67}, {"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_flights"]}], "entry_tool": "search_flights", "answer_schema": "schemas/cheapest-flight.json", "rules": [{"path": "departure_time", "time": true}, {"path": "price", "range": [1, 3000]}], diff --git a/bench/tasks/delta.com/delta-checkout-signed-in.json b/bench/tasks/delta.com/delta-checkout-signed-in.json index d3047404..0f2490a0 100644 --- a/bench/tasks/delta.com/delta-checkout-signed-in.json +++ b/bench/tasks/delta.com/delta-checkout-signed-in.json @@ -22,6 +22,7 @@ } }, "start_url": "https://www.delta.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/delta-flight-status", "pr": 67}, {"repo": "kernel/xwebmcp", "ref": "main"}], "allow": [ "checkout" ], diff --git a/bench/tasks/doordash.com/domain.json b/bench/tasks/doordash.com/domain.json new file mode 100644 index 00000000..780acdc8 --- /dev/null +++ b/bench/tasks/doordash.com/domain.json @@ -0,0 +1,9 @@ +{ + "needs_sign_in": { + "get_cart": "reads the signed-in account's carts", + "get_checkout_summary": "reads the signed-in account's carts" + }, + "actions": { + "set_delivery_address": "sets a signed-out visit's delivery address, as the header's address picker does" + } +} diff --git a/bench/tasks/doordash.com/doordash-pizza-near-address.json b/bench/tasks/doordash.com/doordash-pizza-near-address.json index ef28d5a4..24792f42 100644 --- a/bench/tasks/doordash.com/doordash-pizza-near-address.json +++ b/bench/tasks/doordash.com/doordash-pizza-near-address.json @@ -7,6 +7,7 @@ "address": "Times Square, New York, NY 10036" }, "start_url": "https://www.doordash.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/doordash-get-store", "pr": 57}], "answer_schema": "schemas/named-places.json", "rules": [ { diff --git a/bench/tasks/ebay.com/ebay-checkout-usb-c-cable.json b/bench/tasks/ebay.com/ebay-checkout-usb-c-cable.json index ed7e3770..4ce66286 100644 --- a/bench/tasks/ebay.com/ebay-checkout-usb-c-cable.json +++ b/bench/tasks/ebay.com/ebay-checkout-usb-c-cable.json @@ -13,6 +13,7 @@ "phone": "4155550123" }, "start_url": "https://www.ebay.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/ebay-search-listing", "pr": 80}], "allow": [ "checkout" ], diff --git a/bench/tasks/fandango.com/fandango-nearest-theaters.json b/bench/tasks/fandango.com/fandango-nearest-theaters.json index 54c21f05..09dadb57 100644 --- a/bench/tasks/fandango.com/fandango-nearest-theaters.json +++ b/bench/tasks/fandango.com/fandango-nearest-theaters.json @@ -3,7 +3,7 @@ "domain": "fandango.com", "stage": "find", "prompt": "On fandango.com, find the three theaters nearest ZIP code {{zip}} that show {{film}} on {{date}}, and at each one the first showtime of that film at or after 17:00 that day. Reply with JSON: {\"theaters\": [{\"name\": string, \"first_evening_showtime\": \"HH:MM\"}]}, nearest first, times in 24-hour local time.", - "fixture": {"film": "Verity (2026)", "zip": "10001", "date": "2026-10-13"}, + "fixture": {"film": "Verity (2026)", "zip": "10001", "date": {"days_from_start": 2}}, "start_url": "https://www.fandango.com/", "tools": [ { diff --git a/bench/tasks/google.com/google-flights-cheapest-fare.json b/bench/tasks/google.com/google-flights-cheapest-fare.json index 3d261364..96e9bfb3 100644 --- a/bench/tasks/google.com/google-flights-cheapest-fare.json +++ b/bench/tasks/google.com/google-flights-cheapest-fare.json @@ -9,7 +9,7 @@ "date": {"days_from_start": 28} }, "start_url": "https://www.google.com/travel/flights", - "tools": [{"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_flights"]}], + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/google-finance-maps-tools", "pr": 61}, {"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_flights"]}], "entry_tool": "search_flights", "answer_schema": "schemas/cheapest-flight.json", "rules": [{"path": "departure_time", "time": true}, {"path": "price", "range": [1, 3000]}], diff --git a/bench/tasks/opentable.com/opentable-available-times.json b/bench/tasks/opentable.com/opentable-available-times.json index c60e7843..27fa7ee2 100644 --- a/bench/tasks/opentable.com/opentable-available-times.json +++ b/bench/tasks/opentable.com/opentable-available-times.json @@ -14,6 +14,7 @@ "to": "21:00" }, "start_url": "https://www.opentable.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/opentable-restaurant-tools", "pr": 81}], "answer_schema": "schemas/available-times.json", "rules": [ { diff --git a/bench/tasks/shopify.com/shopify-app-store-compare.json b/bench/tasks/shopify.com/shopify-app-store-compare.json index 91aa933b..148f347c 100644 --- a/bench/tasks/shopify.com/shopify-app-store-compare.json +++ b/bench/tasks/shopify.com/shopify-app-store-compare.json @@ -7,6 +7,7 @@ "query": "subscriptions" }, "start_url": "https://apps.shopify.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/shopify-app-store-tools", "pr": 59}], "answer_schema": "schemas/app-listings.json", "rules": [ { "path": "apps", "count": [3, 3] }, diff --git a/bench/tasks/shopify.com/shopify-app-store-listing.json b/bench/tasks/shopify.com/shopify-app-store-listing.json index 8b0b0933..20bb6e15 100644 --- a/bench/tasks/shopify.com/shopify-app-store-listing.json +++ b/bench/tasks/shopify.com/shopify-app-store-listing.json @@ -7,6 +7,7 @@ "app": "Judge.me Product Reviews" }, "start_url": "https://apps.shopify.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/shopify-app-store-tools", "pr": 59}], "answer_schema": "schemas/app-listing.json", "rules": [ { diff --git a/bench/tasks/shopify.com/shopify-basic-monthly-price.json b/bench/tasks/shopify.com/shopify-basic-monthly-price.json index 9992a638..13b38d4b 100644 --- a/bench/tasks/shopify.com/shopify-basic-monthly-price.json +++ b/bench/tasks/shopify.com/shopify-basic-monthly-price.json @@ -4,6 +4,7 @@ "stage": "inspect", "prompt": "On shopify.com, find the monthly price of Shopify's Basic plan in US dollars when it is paid monthly rather than yearly. Do not start a trial. Reply with JSON: {\"plan\": string, \"monthly_price\": number}.", "start_url": "https://www.shopify.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/shopify-app-store-tools", "pr": 59}], "answer_schema": "schemas/plan-price.json", "rules": [ { diff --git a/bench/tasks/target.com/target-checkout-water-bottle.json b/bench/tasks/target.com/target-checkout-water-bottle.json index d66db119..6076479a 100644 --- a/bench/tasks/target.com/target-checkout-water-bottle.json +++ b/bench/tasks/target.com/target-checkout-water-bottle.json @@ -13,6 +13,7 @@ "phone": "4155550123" }, "start_url": "https://www.target.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "main"}, {"repo": "kernel/xwebmcp", "ref": "hypeship/target-store-reviews", "pr": 68}], "allow": [ "checkout" ], diff --git a/bench/tasks/target.com/target-search-first-result.json b/bench/tasks/target.com/target-search-first-result.json index fae9f58f..06bbba7b 100644 --- a/bench/tasks/target.com/target-search-first-result.json +++ b/bench/tasks/target.com/target-search-first-result.json @@ -5,7 +5,7 @@ "prompt": "On target.com, search for \"{{query}}\" and report the first product in the results: its title as the product, its price and its TCIN (Target's item number). Reply with JSON: {\"product\": string, \"price\": number in US dollars, \"tcin\": string}.", "fixture": {"query": "AirPods Pro"}, "start_url": "https://www.target.com/", - "tools": [{"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_products"]}], + "tools": [{"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_products"]}, {"repo": "kernel/xwebmcp", "ref": "hypeship/target-store-reviews", "pr": 68}], "entry_tool": "search_products", "answer_schema": "schemas/search-result.json", "rules": [{"path": "price", "range": [1, 1000]}], diff --git a/bench/tasks/ticketmaster.com/ticketmaster-concerts-in-city.json b/bench/tasks/ticketmaster.com/ticketmaster-concerts-in-city.json index f06f16b2..218e6820 100644 --- a/bench/tasks/ticketmaster.com/ticketmaster-concerts-in-city.json +++ b/bench/tasks/ticketmaster.com/ticketmaster-concerts-in-city.json @@ -9,7 +9,7 @@ "end": {"days_from_start": 21} }, "start_url": "https://www.ticketmaster.com/", - "tools": [{"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_events"]}], + "tools": [{"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_events"]}, {"repo": "kernel/xwebmcp", "ref": "hypeship/ticketmaster-attractions", "pr": 51}], "entry_tool": "search_events", "answer_schema": "schemas/events.json", "rules": [ diff --git a/bench/tasks/united.com/united-cheapest-fare.json b/bench/tasks/united.com/united-cheapest-fare.json index 0c51ed6f..e109e560 100644 --- a/bench/tasks/united.com/united-cheapest-fare.json +++ b/bench/tasks/united.com/united-cheapest-fare.json @@ -9,7 +9,7 @@ "date": {"days_from_start": 28} }, "start_url": "https://www.united.com/en/us", - "tools": [{"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_flights"]}], + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/united-flight-status", "pr": 71}, {"repo": "kernel/xwebmcp", "ref": "main", "names": ["search_flights"]}], "entry_tool": "search_flights", "answer_schema": "schemas/cheapest-flight.json", "rules": [{"path": "departure_time", "time": true}, {"path": "price", "range": [1, 3000]}], diff --git a/bench/tasks/vividseats.com/vividseats-cheapest-pair.json b/bench/tasks/vividseats.com/vividseats-cheapest-pair.json index 230c7e9a..32aa4241 100644 --- a/bench/tasks/vividseats.com/vividseats-cheapest-pair.json +++ b/bench/tasks/vividseats.com/vividseats-cheapest-pair.json @@ -15,6 +15,7 @@ } }, "start_url": "https://www.vividseats.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/vividseats-events", "pr": 52}], "answer_schema": "schemas/ticket-price.json", "rules": [ { diff --git a/bench/tasks/walmart.com/walmart-checkout-water-bottle.json b/bench/tasks/walmart.com/walmart-checkout-water-bottle.json index 5053629d..71e20b65 100644 --- a/bench/tasks/walmart.com/walmart-checkout-water-bottle.json +++ b/bench/tasks/walmart.com/walmart-checkout-water-bottle.json @@ -13,6 +13,7 @@ "phone": "4155550123" }, "start_url": "https://www.walmart.com/", + "tools": [{"repo": "kernel/xwebmcp", "ref": "hypeship/walmart-read-tools", "pr": 63}], "allow": [ "checkout" ], diff --git a/bench/test/tasks.test.mjs b/bench/test/tasks.test.mjs index f3d1ba17..1993dc5a 100644 --- a/bench/test/tasks.test.mjs +++ b/bench/test/tasks.test.mjs @@ -222,7 +222,7 @@ test("a domain's page actions reach every intent, its actions its prepare intent assert.ok(!actions("google-flights-cheapest-fare").includes("select_flight")); assert.deepEqual(actions("delta-cart-main-classic"), ["select_fare", "select_fare_experience"]); assert.ok(actions("delta-checkout-signed-in").includes("continue_checkout")); - assert.deepEqual(intents.find(i => i.id === "delta-cart-main-classic").tools, [{ repo: "kernel/xwebmcp", ref: "main" }]); + assert.deepEqual(intents.find(i => i.id === "airbnb-cheapest-well-rated").tools, [{ repo: "kernel/xwebmcp", ref: "main" }]); }); // What run.mjs registers at startup, from the tools in this checkout, so that a tool change that