From 79ce0361ca3fabb26bfacd0817f5708cb73d9ec3 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 11 Aug 2025 10:10:47 -0700 Subject: [PATCH 01/43] add --port arg to ep logs --- eval_protocol/cli.py | 1 + eval_protocol/cli_commands/logs.py | 7 +- eval_protocol/utils/logs_server.py | 48 ++++++++++-- eval_protocol/utils/vite_server.py | 45 +++++++++-- vite-app/dist/assets/index-CGYj40Gx.css | 1 + vite-app/dist/assets/index-DWfIf2rx.css | 1 - vite-app/dist/assets/index-D_nkLTVA.js | 88 ---------------------- vite-app/dist/assets/index-D_nkLTVA.js.map | 1 - vite-app/dist/assets/index-t_hsfGP1.js | 88 ++++++++++++++++++++++ vite-app/dist/assets/index-t_hsfGP1.js.map | 1 + vite-app/dist/index.html | 6 +- vite-app/docs/runtime-configuration.md | 73 ++++++++++++++++++ vite-app/src/App.tsx | 11 ++- vite-app/src/config.ts | 63 ++++++++++++++++ vite-app/src/types/global.d.ts | 13 ++++ 15 files changed, 334 insertions(+), 113 deletions(-) create mode 100644 vite-app/dist/assets/index-CGYj40Gx.css delete mode 100644 vite-app/dist/assets/index-DWfIf2rx.css delete mode 100644 vite-app/dist/assets/index-D_nkLTVA.js delete mode 100644 vite-app/dist/assets/index-D_nkLTVA.js.map create mode 100644 vite-app/dist/assets/index-t_hsfGP1.js create mode 100644 vite-app/dist/assets/index-t_hsfGP1.js.map create mode 100644 vite-app/docs/runtime-configuration.md create mode 100644 vite-app/src/config.ts create mode 100644 vite-app/src/types/global.d.ts diff --git a/eval_protocol/cli.py b/eval_protocol/cli.py index f94d14ba..24307eb0 100644 --- a/eval_protocol/cli.py +++ b/eval_protocol/cli.py @@ -289,6 +289,7 @@ def parse_args(args=None): # Logs command logs_parser = subparsers.add_parser("logs", help="Serve logs with file watching and real-time updates") + logs_parser.add_argument("--port", type=int, default=8000, help="Port to bind to (default: 8000)") # Run command (for Hydra-based evaluations) # This subparser intentionally defines no arguments itself. diff --git a/eval_protocol/cli_commands/logs.py b/eval_protocol/cli_commands/logs.py index 3f4eda7e..07c211be 100644 --- a/eval_protocol/cli_commands/logs.py +++ b/eval_protocol/cli_commands/logs.py @@ -11,15 +11,16 @@ def logs_command(args): """Serve logs with file watching and real-time updates""" + port = args.port print(f"šŸš€ Starting Eval Protocol Logs Server") - print(f"šŸŒ URL: http://localhost:8000") - print(f"šŸ”Œ WebSocket: ws://localhost:8000/ws") + print(f"šŸŒ URL: http://localhost:{port}") + print(f"šŸ”Œ WebSocket: ws://localhost:{port}/ws") print(f"šŸ‘€ Watching paths: {['current directory']}") print("Press Ctrl+C to stop the server") print("-" * 50) try: - serve_logs() + serve_logs(port=args.port) return 0 except KeyboardInterrupt: print("\nšŸ›‘ Server stopped by user") diff --git a/eval_protocol/utils/logs_server.py b/eval_protocol/utils/logs_server.py index 97199ccc..815d33ef 100644 --- a/eval_protocol/utils/logs_server.py +++ b/eval_protocol/utils/logs_server.py @@ -319,20 +319,52 @@ def run(self): asyncio.run(self.run_async()) -server = LogsServer() -app = server.app +def create_app(host: str = "localhost", port: int = 8000, build_dir: Optional[str] = None) -> FastAPI: + """ + Factory function to create a FastAPI app instance. + + This allows uvicorn to call it with parameters and avoids top-level variable instantiation. + + Args: + host: Host to bind to + port: Port to bind to + build_dir: Optional custom build directory path + + Returns: + FastAPI app instance + """ + if build_dir is None: + build_dir = os.path.abspath( + os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "vite-app", "dist") + ) + server = LogsServer(host=host, port=port, build_dir=build_dir) + return server.app -def serve_logs(): + +# For backward compatibility and direct usage +def serve_logs(port: Optional[int] = None): """ Convenience function to create and run a LogsServer. """ - global server, app - if server is None: - server = LogsServer() - app = server.app + server = LogsServer(port=port) server.run() if __name__ == "__main__": - serve_logs() + import argparse + + parser = argparse.ArgumentParser(description="Start the evaluation logs server") + parser.add_argument("--host", default="localhost", help="Host to bind to (default: localhost)") + parser.add_argument("--port", type=int, default=8000, help="Port to bind to (default: 8000)") + parser.add_argument("--build-dir", help="Path to Vite build directory") + + args = parser.parse_args() + + # Create server with command line arguments + if args.build_dir: + server = LogsServer(host=args.host, port=args.port, build_dir=args.build_dir) + else: + server = LogsServer(host=args.host, port=args.port) + + server.run() diff --git a/eval_protocol/utils/vite_server.py b/eval_protocol/utils/vite_server.py index c3df4640..4c3143e2 100644 --- a/eval_protocol/utils/vite_server.py +++ b/eval_protocol/utils/vite_server.py @@ -5,7 +5,7 @@ import uvicorn from fastapi import FastAPI, HTTPException -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, HTMLResponse from fastapi.staticfiles import StaticFiles logger = logging.getLogger(__name__) @@ -57,6 +57,40 @@ def __init__( # Setup routes self._setup_routes() + def _inject_config_into_html(self, html_content: str) -> str: + """Inject server configuration into the HTML content.""" + config_script = f""" + +""" + + # Insert the config script before the closing tag + if "" in html_content: + return html_content.replace("", f"{config_script}") + else: + # If no tag, insert at the beginning + return f"{config_script}{html_content}" + + def _serve_index_with_config(self) -> HTMLResponse: + """Serve the index.html file with injected configuration.""" + index_path = self.build_dir / self.index_file + if index_path.exists(): + with open(index_path, "r", encoding="utf-8") as f: + html_content = f.read() + + # Inject server configuration + enhanced_html = self._inject_config_into_html(html_content) + return HTMLResponse(content=enhanced_html) + + raise HTTPException(status_code=404, detail="Index file not found") + def _setup_routes(self): """Set up the API routes for serving the SPA.""" @@ -81,18 +115,15 @@ async def serve_spa(path: str): # For SPA routing, serve index.html for non-existent routes # but exclude API routes and asset requests if not path.startswith(("api/", "assets/")): - index_path = self.build_dir / self.index_file - if index_path.exists(): - return FileResponse(index_path) + return self._serve_index_with_config() # If we get here, the file doesn't exist and it's not a SPA route raise HTTPException(status_code=404, detail="File not found") @self.app.get("/") async def root(): - """Serve the main index.html file.""" - index_path = self.build_dir / self.index_file - return FileResponse(index_path) + """Serve the main index.html file with injected configuration.""" + return self._serve_index_with_config() @self.app.get("/health") async def health(): diff --git a/vite-app/dist/assets/index-CGYj40Gx.css b/vite-app/dist/assets/index-CGYj40Gx.css new file mode 100644 index 00000000..292784b4 --- /dev/null +++ b/vite-app/dist/assets/index-CGYj40Gx.css @@ -0,0 +1 @@ +/*! tailwindcss v4.1.11 | MIT License | https://tailwindcss.com */@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-rotate-x:initial;--tw-rotate-y:initial;--tw-rotate-z:initial;--tw-skew-x:initial;--tw-skew-y:initial;--tw-space-y-reverse:0;--tw-space-x-reverse:0;--tw-divide-y-reverse:0;--tw-border-style:solid;--tw-font-weight:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-outline-style:solid;--tw-blur:initial;--tw-brightness:initial;--tw-contrast:initial;--tw-grayscale:initial;--tw-hue-rotate:initial;--tw-invert:initial;--tw-opacity:initial;--tw-saturate:initial;--tw-sepia:initial;--tw-drop-shadow:initial;--tw-drop-shadow-color:initial;--tw-drop-shadow-alpha:100%;--tw-drop-shadow-size:initial;--tw-duration:initial}}}@layer theme{:root,:host{--font-sans:ui-sans-serif,system-ui,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--font-mono:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;--color-red-500:oklch(63.7% .237 25.331);--color-red-700:oklch(50.5% .213 27.518);--color-yellow-50:oklch(98.7% .026 102.212);--color-yellow-100:oklch(97.3% .071 103.193);--color-yellow-200:oklch(94.5% .129 101.54);--color-yellow-500:oklch(79.5% .184 86.047);--color-yellow-700:oklch(55.4% .135 66.442);--color-yellow-800:oklch(47.6% .114 61.907);--color-yellow-900:oklch(42.1% .095 57.708);--color-green-50:oklch(98.2% .018 155.826);--color-green-100:oklch(96.2% .044 156.743);--color-green-200:oklch(92.5% .084 155.995);--color-green-500:oklch(72.3% .219 149.579);--color-green-700:oklch(52.7% .154 150.069);--color-green-800:oklch(44.8% .119 151.328);--color-green-900:oklch(39.3% .095 152.535);--color-blue-50:oklch(97% .014 254.604);--color-blue-200:oklch(88.2% .059 254.128);--color-blue-500:oklch(62.3% .214 259.815);--color-blue-700:oklch(48.8% .243 264.376);--color-blue-900:oklch(37.9% .146 265.522);--color-gray-50:oklch(98.5% .002 247.839);--color-gray-100:oklch(96.7% .003 264.542);--color-gray-200:oklch(92.8% .006 264.531);--color-gray-300:oklch(87.2% .01 258.338);--color-gray-400:oklch(70.7% .022 261.325);--color-gray-500:oklch(55.1% .027 264.364);--color-gray-600:oklch(44.6% .03 256.802);--color-gray-700:oklch(37.3% .034 259.733);--color-gray-800:oklch(27.8% .033 256.848);--color-gray-900:oklch(21% .034 264.665);--color-white:#fff;--spacing:.25rem;--container-sm:24rem;--container-md:28rem;--container-lg:32rem;--container-7xl:80rem;--text-xs:.75rem;--text-xs--line-height:calc(1/.75);--text-sm:.875rem;--text-sm--line-height:calc(1.25/.875);--font-weight-medium:500;--font-weight-semibold:600;--animate-spin:spin 1s linear infinite;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4,0,.2,1);--default-font-family:var(--font-sans);--default-mono-font-family:var(--font-mono)}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;line-height:1.5;font-family:var(--default-font-family,ui-sans-serif,system-ui,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji");font-feature-settings:var(--default-font-feature-settings,normal);font-variation-settings:var(--default-font-variation-settings,normal);-webkit-tap-highlight-color:transparent}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:var(--default-mono-font-family,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace);font-feature-settings:var(--default-mono-font-feature-settings,normal);font-variation-settings:var(--default-mono-font-variation-settings,normal);font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring{outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab,red,red)){::placeholder{color:color-mix(in oklab,currentcolor 50%,transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{padding-block:0}::-webkit-datetime-edit-month-field{padding-block:0}::-webkit-datetime-edit-day-field{padding-block:0}::-webkit-datetime-edit-hour-field{padding-block:0}::-webkit-datetime-edit-minute-field{padding-block:0}::-webkit-datetime-edit-second-field{padding-block:0}::-webkit-datetime-edit-millisecond-field{padding-block:0}::-webkit-datetime-edit-meridiem-field{padding-block:0}:-moz-ui-invalid{box-shadow:none}button,input:where([type=button],[type=reset],[type=submit]){appearance:button}::file-selector-button{appearance:button}::-webkit-inner-spin-button{height:auto}::-webkit-outer-spin-button{height:auto}[hidden]:where(:not([hidden=until-found])){display:none!important}}@layer components;@layer utilities{.collapse{visibility:collapse}.invisible{visibility:hidden}.visible{visibility:visible}.absolute{position:absolute}.fixed{position:fixed}.relative{position:relative}.static{position:static}.sticky{position:sticky}.top-0{top:calc(var(--spacing)*0)}.right-0{right:calc(var(--spacing)*0)}.left-0{left:calc(var(--spacing)*0)}.\!container{width:100%!important}@media (min-width:40rem){.\!container{max-width:40rem!important}}@media (min-width:48rem){.\!container{max-width:48rem!important}}@media (min-width:64rem){.\!container{max-width:64rem!important}}@media (min-width:80rem){.\!container{max-width:80rem!important}}@media (min-width:96rem){.\!container{max-width:96rem!important}}.container{width:100%}@media (min-width:40rem){.container{max-width:40rem}}@media (min-width:48rem){.container{max-width:48rem}}@media (min-width:64rem){.container{max-width:64rem}}@media (min-width:80rem){.container{max-width:80rem}}@media (min-width:96rem){.container{max-width:96rem}}.mx-auto{margin-inline:auto}.mt-1{margin-top:calc(var(--spacing)*1)}.mt-2{margin-top:calc(var(--spacing)*2)}.mb-0\.5{margin-bottom:calc(var(--spacing)*.5)}.mb-1{margin-bottom:calc(var(--spacing)*1)}.mb-2{margin-bottom:calc(var(--spacing)*2)}.mb-4{margin-bottom:calc(var(--spacing)*4)}.ml-2{margin-left:calc(var(--spacing)*2)}.block{display:block}.flex{display:flex}.hidden{display:none}.inline-flex{display:inline-flex}.table{display:table}.h-1{height:calc(var(--spacing)*1)}.h-1\.5{height:calc(var(--spacing)*1.5)}.h-3{height:calc(var(--spacing)*3)}.h-4{height:calc(var(--spacing)*4)}.h-6{height:calc(var(--spacing)*6)}.h-8{height:calc(var(--spacing)*8)}.h-10{height:calc(var(--spacing)*10)}.h-12{height:calc(var(--spacing)*12)}.min-h-screen{min-height:100vh}.w-1{width:calc(var(--spacing)*1)}.w-1\.5{width:calc(var(--spacing)*1.5)}.w-3{width:calc(var(--spacing)*3)}.w-4{width:calc(var(--spacing)*4)}.w-8{width:calc(var(--spacing)*8)}.w-12{width:calc(var(--spacing)*12)}.w-\[500px\]{width:500px}.w-auto{width:auto}.w-fit{width:fit-content}.w-full{width:100%}.max-w-7xl{max-width:var(--container-7xl)}.max-w-sm{max-width:var(--container-sm)}.min-w-0{min-width:calc(var(--spacing)*0)}.min-w-max{min-width:max-content}.flex-shrink-0{flex-shrink:0}.shrink{flex-shrink:1}.grow{flex-grow:1}.rotate-90{rotate:90deg}.rotate-180{rotate:180deg}.transform{transform:var(--tw-rotate-x,)var(--tw-rotate-y,)var(--tw-rotate-z,)var(--tw-skew-x,)var(--tw-skew-y,)}.animate-spin{animation:var(--animate-spin)}.cursor-col-resize{cursor:col-resize}.cursor-nw-resize{cursor:nw-resize}.cursor-pointer{cursor:pointer}.cursor-row-resize{cursor:row-resize}.resize{resize:both}.items-center{align-items:center}.justify-between{justify-content:space-between}.justify-center{justify-content:center}.justify-end{justify-content:flex-end}.justify-start{justify-content:flex-start}.gap-1{gap:calc(var(--spacing)*1)}.gap-1\.5{gap:calc(var(--spacing)*1.5)}.gap-2{gap:calc(var(--spacing)*2)}.gap-3{gap:calc(var(--spacing)*3)}.gap-4{gap:calc(var(--spacing)*4)}:where(.space-y-3>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing)*3)*var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing)*3)*calc(1 - var(--tw-space-y-reverse)))}:where(.space-x-2>:not(:last-child)){--tw-space-x-reverse:0;margin-inline-start:calc(calc(var(--spacing)*2)*var(--tw-space-x-reverse));margin-inline-end:calc(calc(var(--spacing)*2)*calc(1 - var(--tw-space-x-reverse)))}:where(.divide-y>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px*var(--tw-divide-y-reverse));border-bottom-width:calc(1px*calc(1 - var(--tw-divide-y-reverse)))}:where(.divide-gray-200>:not(:last-child)){border-color:var(--color-gray-200)}.truncate{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.overflow-hidden{overflow:hidden}.overflow-x-auto{overflow-x:auto}.overflow-y-auto{overflow-y:auto}.rounded{border-radius:.25rem}.rounded-full{border-radius:3.40282e38px}.border{border-style:var(--tw-border-style);border-width:1px}.border-t{border-top-style:var(--tw-border-style);border-top-width:1px}.border-b{border-bottom-style:var(--tw-border-style);border-bottom-width:1px}.border-b-2{border-bottom-style:var(--tw-border-style);border-bottom-width:2px}.border-blue-200{border-color:var(--color-blue-200)}.border-current{border-color:currentColor}.border-gray-200{border-color:var(--color-gray-200)}.border-gray-300{border-color:var(--color-gray-300)}.border-gray-900{border-color:var(--color-gray-900)}.border-green-200{border-color:var(--color-green-200)}.border-transparent{border-color:#0000}.border-yellow-200{border-color:var(--color-yellow-200)}.border-t-transparent{border-top-color:#0000}.bg-blue-50{background-color:var(--color-blue-50)}.bg-blue-500{background-color:var(--color-blue-500)}.bg-gray-50{background-color:var(--color-gray-50)}.bg-gray-100{background-color:var(--color-gray-100)}.bg-gray-300{background-color:var(--color-gray-300)}.bg-gray-500{background-color:var(--color-gray-500)}.bg-green-50{background-color:var(--color-green-50)}.bg-green-100{background-color:var(--color-green-100)}.bg-green-500{background-color:var(--color-green-500)}.bg-red-500{background-color:var(--color-red-500)}.bg-transparent{background-color:#0000}.bg-white{background-color:var(--color-white)}.bg-yellow-50{background-color:var(--color-yellow-50)}.bg-yellow-100{background-color:var(--color-yellow-100)}.bg-yellow-500{background-color:var(--color-yellow-500)}.p-0{padding:calc(var(--spacing)*0)}.p-1{padding:calc(var(--spacing)*1)}.p-2{padding:calc(var(--spacing)*2)}.p-3{padding:calc(var(--spacing)*3)}.p-4{padding:calc(var(--spacing)*4)}.p-8{padding:calc(var(--spacing)*8)}.px-2{padding-inline:calc(var(--spacing)*2)}.px-3{padding-inline:calc(var(--spacing)*3)}.py-0\.5{padding-block:calc(var(--spacing)*.5)}.py-1{padding-block:calc(var(--spacing)*1)}.py-2{padding-block:calc(var(--spacing)*2)}.py-3{padding-block:calc(var(--spacing)*3)}.py-4{padding-block:calc(var(--spacing)*4)}.pt-1{padding-top:calc(var(--spacing)*1)}.pt-2{padding-top:calc(var(--spacing)*2)}.pb-2{padding-bottom:calc(var(--spacing)*2)}.text-center{text-align:center}.text-left{text-align:left}.text-right{text-align:right}.font-mono{font-family:var(--font-mono)}.text-sm{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.text-xs{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height))}.font-medium{--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium)}.font-semibold{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold)}.break-words{overflow-wrap:break-word}.break-all{word-break:break-all}.whitespace-nowrap{white-space:nowrap}.whitespace-pre-wrap{white-space:pre-wrap}.text-blue-700{color:var(--color-blue-700)}.text-blue-900{color:var(--color-blue-900)}.text-gray-400{color:var(--color-gray-400)}.text-gray-500{color:var(--color-gray-500)}.text-gray-600{color:var(--color-gray-600)}.text-gray-700{color:var(--color-gray-700)}.text-gray-800{color:var(--color-gray-800)}.text-gray-900{color:var(--color-gray-900)}.text-green-700{color:var(--color-green-700)}.text-green-800{color:var(--color-green-800)}.text-green-900{color:var(--color-green-900)}.text-red-700{color:var(--color-red-700)}.text-yellow-700{color:var(--color-yellow-700)}.text-yellow-800{color:var(--color-yellow-800)}.text-yellow-900{color:var(--color-yellow-900)}.capitalize{text-transform:capitalize}.lowercase{text-transform:lowercase}.uppercase{text-transform:uppercase}.italic{font-style:italic}.underline{text-decoration-line:underline}.shadow{--tw-shadow:0 1px 3px 0 var(--tw-shadow-color,#0000001a),0 1px 2px -1px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow),var(--tw-inset-ring-shadow),var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow)}.outline{outline-style:var(--tw-outline-style);outline-width:1px}.blur{--tw-blur:blur(8px);filter:var(--tw-blur,)var(--tw-brightness,)var(--tw-contrast,)var(--tw-grayscale,)var(--tw-hue-rotate,)var(--tw-invert,)var(--tw-saturate,)var(--tw-sepia,)var(--tw-drop-shadow,)}.filter{filter:var(--tw-blur,)var(--tw-brightness,)var(--tw-contrast,)var(--tw-grayscale,)var(--tw-hue-rotate,)var(--tw-invert,)var(--tw-saturate,)var(--tw-sepia,)var(--tw-drop-shadow,)}.transition{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to,opacity,box-shadow,transform,translate,scale,rotate,filter,-webkit-backdrop-filter,backdrop-filter,display,visibility,content-visibility,overlay,pointer-events;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-colors{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-transform{transition-property:transform,translate,scale,rotate;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.duration-200{--tw-duration:.2s;transition-duration:.2s}.select-none{-webkit-user-select:none;user-select:none}@media (hover:hover){.hover\:border-gray-400:hover{border-color:var(--color-gray-400)}.hover\:bg-gray-50:hover{background-color:var(--color-gray-50)}.hover\:bg-gray-100:hover{background-color:var(--color-gray-100)}.hover\:bg-gray-200:hover{background-color:var(--color-gray-200)}.hover\:bg-gray-400:hover{background-color:var(--color-gray-400)}.hover\:text-gray-900:hover{color:var(--color-gray-900)}.hover\:no-underline:hover{text-decoration-line:none}}.focus\:border-gray-500:focus{border-color:var(--color-gray-500)}.focus\:outline-none:focus{--tw-outline-style:none;outline-style:none}@media (min-width:64rem){.lg\:max-w-md{max-width:var(--container-md)}}@media (min-width:80rem){.xl\:max-w-lg{max-width:var(--container-lg)}}}@property --tw-rotate-x{syntax:"*";inherits:false}@property --tw-rotate-y{syntax:"*";inherits:false}@property --tw-rotate-z{syntax:"*";inherits:false}@property --tw-skew-x{syntax:"*";inherits:false}@property --tw-skew-y{syntax:"*";inherits:false}@property --tw-space-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-space-x-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-divide-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-border-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-font-weight{syntax:"*";inherits:false}@property --tw-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-shadow-color{syntax:"*";inherits:false}@property --tw-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-inset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-shadow-color{syntax:"*";inherits:false}@property --tw-inset-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-ring-color{syntax:"*";inherits:false}@property --tw-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-ring-color{syntax:"*";inherits:false}@property --tw-inset-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-ring-inset{syntax:"*";inherits:false}@property --tw-ring-offset-width{syntax:"";inherits:false;initial-value:0}@property --tw-ring-offset-color{syntax:"*";inherits:false;initial-value:#fff}@property --tw-ring-offset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-outline-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-blur{syntax:"*";inherits:false}@property --tw-brightness{syntax:"*";inherits:false}@property --tw-contrast{syntax:"*";inherits:false}@property --tw-grayscale{syntax:"*";inherits:false}@property --tw-hue-rotate{syntax:"*";inherits:false}@property --tw-invert{syntax:"*";inherits:false}@property --tw-opacity{syntax:"*";inherits:false}@property --tw-saturate{syntax:"*";inherits:false}@property --tw-sepia{syntax:"*";inherits:false}@property --tw-drop-shadow{syntax:"*";inherits:false}@property --tw-drop-shadow-color{syntax:"*";inherits:false}@property --tw-drop-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-drop-shadow-size{syntax:"*";inherits:false}@property --tw-duration{syntax:"*";inherits:false}@keyframes spin{to{transform:rotate(360deg)}} diff --git a/vite-app/dist/assets/index-DWfIf2rx.css b/vite-app/dist/assets/index-DWfIf2rx.css deleted file mode 100644 index 10fe5c29..00000000 --- a/vite-app/dist/assets/index-DWfIf2rx.css +++ /dev/null @@ -1 +0,0 @@ -/*! tailwindcss v4.1.11 | MIT License | https://tailwindcss.com */@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-rotate-x:initial;--tw-rotate-y:initial;--tw-rotate-z:initial;--tw-skew-x:initial;--tw-skew-y:initial;--tw-space-y-reverse:0;--tw-space-x-reverse:0;--tw-divide-y-reverse:0;--tw-border-style:solid;--tw-font-weight:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-outline-style:solid;--tw-blur:initial;--tw-brightness:initial;--tw-contrast:initial;--tw-grayscale:initial;--tw-hue-rotate:initial;--tw-invert:initial;--tw-opacity:initial;--tw-saturate:initial;--tw-sepia:initial;--tw-drop-shadow:initial;--tw-drop-shadow-color:initial;--tw-drop-shadow-alpha:100%;--tw-drop-shadow-size:initial;--tw-duration:initial}}}@layer theme{:root,:host{--font-sans:ui-sans-serif,system-ui,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--font-mono:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;--color-red-500:oklch(63.7% .237 25.331);--color-red-700:oklch(50.5% .213 27.518);--color-yellow-50:oklch(98.7% .026 102.212);--color-yellow-100:oklch(97.3% .071 103.193);--color-yellow-200:oklch(94.5% .129 101.54);--color-yellow-500:oklch(79.5% .184 86.047);--color-yellow-700:oklch(55.4% .135 66.442);--color-yellow-800:oklch(47.6% .114 61.907);--color-yellow-900:oklch(42.1% .095 57.708);--color-green-50:oklch(98.2% .018 155.826);--color-green-100:oklch(96.2% .044 156.743);--color-green-200:oklch(92.5% .084 155.995);--color-green-500:oklch(72.3% .219 149.579);--color-green-700:oklch(52.7% .154 150.069);--color-green-800:oklch(44.8% .119 151.328);--color-green-900:oklch(39.3% .095 152.535);--color-blue-50:oklch(97% .014 254.604);--color-blue-200:oklch(88.2% .059 254.128);--color-blue-500:oklch(62.3% .214 259.815);--color-blue-700:oklch(48.8% .243 264.376);--color-blue-900:oklch(37.9% .146 265.522);--color-gray-50:oklch(98.5% .002 247.839);--color-gray-100:oklch(96.7% .003 264.542);--color-gray-200:oklch(92.8% .006 264.531);--color-gray-300:oklch(87.2% .01 258.338);--color-gray-400:oklch(70.7% .022 261.325);--color-gray-500:oklch(55.1% .027 264.364);--color-gray-600:oklch(44.6% .03 256.802);--color-gray-700:oklch(37.3% .034 259.733);--color-gray-800:oklch(27.8% .033 256.848);--color-gray-900:oklch(21% .034 264.665);--color-white:#fff;--spacing:.25rem;--container-sm:24rem;--container-md:28rem;--container-lg:32rem;--container-7xl:80rem;--text-xs:.75rem;--text-xs--line-height:calc(1/.75);--text-sm:.875rem;--text-sm--line-height:calc(1.25/.875);--font-weight-medium:500;--font-weight-semibold:600;--animate-spin:spin 1s linear infinite;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4,0,.2,1);--default-font-family:var(--font-sans);--default-mono-font-family:var(--font-mono)}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;line-height:1.5;font-family:var(--default-font-family,ui-sans-serif,system-ui,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji");font-feature-settings:var(--default-font-feature-settings,normal);font-variation-settings:var(--default-font-variation-settings,normal);-webkit-tap-highlight-color:transparent}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:var(--default-mono-font-family,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace);font-feature-settings:var(--default-mono-font-feature-settings,normal);font-variation-settings:var(--default-mono-font-variation-settings,normal);font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring{outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab,red,red)){::placeholder{color:color-mix(in oklab,currentcolor 50%,transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{padding-block:0}::-webkit-datetime-edit-month-field{padding-block:0}::-webkit-datetime-edit-day-field{padding-block:0}::-webkit-datetime-edit-hour-field{padding-block:0}::-webkit-datetime-edit-minute-field{padding-block:0}::-webkit-datetime-edit-second-field{padding-block:0}::-webkit-datetime-edit-millisecond-field{padding-block:0}::-webkit-datetime-edit-meridiem-field{padding-block:0}:-moz-ui-invalid{box-shadow:none}button,input:where([type=button],[type=reset],[type=submit]){appearance:button}::file-selector-button{appearance:button}::-webkit-inner-spin-button{height:auto}::-webkit-outer-spin-button{height:auto}[hidden]:where(:not([hidden=until-found])){display:none!important}}@layer components;@layer utilities{.collapse{visibility:collapse}.invisible{visibility:hidden}.visible{visibility:visible}.absolute{position:absolute}.fixed{position:fixed}.relative{position:relative}.static{position:static}.sticky{position:sticky}.top-0{top:calc(var(--spacing)*0)}.right-0{right:calc(var(--spacing)*0)}.left-0{left:calc(var(--spacing)*0)}.\!container{width:100%!important}@media (min-width:40rem){.\!container{max-width:40rem!important}}@media (min-width:48rem){.\!container{max-width:48rem!important}}@media (min-width:64rem){.\!container{max-width:64rem!important}}@media (min-width:80rem){.\!container{max-width:80rem!important}}@media (min-width:96rem){.\!container{max-width:96rem!important}}.container{width:100%}@media (min-width:40rem){.container{max-width:40rem}}@media (min-width:48rem){.container{max-width:48rem}}@media (min-width:64rem){.container{max-width:64rem}}@media (min-width:80rem){.container{max-width:80rem}}@media (min-width:96rem){.container{max-width:96rem}}.mx-auto{margin-inline:auto}.mt-1{margin-top:calc(var(--spacing)*1)}.mt-2{margin-top:calc(var(--spacing)*2)}.mb-0\.5{margin-bottom:calc(var(--spacing)*.5)}.mb-1{margin-bottom:calc(var(--spacing)*1)}.mb-2{margin-bottom:calc(var(--spacing)*2)}.mb-4{margin-bottom:calc(var(--spacing)*4)}.ml-2{margin-left:calc(var(--spacing)*2)}.block{display:block}.flex{display:flex}.hidden{display:none}.inline-flex{display:inline-flex}.table{display:table}.h-1{height:calc(var(--spacing)*1)}.h-1\.5{height:calc(var(--spacing)*1.5)}.h-3{height:calc(var(--spacing)*3)}.h-4{height:calc(var(--spacing)*4)}.h-6{height:calc(var(--spacing)*6)}.h-8{height:calc(var(--spacing)*8)}.h-10{height:calc(var(--spacing)*10)}.h-12{height:calc(var(--spacing)*12)}.min-h-screen{min-height:100vh}.w-1{width:calc(var(--spacing)*1)}.w-1\.5{width:calc(var(--spacing)*1.5)}.w-3{width:calc(var(--spacing)*3)}.w-4{width:calc(var(--spacing)*4)}.w-8{width:calc(var(--spacing)*8)}.w-12{width:calc(var(--spacing)*12)}.w-\[500px\]{width:500px}.w-auto{width:auto}.w-fit{width:fit-content}.w-full{width:100%}.max-w-7xl{max-width:var(--container-7xl)}.max-w-sm{max-width:var(--container-sm)}.min-w-0{min-width:calc(var(--spacing)*0)}.min-w-max{min-width:max-content}.flex-shrink-0{flex-shrink:0}.shrink{flex-shrink:1}.grow{flex-grow:1}.rotate-90{rotate:90deg}.rotate-180{rotate:180deg}.transform{transform:var(--tw-rotate-x,)var(--tw-rotate-y,)var(--tw-rotate-z,)var(--tw-skew-x,)var(--tw-skew-y,)}.animate-spin{animation:var(--animate-spin)}.cursor-col-resize{cursor:col-resize}.cursor-nw-resize{cursor:nw-resize}.cursor-pointer{cursor:pointer}.cursor-row-resize{cursor:row-resize}.resize{resize:both}.items-center{align-items:center}.justify-between{justify-content:space-between}.justify-center{justify-content:center}.justify-end{justify-content:flex-end}.justify-start{justify-content:flex-start}.gap-1{gap:calc(var(--spacing)*1)}.gap-1\.5{gap:calc(var(--spacing)*1.5)}.gap-2{gap:calc(var(--spacing)*2)}.gap-3{gap:calc(var(--spacing)*3)}.gap-4{gap:calc(var(--spacing)*4)}:where(.space-y-1>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing)*1)*var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing)*1)*calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-3>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing)*3)*var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing)*3)*calc(1 - var(--tw-space-y-reverse)))}:where(.space-x-2>:not(:last-child)){--tw-space-x-reverse:0;margin-inline-start:calc(calc(var(--spacing)*2)*var(--tw-space-x-reverse));margin-inline-end:calc(calc(var(--spacing)*2)*calc(1 - var(--tw-space-x-reverse)))}:where(.divide-y>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px*var(--tw-divide-y-reverse));border-bottom-width:calc(1px*calc(1 - var(--tw-divide-y-reverse)))}:where(.divide-gray-200>:not(:last-child)){border-color:var(--color-gray-200)}.truncate{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.overflow-hidden{overflow:hidden}.overflow-x-auto{overflow-x:auto}.overflow-y-auto{overflow-y:auto}.rounded{border-radius:.25rem}.rounded-full{border-radius:3.40282e38px}.border{border-style:var(--tw-border-style);border-width:1px}.border-t{border-top-style:var(--tw-border-style);border-top-width:1px}.border-b{border-bottom-style:var(--tw-border-style);border-bottom-width:1px}.border-b-2{border-bottom-style:var(--tw-border-style);border-bottom-width:2px}.border-blue-200{border-color:var(--color-blue-200)}.border-current{border-color:currentColor}.border-gray-200{border-color:var(--color-gray-200)}.border-gray-300{border-color:var(--color-gray-300)}.border-gray-900{border-color:var(--color-gray-900)}.border-green-200{border-color:var(--color-green-200)}.border-transparent{border-color:#0000}.border-yellow-200{border-color:var(--color-yellow-200)}.border-t-transparent{border-top-color:#0000}.bg-blue-50{background-color:var(--color-blue-50)}.bg-blue-500{background-color:var(--color-blue-500)}.bg-gray-50{background-color:var(--color-gray-50)}.bg-gray-100{background-color:var(--color-gray-100)}.bg-gray-300{background-color:var(--color-gray-300)}.bg-gray-500{background-color:var(--color-gray-500)}.bg-green-50{background-color:var(--color-green-50)}.bg-green-100{background-color:var(--color-green-100)}.bg-green-500{background-color:var(--color-green-500)}.bg-red-500{background-color:var(--color-red-500)}.bg-transparent{background-color:#0000}.bg-white{background-color:var(--color-white)}.bg-yellow-50{background-color:var(--color-yellow-50)}.bg-yellow-100{background-color:var(--color-yellow-100)}.bg-yellow-500{background-color:var(--color-yellow-500)}.p-0{padding:calc(var(--spacing)*0)}.p-1{padding:calc(var(--spacing)*1)}.p-2{padding:calc(var(--spacing)*2)}.p-3{padding:calc(var(--spacing)*3)}.p-4{padding:calc(var(--spacing)*4)}.p-8{padding:calc(var(--spacing)*8)}.px-2{padding-inline:calc(var(--spacing)*2)}.px-3{padding-inline:calc(var(--spacing)*3)}.py-0\.5{padding-block:calc(var(--spacing)*.5)}.py-1{padding-block:calc(var(--spacing)*1)}.py-2{padding-block:calc(var(--spacing)*2)}.py-3{padding-block:calc(var(--spacing)*3)}.py-4{padding-block:calc(var(--spacing)*4)}.pt-1{padding-top:calc(var(--spacing)*1)}.pt-2{padding-top:calc(var(--spacing)*2)}.pb-2{padding-bottom:calc(var(--spacing)*2)}.text-center{text-align:center}.text-left{text-align:left}.text-right{text-align:right}.font-mono{font-family:var(--font-mono)}.text-sm{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.text-xs{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height))}.font-medium{--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium)}.font-semibold{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold)}.break-words{overflow-wrap:break-word}.break-all{word-break:break-all}.whitespace-nowrap{white-space:nowrap}.whitespace-pre-wrap{white-space:pre-wrap}.text-blue-700{color:var(--color-blue-700)}.text-blue-900{color:var(--color-blue-900)}.text-gray-400{color:var(--color-gray-400)}.text-gray-500{color:var(--color-gray-500)}.text-gray-600{color:var(--color-gray-600)}.text-gray-700{color:var(--color-gray-700)}.text-gray-800{color:var(--color-gray-800)}.text-gray-900{color:var(--color-gray-900)}.text-green-700{color:var(--color-green-700)}.text-green-800{color:var(--color-green-800)}.text-green-900{color:var(--color-green-900)}.text-red-700{color:var(--color-red-700)}.text-yellow-700{color:var(--color-yellow-700)}.text-yellow-800{color:var(--color-yellow-800)}.text-yellow-900{color:var(--color-yellow-900)}.capitalize{text-transform:capitalize}.lowercase{text-transform:lowercase}.uppercase{text-transform:uppercase}.italic{font-style:italic}.underline{text-decoration-line:underline}.shadow{--tw-shadow:0 1px 3px 0 var(--tw-shadow-color,#0000001a),0 1px 2px -1px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow),var(--tw-inset-ring-shadow),var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow)}.outline{outline-style:var(--tw-outline-style);outline-width:1px}.blur{--tw-blur:blur(8px);filter:var(--tw-blur,)var(--tw-brightness,)var(--tw-contrast,)var(--tw-grayscale,)var(--tw-hue-rotate,)var(--tw-invert,)var(--tw-saturate,)var(--tw-sepia,)var(--tw-drop-shadow,)}.filter{filter:var(--tw-blur,)var(--tw-brightness,)var(--tw-contrast,)var(--tw-grayscale,)var(--tw-hue-rotate,)var(--tw-invert,)var(--tw-saturate,)var(--tw-sepia,)var(--tw-drop-shadow,)}.transition{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to,opacity,box-shadow,transform,translate,scale,rotate,filter,-webkit-backdrop-filter,backdrop-filter,display,visibility,content-visibility,overlay,pointer-events;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-colors{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-transform{transition-property:transform,translate,scale,rotate;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.duration-200{--tw-duration:.2s;transition-duration:.2s}.select-none{-webkit-user-select:none;user-select:none}@media (hover:hover){.hover\:border-gray-400:hover{border-color:var(--color-gray-400)}.hover\:bg-gray-50:hover{background-color:var(--color-gray-50)}.hover\:bg-gray-100:hover{background-color:var(--color-gray-100)}.hover\:bg-gray-200:hover{background-color:var(--color-gray-200)}.hover\:bg-gray-400:hover{background-color:var(--color-gray-400)}.hover\:text-gray-900:hover{color:var(--color-gray-900)}.hover\:no-underline:hover{text-decoration-line:none}}.focus\:border-gray-500:focus{border-color:var(--color-gray-500)}.focus\:outline-none:focus{--tw-outline-style:none;outline-style:none}@media (min-width:64rem){.lg\:max-w-md{max-width:var(--container-md)}}@media (min-width:80rem){.xl\:max-w-lg{max-width:var(--container-lg)}}}@property --tw-rotate-x{syntax:"*";inherits:false}@property --tw-rotate-y{syntax:"*";inherits:false}@property --tw-rotate-z{syntax:"*";inherits:false}@property --tw-skew-x{syntax:"*";inherits:false}@property --tw-skew-y{syntax:"*";inherits:false}@property --tw-space-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-space-x-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-divide-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-border-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-font-weight{syntax:"*";inherits:false}@property --tw-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-shadow-color{syntax:"*";inherits:false}@property --tw-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-inset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-shadow-color{syntax:"*";inherits:false}@property --tw-inset-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-ring-color{syntax:"*";inherits:false}@property --tw-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-ring-color{syntax:"*";inherits:false}@property --tw-inset-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-ring-inset{syntax:"*";inherits:false}@property --tw-ring-offset-width{syntax:"";inherits:false;initial-value:0}@property --tw-ring-offset-color{syntax:"*";inherits:false;initial-value:#fff}@property --tw-ring-offset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-outline-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-blur{syntax:"*";inherits:false}@property --tw-brightness{syntax:"*";inherits:false}@property --tw-contrast{syntax:"*";inherits:false}@property --tw-grayscale{syntax:"*";inherits:false}@property --tw-hue-rotate{syntax:"*";inherits:false}@property --tw-invert{syntax:"*";inherits:false}@property --tw-opacity{syntax:"*";inherits:false}@property --tw-saturate{syntax:"*";inherits:false}@property --tw-sepia{syntax:"*";inherits:false}@property --tw-drop-shadow{syntax:"*";inherits:false}@property --tw-drop-shadow-color{syntax:"*";inherits:false}@property --tw-drop-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-drop-shadow-size{syntax:"*";inherits:false}@property --tw-duration{syntax:"*";inherits:false}@keyframes spin{to{transform:rotate(360deg)}} diff --git a/vite-app/dist/assets/index-D_nkLTVA.js b/vite-app/dist/assets/index-D_nkLTVA.js deleted file mode 100644 index e813086a..00000000 --- a/vite-app/dist/assets/index-D_nkLTVA.js +++ /dev/null @@ -1,88 +0,0 @@ -(function(){const l=document.createElement("link").relList;if(l&&l.supports&&l.supports("modulepreload"))return;for(const o of document.querySelectorAll('link[rel="modulepreload"]'))r(o);new MutationObserver(o=>{for(const f of o)if(f.type==="childList")for(const d of f.addedNodes)d.tagName==="LINK"&&d.rel==="modulepreload"&&r(d)}).observe(document,{childList:!0,subtree:!0});function u(o){const f={};return o.integrity&&(f.integrity=o.integrity),o.referrerPolicy&&(f.referrerPolicy=o.referrerPolicy),o.crossOrigin==="use-credentials"?f.credentials="include":o.crossOrigin==="anonymous"?f.credentials="omit":f.credentials="same-origin",f}function r(o){if(o.ep)return;o.ep=!0;const f=u(o);fetch(o.href,f)}})();function ag(n){return n&&n.__esModule&&Object.prototype.hasOwnProperty.call(n,"default")?n.default:n}var Xc={exports:{}},Wi={};/** - * @license React - * react-jsx-runtime.production.js - * - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */var km;function cb(){if(km)return Wi;km=1;var n=Symbol.for("react.transitional.element"),l=Symbol.for("react.fragment");function u(r,o,f){var d=null;if(f!==void 0&&(d=""+f),o.key!==void 0&&(d=""+o.key),"key"in o){f={};for(var v in o)v!=="key"&&(f[v]=o[v])}else f=o;return o=f.ref,{$$typeof:n,type:r,key:d,ref:o!==void 0?o:null,props:f}}return Wi.Fragment=l,Wi.jsx=u,Wi.jsxs=u,Wi}var qm;function fb(){return qm||(qm=1,Xc.exports=cb()),Xc.exports}var x=fb(),Kc={exports:{}},oe={};/** - * @license React - * react.production.js - * - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */var Vm;function db(){if(Vm)return oe;Vm=1;var n=Symbol.for("react.transitional.element"),l=Symbol.for("react.portal"),u=Symbol.for("react.fragment"),r=Symbol.for("react.strict_mode"),o=Symbol.for("react.profiler"),f=Symbol.for("react.consumer"),d=Symbol.for("react.context"),v=Symbol.for("react.forward_ref"),m=Symbol.for("react.suspense"),p=Symbol.for("react.memo"),_=Symbol.for("react.lazy"),O=Symbol.iterator;function w(b){return b===null||typeof b!="object"?null:(b=O&&b[O]||b["@@iterator"],typeof b=="function"?b:null)}var Z={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},k=Object.assign,B={};function U(b,H,K){this.props=b,this.context=H,this.refs=B,this.updater=K||Z}U.prototype.isReactComponent={},U.prototype.setState=function(b,H){if(typeof b!="object"&&typeof b!="function"&&b!=null)throw Error("takes an object of state variables to update or a function which returns an object of state variables.");this.updater.enqueueSetState(this,b,H,"setState")},U.prototype.forceUpdate=function(b){this.updater.enqueueForceUpdate(this,b,"forceUpdate")};function $(){}$.prototype=U.prototype;function X(b,H,K){this.props=b,this.context=H,this.refs=B,this.updater=K||Z}var q=X.prototype=new $;q.constructor=X,k(q,U.prototype),q.isPureReactComponent=!0;var W=Array.isArray,Q={H:null,A:null,T:null,S:null,V:null},he=Object.prototype.hasOwnProperty;function xe(b,H,K,Y,ee,pe){return K=pe.ref,{$$typeof:n,type:b,key:H,ref:K!==void 0?K:null,props:pe}}function Xe(b,H){return xe(b.type,H,void 0,void 0,void 0,b.props)}function ae(b){return typeof b=="object"&&b!==null&&b.$$typeof===n}function qe(b){var H={"=":"=0",":":"=2"};return"$"+b.replace(/[=:]/g,function(K){return H[K]})}var Fe=/\/+/g;function Ke(b,H){return typeof b=="object"&&b!==null&&b.key!=null?qe(""+b.key):H.toString(36)}function Pt(){}function Kn(b){switch(b.status){case"fulfilled":return b.value;case"rejected":throw b.reason;default:switch(typeof b.status=="string"?b.then(Pt,Pt):(b.status="pending",b.then(function(H){b.status==="pending"&&(b.status="fulfilled",b.value=H)},function(H){b.status==="pending"&&(b.status="rejected",b.reason=H)})),b.status){case"fulfilled":return b.value;case"rejected":throw b.reason}}throw b}function lt(b,H,K,Y,ee){var pe=typeof b;(pe==="undefined"||pe==="boolean")&&(b=null);var re=!1;if(b===null)re=!0;else switch(pe){case"bigint":case"string":case"number":re=!0;break;case"object":switch(b.$$typeof){case n:case l:re=!0;break;case _:return re=b._init,lt(re(b._payload),H,K,Y,ee)}}if(re)return ee=ee(b),re=Y===""?"."+Ke(b,0):Y,W(ee)?(K="",re!=null&&(K=re.replace(Fe,"$&/")+"/"),lt(ee,H,K,"",function(Qn){return Qn})):ee!=null&&(ae(ee)&&(ee=Xe(ee,K+(ee.key==null||b&&b.key===ee.key?"":(""+ee.key).replace(Fe,"$&/")+"/")+re)),H.push(ee)),1;re=0;var St=Y===""?".":Y+":";if(W(b))for(var je=0;je>>1,b=N[Te];if(0>>1;Teo(Y,le))eeo(pe,Y)?(N[Te]=pe,N[ee]=le,Te=ee):(N[Te]=Y,N[K]=le,Te=K);else if(eeo(pe,le))N[Te]=pe,N[ee]=le,Te=ee;else break e}}return G}function o(N,G){var le=N.sortIndex-G.sortIndex;return le!==0?le:N.id-G.id}if(n.unstable_now=void 0,typeof performance=="object"&&typeof performance.now=="function"){var f=performance;n.unstable_now=function(){return f.now()}}else{var d=Date,v=d.now();n.unstable_now=function(){return d.now()-v}}var m=[],p=[],_=1,O=null,w=3,Z=!1,k=!1,B=!1,U=!1,$=typeof setTimeout=="function"?setTimeout:null,X=typeof clearTimeout=="function"?clearTimeout:null,q=typeof setImmediate<"u"?setImmediate:null;function W(N){for(var G=u(p);G!==null;){if(G.callback===null)r(p);else if(G.startTime<=N)r(p),G.sortIndex=G.expirationTime,l(m,G);else break;G=u(p)}}function Q(N){if(B=!1,W(N),!k)if(u(m)!==null)k=!0,he||(he=!0,Ke());else{var G=u(p);G!==null&<(Q,G.startTime-N)}}var he=!1,xe=-1,Xe=5,ae=-1;function qe(){return U?!0:!(n.unstable_now()-aeN&&qe());){var Te=O.callback;if(typeof Te=="function"){O.callback=null,w=O.priorityLevel;var b=Te(O.expirationTime<=N);if(N=n.unstable_now(),typeof b=="function"){O.callback=b,W(N),G=!0;break t}O===u(m)&&r(m),W(N)}else r(m);O=u(m)}if(O!==null)G=!0;else{var H=u(p);H!==null&<(Q,H.startTime-N),G=!1}}break e}finally{O=null,w=le,Z=!1}G=void 0}}finally{G?Ke():he=!1}}}var Ke;if(typeof q=="function")Ke=function(){q(Fe)};else if(typeof MessageChannel<"u"){var Pt=new MessageChannel,Kn=Pt.port2;Pt.port1.onmessage=Fe,Ke=function(){Kn.postMessage(null)}}else Ke=function(){$(Fe,0)};function lt(N,G){xe=$(function(){N(n.unstable_now())},G)}n.unstable_IdlePriority=5,n.unstable_ImmediatePriority=1,n.unstable_LowPriority=4,n.unstable_NormalPriority=3,n.unstable_Profiling=null,n.unstable_UserBlockingPriority=2,n.unstable_cancelCallback=function(N){N.callback=null},n.unstable_forceFrameRate=function(N){0>N||125Te?(N.sortIndex=le,l(p,N),u(m)===null&&N===u(p)&&(B?(X(xe),xe=-1):B=!0,lt(Q,le-Te))):(N.sortIndex=b,l(m,N),k||Z||(k=!0,he||(he=!0,Ke()))),N},n.unstable_shouldYield=qe,n.unstable_wrapCallback=function(N){var G=w;return function(){var le=w;w=G;try{return N.apply(this,arguments)}finally{w=le}}}}(Pc)),Pc}var Xm;function vb(){return Xm||(Xm=1,Jc.exports=hb()),Jc.exports}var Wc={exports:{}},dt={};/** - * @license React - * react-dom.production.js - * - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */var Km;function mb(){if(Km)return dt;Km=1;var n=fo();function l(m){var p="https://react.dev/errors/"+m;if(1"u"||typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE!="function"))try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(n)}catch(l){console.error(l)}}return n(),Wc.exports=mb(),Wc.exports}/** - * @license React - * react-dom-client.production.js - * - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */var Jm;function pb(){if(Jm)return Fi;Jm=1;var n=vb(),l=fo(),u=lg();function r(e){var t="https://react.dev/errors/"+e;if(1b||(e.current=Te[b],Te[b]=null,b--)}function Y(e,t){b++,Te[b]=e.current,e.current=t}var ee=H(null),pe=H(null),re=H(null),St=H(null);function je(e,t){switch(Y(re,t),Y(pe,e),Y(ee,null),t.nodeType){case 9:case 11:e=(e=t.documentElement)&&(e=e.namespaceURI)?mm(e):0;break;default:if(e=t.tagName,t=t.namespaceURI)t=mm(t),e=pm(t,e);else switch(e){case"svg":e=1;break;case"math":e=2;break;default:e=0}}K(ee),Y(ee,e)}function Qn(){K(ee),K(pe),K(re)}function No(e){e.memoizedState!==null&&Y(St,e);var t=ee.current,a=pm(t,e.type);t!==a&&(Y(pe,e),Y(ee,a))}function bu(e){pe.current===e&&(K(ee),K(pe)),St.current===e&&(K(St),Xi._currentValue=le)}var jo=Object.prototype.hasOwnProperty,Mo=n.unstable_scheduleCallback,Co=n.unstable_cancelCallback,q_=n.unstable_shouldYield,V_=n.unstable_requestPaint,on=n.unstable_now,Y_=n.unstable_getCurrentPriorityLevel,Jf=n.unstable_ImmediatePriority,Pf=n.unstable_UserBlockingPriority,Su=n.unstable_NormalPriority,G_=n.unstable_LowPriority,Wf=n.unstable_IdlePriority,X_=n.log,K_=n.unstable_setDisableYieldValue,ei=null,xt=null;function Jn(e){if(typeof X_=="function"&&K_(e),xt&&typeof xt.setStrictMode=="function")try{xt.setStrictMode(ei,e)}catch{}}var Et=Math.clz32?Math.clz32:P_,Q_=Math.log,J_=Math.LN2;function P_(e){return e>>>=0,e===0?32:31-(Q_(e)/J_|0)|0}var xu=256,Eu=4194304;function Oa(e){var t=e&42;if(t!==0)return t;switch(e&-e){case 1:return 1;case 2:return 2;case 4:return 4;case 8:return 8;case 16:return 16;case 32:return 32;case 64:return 64;case 128:return 128;case 256:case 512:case 1024:case 2048:case 4096:case 8192:case 16384:case 32768:case 65536:case 131072:case 262144:case 524288:case 1048576:case 2097152:return e&4194048;case 4194304:case 8388608:case 16777216:case 33554432:return e&62914560;case 67108864:return 67108864;case 134217728:return 134217728;case 268435456:return 268435456;case 536870912:return 536870912;case 1073741824:return 0;default:return e}}function Ou(e,t,a){var i=e.pendingLanes;if(i===0)return 0;var s=0,c=e.suspendedLanes,h=e.pingedLanes;e=e.warmLanes;var g=i&134217727;return g!==0?(i=g&~c,i!==0?s=Oa(i):(h&=g,h!==0?s=Oa(h):a||(a=g&~e,a!==0&&(s=Oa(a))))):(g=i&~c,g!==0?s=Oa(g):h!==0?s=Oa(h):a||(a=i&~e,a!==0&&(s=Oa(a)))),s===0?0:t!==0&&t!==s&&(t&c)===0&&(c=s&-s,a=t&-t,c>=a||c===32&&(a&4194048)!==0)?t:s}function ti(e,t){return(e.pendingLanes&~(e.suspendedLanes&~e.pingedLanes)&t)===0}function W_(e,t){switch(e){case 1:case 2:case 4:case 8:case 64:return t+250;case 16:case 32:case 128:case 256:case 512:case 1024:case 2048:case 4096:case 8192:case 16384:case 32768:case 65536:case 131072:case 262144:case 524288:case 1048576:case 2097152:return t+5e3;case 4194304:case 8388608:case 16777216:case 33554432:return-1;case 67108864:case 134217728:case 268435456:case 536870912:case 1073741824:return-1;default:return-1}}function Ff(){var e=xu;return xu<<=1,(xu&4194048)===0&&(xu=256),e}function If(){var e=Eu;return Eu<<=1,(Eu&62914560)===0&&(Eu=4194304),e}function Uo(e){for(var t=[],a=0;31>a;a++)t.push(e);return t}function ni(e,t){e.pendingLanes|=t,t!==268435456&&(e.suspendedLanes=0,e.pingedLanes=0,e.warmLanes=0)}function F_(e,t,a,i,s,c){var h=e.pendingLanes;e.pendingLanes=a,e.suspendedLanes=0,e.pingedLanes=0,e.warmLanes=0,e.expiredLanes&=a,e.entangledLanes&=a,e.errorRecoveryDisabledLanes&=a,e.shellSuspendCounter=0;var g=e.entanglements,y=e.expirationTimes,T=e.hiddenUpdates;for(a=h&~a;0)":-1s||y[i]!==T[s]){var j=` -`+y[i].replace(" at new "," at ");return e.displayName&&j.includes("")&&(j=j.replace("",e.displayName)),j}while(1<=i&&0<=s);break}}}finally{ko=!1,Error.prepareStackTrace=a}return(a=e?e.displayName||e.name:"")?il(a):""}function ly(e){switch(e.tag){case 26:case 27:case 5:return il(e.type);case 16:return il("Lazy");case 13:return il("Suspense");case 19:return il("SuspenseList");case 0:case 15:return qo(e.type,!1);case 11:return qo(e.type.render,!1);case 1:return qo(e.type,!0);case 31:return il("Activity");default:return""}}function sd(e){try{var t="";do t+=ly(e),e=e.return;while(e);return t}catch(a){return` -Error generating stack: `+a.message+` -`+a.stack}}function Bt(e){switch(typeof e){case"bigint":case"boolean":case"number":case"string":case"undefined":return e;case"object":return e;default:return""}}function cd(e){var t=e.type;return(e=e.nodeName)&&e.toLowerCase()==="input"&&(t==="checkbox"||t==="radio")}function iy(e){var t=cd(e)?"checked":"value",a=Object.getOwnPropertyDescriptor(e.constructor.prototype,t),i=""+e[t];if(!e.hasOwnProperty(t)&&typeof a<"u"&&typeof a.get=="function"&&typeof a.set=="function"){var s=a.get,c=a.set;return Object.defineProperty(e,t,{configurable:!0,get:function(){return s.call(this)},set:function(h){i=""+h,c.call(this,h)}}),Object.defineProperty(e,t,{enumerable:a.enumerable}),{getValue:function(){return i},setValue:function(h){i=""+h},stopTracking:function(){e._valueTracker=null,delete e[t]}}}}function wu(e){e._valueTracker||(e._valueTracker=iy(e))}function fd(e){if(!e)return!1;var t=e._valueTracker;if(!t)return!0;var a=t.getValue(),i="";return e&&(i=cd(e)?e.checked?"true":"false":e.value),e=i,e!==a?(t.setValue(e),!0):!1}function zu(e){if(e=e||(typeof document<"u"?document:void 0),typeof e>"u")return null;try{return e.activeElement||e.body}catch{return e.body}}var uy=/[\n"\\]/g;function Lt(e){return e.replace(uy,function(t){return"\\"+t.charCodeAt(0).toString(16)+" "})}function Vo(e,t,a,i,s,c,h,g){e.name="",h!=null&&typeof h!="function"&&typeof h!="symbol"&&typeof h!="boolean"?e.type=h:e.removeAttribute("type"),t!=null?h==="number"?(t===0&&e.value===""||e.value!=t)&&(e.value=""+Bt(t)):e.value!==""+Bt(t)&&(e.value=""+Bt(t)):h!=="submit"&&h!=="reset"||e.removeAttribute("value"),t!=null?Yo(e,h,Bt(t)):a!=null?Yo(e,h,Bt(a)):i!=null&&e.removeAttribute("value"),s==null&&c!=null&&(e.defaultChecked=!!c),s!=null&&(e.checked=s&&typeof s!="function"&&typeof s!="symbol"),g!=null&&typeof g!="function"&&typeof g!="symbol"&&typeof g!="boolean"?e.name=""+Bt(g):e.removeAttribute("name")}function dd(e,t,a,i,s,c,h,g){if(c!=null&&typeof c!="function"&&typeof c!="symbol"&&typeof c!="boolean"&&(e.type=c),t!=null||a!=null){if(!(c!=="submit"&&c!=="reset"||t!=null))return;a=a!=null?""+Bt(a):"",t=t!=null?""+Bt(t):a,g||t===e.value||(e.value=t),e.defaultValue=t}i=i??s,i=typeof i!="function"&&typeof i!="symbol"&&!!i,e.checked=g?e.checked:!!i,e.defaultChecked=!!i,h!=null&&typeof h!="function"&&typeof h!="symbol"&&typeof h!="boolean"&&(e.name=h)}function Yo(e,t,a){t==="number"&&zu(e.ownerDocument)===e||e.defaultValue===""+a||(e.defaultValue=""+a)}function ul(e,t,a,i){if(e=e.options,t){t={};for(var s=0;s"u"||typeof window.document>"u"||typeof window.document.createElement>"u"),Jo=!1;if(On)try{var ui={};Object.defineProperty(ui,"passive",{get:function(){Jo=!0}}),window.addEventListener("test",ui,ui),window.removeEventListener("test",ui,ui)}catch{Jo=!1}var Wn=null,Po=null,Du=null;function yd(){if(Du)return Du;var e,t=Po,a=t.length,i,s="value"in Wn?Wn.value:Wn.textContent,c=s.length;for(e=0;e=si),Ad=" ",Td=!1;function wd(e,t){switch(e){case"keyup":return Cy.indexOf(t.keyCode)!==-1;case"keydown":return t.keyCode!==229;case"keypress":case"mousedown":case"focusout":return!0;default:return!1}}function zd(e){return e=e.detail,typeof e=="object"&&"data"in e?e.data:null}var cl=!1;function Zy(e,t){switch(e){case"compositionend":return zd(t);case"keypress":return t.which!==32?null:(Td=!0,Ad);case"textInput":return e=t.data,e===Ad&&Td?null:e;default:return null}}function By(e,t){if(cl)return e==="compositionend"||!ts&&wd(e,t)?(e=yd(),Du=Po=Wn=null,cl=!1,e):null;switch(e){case"paste":return null;case"keypress":if(!(t.ctrlKey||t.altKey||t.metaKey)||t.ctrlKey&&t.altKey){if(t.char&&1=t)return{node:a,offset:t-e};e=i}e:{for(;a;){if(a.nextSibling){a=a.nextSibling;break e}a=a.parentNode}a=void 0}a=Zd(a)}}function Ld(e,t){return e&&t?e===t?!0:e&&e.nodeType===3?!1:t&&t.nodeType===3?Ld(e,t.parentNode):"contains"in e?e.contains(t):e.compareDocumentPosition?!!(e.compareDocumentPosition(t)&16):!1:!1}function $d(e){e=e!=null&&e.ownerDocument!=null&&e.ownerDocument.defaultView!=null?e.ownerDocument.defaultView:window;for(var t=zu(e.document);t instanceof e.HTMLIFrameElement;){try{var a=typeof t.contentWindow.location.href=="string"}catch{a=!1}if(a)e=t.contentWindow;else break;t=zu(e.document)}return t}function ls(e){var t=e&&e.nodeName&&e.nodeName.toLowerCase();return t&&(t==="input"&&(e.type==="text"||e.type==="search"||e.type==="tel"||e.type==="url"||e.type==="password")||t==="textarea"||e.contentEditable==="true")}var Gy=On&&"documentMode"in document&&11>=document.documentMode,fl=null,is=null,hi=null,us=!1;function Hd(e,t,a){var i=a.window===a?a.document:a.nodeType===9?a:a.ownerDocument;us||fl==null||fl!==zu(i)||(i=fl,"selectionStart"in i&&ls(i)?i={start:i.selectionStart,end:i.selectionEnd}:(i=(i.ownerDocument&&i.ownerDocument.defaultView||window).getSelection(),i={anchorNode:i.anchorNode,anchorOffset:i.anchorOffset,focusNode:i.focusNode,focusOffset:i.focusOffset}),hi&&di(hi,i)||(hi=i,i=br(is,"onSelect"),0>=h,s-=h,Tn=1<<32-Et(t)+s|a<c?c:8;var h=N.T,g={};N.T=g,Gs(e,!1,t,a);try{var y=s(),T=N.S;if(T!==null&&T(g,y),y!==null&&typeof y=="object"&&typeof y.then=="function"){var j=e0(y,i);zi(e,t,j,Rt(e))}else zi(e,t,i,Rt(e))}catch(L){zi(e,t,{then:function(){},status:"rejected",reason:L},Rt())}finally{G.p=c,N.T=h}}function i0(){}function Vs(e,t,a,i){if(e.tag!==5)throw Error(r(476));var s=kh(e).queue;Hh(e,s,t,le,a===null?i0:function(){return qh(e),a(i)})}function kh(e){var t=e.memoizedState;if(t!==null)return t;t={memoizedState:le,baseState:le,baseQueue:null,queue:{pending:null,lanes:0,dispatch:null,lastRenderedReducer:Dn,lastRenderedState:le},next:null};var a={};return t.next={memoizedState:a,baseState:a,baseQueue:null,queue:{pending:null,lanes:0,dispatch:null,lastRenderedReducer:Dn,lastRenderedState:a},next:null},e.memoizedState=t,e=e.alternate,e!==null&&(e.memoizedState=t),t}function qh(e){var t=kh(e).next.queue;zi(e,t,{},Rt())}function Ys(){return ft(Xi)}function Vh(){return Je().memoizedState}function Yh(){return Je().memoizedState}function u0(e){for(var t=e.return;t!==null;){switch(t.tag){case 24:case 3:var a=Rt();e=ea(a);var i=ta(t,e,a);i!==null&&(Dt(i,t,a),xi(i,t,a)),t={cache:bs()},e.payload=t;return}t=t.return}}function r0(e,t,a){var i=Rt();a={lane:i,revertLane:0,action:a,hasEagerState:!1,eagerState:null,next:null},er(e)?Xh(t,a):(a=cs(e,t,a,i),a!==null&&(Dt(a,e,i),Kh(a,t,i)))}function Gh(e,t,a){var i=Rt();zi(e,t,a,i)}function zi(e,t,a,i){var s={lane:i,revertLane:0,action:a,hasEagerState:!1,eagerState:null,next:null};if(er(e))Xh(t,s);else{var c=e.alternate;if(e.lanes===0&&(c===null||c.lanes===0)&&(c=t.lastRenderedReducer,c!==null))try{var h=t.lastRenderedState,g=c(h,a);if(s.hasEagerState=!0,s.eagerState=g,Ot(g,h))return Bu(e,t,s,0),Re===null&&Zu(),!1}catch{}finally{}if(a=cs(e,t,s,i),a!==null)return Dt(a,e,i),Kh(a,t,i),!0}return!1}function Gs(e,t,a,i){if(i={lane:2,revertLane:Ec(),action:i,hasEagerState:!1,eagerState:null,next:null},er(e)){if(t)throw Error(r(479))}else t=cs(e,a,i,2),t!==null&&Dt(t,e,2)}function er(e){var t=e.alternate;return e===se||t!==null&&t===se}function Xh(e,t){Sl=Qu=!0;var a=e.pending;a===null?t.next=t:(t.next=a.next,a.next=t),e.pending=t}function Kh(e,t,a){if((a&4194048)!==0){var i=t.lanes;i&=e.pendingLanes,a|=i,t.lanes=a,td(e,a)}}var tr={readContext:ft,use:Pu,useCallback:Ve,useContext:Ve,useEffect:Ve,useImperativeHandle:Ve,useLayoutEffect:Ve,useInsertionEffect:Ve,useMemo:Ve,useReducer:Ve,useRef:Ve,useState:Ve,useDebugValue:Ve,useDeferredValue:Ve,useTransition:Ve,useSyncExternalStore:Ve,useId:Ve,useHostTransitionStatus:Ve,useFormState:Ve,useActionState:Ve,useOptimistic:Ve,useMemoCache:Ve,useCacheRefresh:Ve},Qh={readContext:ft,use:Pu,useCallback:function(e,t){return pt().memoizedState=[e,t===void 0?null:t],e},useContext:ft,useEffect:Nh,useImperativeHandle:function(e,t,a){a=a!=null?a.concat([e]):null,Iu(4194308,4,Uh.bind(null,t,e),a)},useLayoutEffect:function(e,t){return Iu(4194308,4,e,t)},useInsertionEffect:function(e,t){Iu(4,2,e,t)},useMemo:function(e,t){var a=pt();t=t===void 0?null:t;var i=e();if(Ba){Jn(!0);try{e()}finally{Jn(!1)}}return a.memoizedState=[i,t],i},useReducer:function(e,t,a){var i=pt();if(a!==void 0){var s=a(t);if(Ba){Jn(!0);try{a(t)}finally{Jn(!1)}}}else s=t;return i.memoizedState=i.baseState=s,e={pending:null,lanes:0,dispatch:null,lastRenderedReducer:e,lastRenderedState:s},i.queue=e,e=e.dispatch=r0.bind(null,se,e),[i.memoizedState,e]},useRef:function(e){var t=pt();return e={current:e},t.memoizedState=e},useState:function(e){e=$s(e);var t=e.queue,a=Gh.bind(null,se,t);return t.dispatch=a,[e.memoizedState,a]},useDebugValue:ks,useDeferredValue:function(e,t){var a=pt();return qs(a,e,t)},useTransition:function(){var e=$s(!1);return e=Hh.bind(null,se,e.queue,!0,!1),pt().memoizedState=e,[!1,e]},useSyncExternalStore:function(e,t,a){var i=se,s=pt();if(ye){if(a===void 0)throw Error(r(407));a=a()}else{if(a=t(),Re===null)throw Error(r(349));(ve&124)!==0||mh(i,t,a)}s.memoizedState=a;var c={value:a,getSnapshot:t};return s.queue=c,Nh(gh.bind(null,i,c,e),[e]),i.flags|=2048,El(9,Fu(),ph.bind(null,i,c,a,t),null),a},useId:function(){var e=pt(),t=Re.identifierPrefix;if(ye){var a=wn,i=Tn;a=(i&~(1<<32-Et(i)-1)).toString(32)+a,t="Ā«"+t+"R"+a,a=Ju++,0ne?(nt=I,I=null):nt=I.sibling;var ge=z(E,I,A[ne],C);if(ge===null){I===null&&(I=nt);break}e&&I&&ge.alternate===null&&t(E,I),S=c(ge,S,ne),ce===null?J=ge:ce.sibling=ge,ce=ge,I=nt}if(ne===A.length)return a(E,I),ye&&Na(E,ne),J;if(I===null){for(;nene?(nt=I,I=null):nt=I.sibling;var _a=z(E,I,ge.value,C);if(_a===null){I===null&&(I=nt);break}e&&I&&_a.alternate===null&&t(E,I),S=c(_a,S,ne),ce===null?J=_a:ce.sibling=_a,ce=_a,I=nt}if(ge.done)return a(E,I),ye&&Na(E,ne),J;if(I===null){for(;!ge.done;ne++,ge=A.next())ge=L(E,ge.value,C),ge!==null&&(S=c(ge,S,ne),ce===null?J=ge:ce.sibling=ge,ce=ge);return ye&&Na(E,ne),J}for(I=i(I);!ge.done;ne++,ge=A.next())ge=D(I,E,ne,ge.value,C),ge!==null&&(e&&ge.alternate!==null&&I.delete(ge.key===null?ne:ge.key),S=c(ge,S,ne),ce===null?J=ge:ce.sibling=ge,ce=ge);return e&&I.forEach(function(sb){return t(E,sb)}),ye&&Na(E,ne),J}function Ae(E,S,A,C){if(typeof A=="object"&&A!==null&&A.type===k&&A.key===null&&(A=A.props.children),typeof A=="object"&&A!==null){switch(A.$$typeof){case w:e:{for(var J=A.key;S!==null;){if(S.key===J){if(J=A.type,J===k){if(S.tag===7){a(E,S.sibling),C=s(S,A.props.children),C.return=E,E=C;break e}}else if(S.elementType===J||typeof J=="object"&&J!==null&&J.$$typeof===Xe&&Ph(J)===S.type){a(E,S.sibling),C=s(S,A.props),Di(C,A),C.return=E,E=C;break e}a(E,S);break}else t(E,S);S=S.sibling}A.type===k?(C=Ra(A.props.children,E.mode,C,A.key),C.return=E,E=C):(C=$u(A.type,A.key,A.props,null,E.mode,C),Di(C,A),C.return=E,E=C)}return h(E);case Z:e:{for(J=A.key;S!==null;){if(S.key===J)if(S.tag===4&&S.stateNode.containerInfo===A.containerInfo&&S.stateNode.implementation===A.implementation){a(E,S.sibling),C=s(S,A.children||[]),C.return=E,E=C;break e}else{a(E,S);break}else t(E,S);S=S.sibling}C=hs(A,E.mode,C),C.return=E,E=C}return h(E);case Xe:return J=A._init,A=J(A._payload),Ae(E,S,A,C)}if(lt(A))return ie(E,S,A,C);if(Ke(A)){if(J=Ke(A),typeof J!="function")throw Error(r(150));return A=J.call(A),te(E,S,A,C)}if(typeof A.then=="function")return Ae(E,S,nr(A),C);if(A.$$typeof===q)return Ae(E,S,Vu(E,A),C);ar(E,A)}return typeof A=="string"&&A!==""||typeof A=="number"||typeof A=="bigint"?(A=""+A,S!==null&&S.tag===6?(a(E,S.sibling),C=s(S,A),C.return=E,E=C):(a(E,S),C=ds(A,E.mode,C),C.return=E,E=C),h(E)):a(E,S)}return function(E,S,A,C){try{Ri=0;var J=Ae(E,S,A,C);return Ol=null,J}catch(I){if(I===bi||I===Gu)throw I;var ce=At(29,I,null,E.mode);return ce.lanes=C,ce.return=E,ce}finally{}}}var Al=Wh(!0),Fh=Wh(!1),Vt=H(null),cn=null;function aa(e){var t=e.alternate;Y(We,We.current&1),Y(Vt,e),cn===null&&(t===null||bl.current!==null||t.memoizedState!==null)&&(cn=e)}function Ih(e){if(e.tag===22){if(Y(We,We.current),Y(Vt,e),cn===null){var t=e.alternate;t!==null&&t.memoizedState!==null&&(cn=e)}}else la()}function la(){Y(We,We.current),Y(Vt,Vt.current)}function Nn(e){K(Vt),cn===e&&(cn=null),K(We)}var We=H(0);function lr(e){for(var t=e;t!==null;){if(t.tag===13){var a=t.memoizedState;if(a!==null&&(a=a.dehydrated,a===null||a.data==="$?"||Uc(a)))return t}else if(t.tag===19&&t.memoizedProps.revealOrder!==void 0){if((t.flags&128)!==0)return t}else if(t.child!==null){t.child.return=t,t=t.child;continue}if(t===e)break;for(;t.sibling===null;){if(t.return===null||t.return===e)return null;t=t.return}t.sibling.return=t.return,t=t.sibling}return null}function Xs(e,t,a,i){t=e.memoizedState,a=a(i,t),a=a==null?t:_({},t,a),e.memoizedState=a,e.lanes===0&&(e.updateQueue.baseState=a)}var Ks={enqueueSetState:function(e,t,a){e=e._reactInternals;var i=Rt(),s=ea(i);s.payload=t,a!=null&&(s.callback=a),t=ta(e,s,i),t!==null&&(Dt(t,e,i),xi(t,e,i))},enqueueReplaceState:function(e,t,a){e=e._reactInternals;var i=Rt(),s=ea(i);s.tag=1,s.payload=t,a!=null&&(s.callback=a),t=ta(e,s,i),t!==null&&(Dt(t,e,i),xi(t,e,i))},enqueueForceUpdate:function(e,t){e=e._reactInternals;var a=Rt(),i=ea(a);i.tag=2,t!=null&&(i.callback=t),t=ta(e,i,a),t!==null&&(Dt(t,e,a),xi(t,e,a))}};function ev(e,t,a,i,s,c,h){return e=e.stateNode,typeof e.shouldComponentUpdate=="function"?e.shouldComponentUpdate(i,c,h):t.prototype&&t.prototype.isPureReactComponent?!di(a,i)||!di(s,c):!0}function tv(e,t,a,i){e=t.state,typeof t.componentWillReceiveProps=="function"&&t.componentWillReceiveProps(a,i),typeof t.UNSAFE_componentWillReceiveProps=="function"&&t.UNSAFE_componentWillReceiveProps(a,i),t.state!==e&&Ks.enqueueReplaceState(t,t.state,null)}function La(e,t){var a=t;if("ref"in t){a={};for(var i in t)i!=="ref"&&(a[i]=t[i])}if(e=e.defaultProps){a===t&&(a=_({},a));for(var s in e)a[s]===void 0&&(a[s]=e[s])}return a}var ir=typeof reportError=="function"?reportError:function(e){if(typeof window=="object"&&typeof window.ErrorEvent=="function"){var t=new window.ErrorEvent("error",{bubbles:!0,cancelable:!0,message:typeof e=="object"&&e!==null&&typeof e.message=="string"?String(e.message):String(e),error:e});if(!window.dispatchEvent(t))return}else if(typeof process=="object"&&typeof process.emit=="function"){process.emit("uncaughtException",e);return}console.error(e)};function nv(e){ir(e)}function av(e){console.error(e)}function lv(e){ir(e)}function ur(e,t){try{var a=e.onUncaughtError;a(t.value,{componentStack:t.stack})}catch(i){setTimeout(function(){throw i})}}function iv(e,t,a){try{var i=e.onCaughtError;i(a.value,{componentStack:a.stack,errorBoundary:t.tag===1?t.stateNode:null})}catch(s){setTimeout(function(){throw s})}}function Qs(e,t,a){return a=ea(a),a.tag=3,a.payload={element:null},a.callback=function(){ur(e,t)},a}function uv(e){return e=ea(e),e.tag=3,e}function rv(e,t,a,i){var s=a.type.getDerivedStateFromError;if(typeof s=="function"){var c=i.value;e.payload=function(){return s(c)},e.callback=function(){iv(t,a,i)}}var h=a.stateNode;h!==null&&typeof h.componentDidCatch=="function"&&(e.callback=function(){iv(t,a,i),typeof s!="function"&&(ca===null?ca=new Set([this]):ca.add(this));var g=i.stack;this.componentDidCatch(i.value,{componentStack:g!==null?g:""})})}function s0(e,t,a,i,s){if(a.flags|=32768,i!==null&&typeof i=="object"&&typeof i.then=="function"){if(t=a.alternate,t!==null&&gi(t,a,s,!0),a=Vt.current,a!==null){switch(a.tag){case 13:return cn===null?_c():a.alternate===null&&ke===0&&(ke=3),a.flags&=-257,a.flags|=65536,a.lanes=s,i===Es?a.flags|=16384:(t=a.updateQueue,t===null?a.updateQueue=new Set([i]):t.add(i),bc(e,i,s)),!1;case 22:return a.flags|=65536,i===Es?a.flags|=16384:(t=a.updateQueue,t===null?(t={transitions:null,markerInstances:null,retryQueue:new Set([i])},a.updateQueue=t):(a=t.retryQueue,a===null?t.retryQueue=new Set([i]):a.add(i)),bc(e,i,s)),!1}throw Error(r(435,a.tag))}return bc(e,i,s),_c(),!1}if(ye)return t=Vt.current,t!==null?((t.flags&65536)===0&&(t.flags|=256),t.flags|=65536,t.lanes=s,i!==ps&&(e=Error(r(422),{cause:i}),pi($t(e,a)))):(i!==ps&&(t=Error(r(423),{cause:i}),pi($t(t,a))),e=e.current.alternate,e.flags|=65536,s&=-s,e.lanes|=s,i=$t(i,a),s=Qs(e.stateNode,i,s),Ts(e,s),ke!==4&&(ke=2)),!1;var c=Error(r(520),{cause:i});if(c=$t(c,a),Bi===null?Bi=[c]:Bi.push(c),ke!==4&&(ke=2),t===null)return!0;i=$t(i,a),a=t;do{switch(a.tag){case 3:return a.flags|=65536,e=s&-s,a.lanes|=e,e=Qs(a.stateNode,i,e),Ts(a,e),!1;case 1:if(t=a.type,c=a.stateNode,(a.flags&128)===0&&(typeof t.getDerivedStateFromError=="function"||c!==null&&typeof c.componentDidCatch=="function"&&(ca===null||!ca.has(c))))return a.flags|=65536,s&=-s,a.lanes|=s,s=uv(s),rv(s,e,a,i),Ts(a,s),!1}a=a.return}while(a!==null);return!1}var ov=Error(r(461)),et=!1;function it(e,t,a,i){t.child=e===null?Fh(t,null,a,i):Al(t,e.child,a,i)}function sv(e,t,a,i,s){a=a.render;var c=t.ref;if("ref"in i){var h={};for(var g in i)g!=="ref"&&(h[g]=i[g])}else h=i;return Ua(t),i=Ns(e,t,a,h,c,s),g=js(),e!==null&&!et?(Ms(e,t,s),jn(e,t,s)):(ye&&g&&vs(t),t.flags|=1,it(e,t,i,s),t.child)}function cv(e,t,a,i,s){if(e===null){var c=a.type;return typeof c=="function"&&!fs(c)&&c.defaultProps===void 0&&a.compare===null?(t.tag=15,t.type=c,fv(e,t,c,i,s)):(e=$u(a.type,null,i,t,t.mode,s),e.ref=t.ref,e.return=t,t.child=e)}if(c=e.child,!nc(e,s)){var h=c.memoizedProps;if(a=a.compare,a=a!==null?a:di,a(h,i)&&e.ref===t.ref)return jn(e,t,s)}return t.flags|=1,e=An(c,i),e.ref=t.ref,e.return=t,t.child=e}function fv(e,t,a,i,s){if(e!==null){var c=e.memoizedProps;if(di(c,i)&&e.ref===t.ref)if(et=!1,t.pendingProps=i=c,nc(e,s))(e.flags&131072)!==0&&(et=!0);else return t.lanes=e.lanes,jn(e,t,s)}return Js(e,t,a,i,s)}function dv(e,t,a){var i=t.pendingProps,s=i.children,c=e!==null?e.memoizedState:null;if(i.mode==="hidden"){if((t.flags&128)!==0){if(i=c!==null?c.baseLanes|a:a,e!==null){for(s=t.child=e.child,c=0;s!==null;)c=c|s.lanes|s.childLanes,s=s.sibling;t.childLanes=c&~i}else t.childLanes=0,t.child=null;return hv(e,t,i,a)}if((a&536870912)!==0)t.memoizedState={baseLanes:0,cachePool:null},e!==null&&Yu(t,c!==null?c.cachePool:null),c!==null?fh(t,c):zs(),Ih(t);else return t.lanes=t.childLanes=536870912,hv(e,t,c!==null?c.baseLanes|a:a,a)}else c!==null?(Yu(t,c.cachePool),fh(t,c),la(),t.memoizedState=null):(e!==null&&Yu(t,null),zs(),la());return it(e,t,s,a),t.child}function hv(e,t,a,i){var s=xs();return s=s===null?null:{parent:Pe._currentValue,pool:s},t.memoizedState={baseLanes:a,cachePool:s},e!==null&&Yu(t,null),zs(),Ih(t),e!==null&&gi(e,t,i,!0),null}function rr(e,t){var a=t.ref;if(a===null)e!==null&&e.ref!==null&&(t.flags|=4194816);else{if(typeof a!="function"&&typeof a!="object")throw Error(r(284));(e===null||e.ref!==a)&&(t.flags|=4194816)}}function Js(e,t,a,i,s){return Ua(t),a=Ns(e,t,a,i,void 0,s),i=js(),e!==null&&!et?(Ms(e,t,s),jn(e,t,s)):(ye&&i&&vs(t),t.flags|=1,it(e,t,a,s),t.child)}function vv(e,t,a,i,s,c){return Ua(t),t.updateQueue=null,a=hh(t,i,a,s),dh(e),i=js(),e!==null&&!et?(Ms(e,t,c),jn(e,t,c)):(ye&&i&&vs(t),t.flags|=1,it(e,t,a,c),t.child)}function mv(e,t,a,i,s){if(Ua(t),t.stateNode===null){var c=ml,h=a.contextType;typeof h=="object"&&h!==null&&(c=ft(h)),c=new a(i,c),t.memoizedState=c.state!==null&&c.state!==void 0?c.state:null,c.updater=Ks,t.stateNode=c,c._reactInternals=t,c=t.stateNode,c.props=i,c.state=t.memoizedState,c.refs={},Os(t),h=a.contextType,c.context=typeof h=="object"&&h!==null?ft(h):ml,c.state=t.memoizedState,h=a.getDerivedStateFromProps,typeof h=="function"&&(Xs(t,a,h,i),c.state=t.memoizedState),typeof a.getDerivedStateFromProps=="function"||typeof c.getSnapshotBeforeUpdate=="function"||typeof c.UNSAFE_componentWillMount!="function"&&typeof c.componentWillMount!="function"||(h=c.state,typeof c.componentWillMount=="function"&&c.componentWillMount(),typeof c.UNSAFE_componentWillMount=="function"&&c.UNSAFE_componentWillMount(),h!==c.state&&Ks.enqueueReplaceState(c,c.state,null),Oi(t,i,c,s),Ei(),c.state=t.memoizedState),typeof c.componentDidMount=="function"&&(t.flags|=4194308),i=!0}else if(e===null){c=t.stateNode;var g=t.memoizedProps,y=La(a,g);c.props=y;var T=c.context,j=a.contextType;h=ml,typeof j=="object"&&j!==null&&(h=ft(j));var L=a.getDerivedStateFromProps;j=typeof L=="function"||typeof c.getSnapshotBeforeUpdate=="function",g=t.pendingProps!==g,j||typeof c.UNSAFE_componentWillReceiveProps!="function"&&typeof c.componentWillReceiveProps!="function"||(g||T!==h)&&tv(t,c,i,h),In=!1;var z=t.memoizedState;c.state=z,Oi(t,i,c,s),Ei(),T=t.memoizedState,g||z!==T||In?(typeof L=="function"&&(Xs(t,a,L,i),T=t.memoizedState),(y=In||ev(t,a,y,i,z,T,h))?(j||typeof c.UNSAFE_componentWillMount!="function"&&typeof c.componentWillMount!="function"||(typeof c.componentWillMount=="function"&&c.componentWillMount(),typeof c.UNSAFE_componentWillMount=="function"&&c.UNSAFE_componentWillMount()),typeof c.componentDidMount=="function"&&(t.flags|=4194308)):(typeof c.componentDidMount=="function"&&(t.flags|=4194308),t.memoizedProps=i,t.memoizedState=T),c.props=i,c.state=T,c.context=h,i=y):(typeof c.componentDidMount=="function"&&(t.flags|=4194308),i=!1)}else{c=t.stateNode,As(e,t),h=t.memoizedProps,j=La(a,h),c.props=j,L=t.pendingProps,z=c.context,T=a.contextType,y=ml,typeof T=="object"&&T!==null&&(y=ft(T)),g=a.getDerivedStateFromProps,(T=typeof g=="function"||typeof c.getSnapshotBeforeUpdate=="function")||typeof c.UNSAFE_componentWillReceiveProps!="function"&&typeof c.componentWillReceiveProps!="function"||(h!==L||z!==y)&&tv(t,c,i,y),In=!1,z=t.memoizedState,c.state=z,Oi(t,i,c,s),Ei();var D=t.memoizedState;h!==L||z!==D||In||e!==null&&e.dependencies!==null&&qu(e.dependencies)?(typeof g=="function"&&(Xs(t,a,g,i),D=t.memoizedState),(j=In||ev(t,a,j,i,z,D,y)||e!==null&&e.dependencies!==null&&qu(e.dependencies))?(T||typeof c.UNSAFE_componentWillUpdate!="function"&&typeof c.componentWillUpdate!="function"||(typeof c.componentWillUpdate=="function"&&c.componentWillUpdate(i,D,y),typeof c.UNSAFE_componentWillUpdate=="function"&&c.UNSAFE_componentWillUpdate(i,D,y)),typeof c.componentDidUpdate=="function"&&(t.flags|=4),typeof c.getSnapshotBeforeUpdate=="function"&&(t.flags|=1024)):(typeof c.componentDidUpdate!="function"||h===e.memoizedProps&&z===e.memoizedState||(t.flags|=4),typeof c.getSnapshotBeforeUpdate!="function"||h===e.memoizedProps&&z===e.memoizedState||(t.flags|=1024),t.memoizedProps=i,t.memoizedState=D),c.props=i,c.state=D,c.context=y,i=j):(typeof c.componentDidUpdate!="function"||h===e.memoizedProps&&z===e.memoizedState||(t.flags|=4),typeof c.getSnapshotBeforeUpdate!="function"||h===e.memoizedProps&&z===e.memoizedState||(t.flags|=1024),i=!1)}return c=i,rr(e,t),i=(t.flags&128)!==0,c||i?(c=t.stateNode,a=i&&typeof a.getDerivedStateFromError!="function"?null:c.render(),t.flags|=1,e!==null&&i?(t.child=Al(t,e.child,null,s),t.child=Al(t,null,a,s)):it(e,t,a,s),t.memoizedState=c.state,e=t.child):e=jn(e,t,s),e}function pv(e,t,a,i){return mi(),t.flags|=256,it(e,t,a,i),t.child}var Ps={dehydrated:null,treeContext:null,retryLane:0,hydrationErrors:null};function Ws(e){return{baseLanes:e,cachePool:ah()}}function Fs(e,t,a){return e=e!==null?e.childLanes&~a:0,t&&(e|=Yt),e}function gv(e,t,a){var i=t.pendingProps,s=!1,c=(t.flags&128)!==0,h;if((h=c)||(h=e!==null&&e.memoizedState===null?!1:(We.current&2)!==0),h&&(s=!0,t.flags&=-129),h=(t.flags&32)!==0,t.flags&=-33,e===null){if(ye){if(s?aa(t):la(),ye){var g=He,y;if(y=g){e:{for(y=g,g=sn;y.nodeType!==8;){if(!g){g=null;break e}if(y=It(y.nextSibling),y===null){g=null;break e}}g=y}g!==null?(t.memoizedState={dehydrated:g,treeContext:Da!==null?{id:Tn,overflow:wn}:null,retryLane:536870912,hydrationErrors:null},y=At(18,null,null,0),y.stateNode=g,y.return=t,t.child=y,ht=t,He=null,y=!0):y=!1}y||Ma(t)}if(g=t.memoizedState,g!==null&&(g=g.dehydrated,g!==null))return Uc(g)?t.lanes=32:t.lanes=536870912,null;Nn(t)}return g=i.children,i=i.fallback,s?(la(),s=t.mode,g=or({mode:"hidden",children:g},s),i=Ra(i,s,a,null),g.return=t,i.return=t,g.sibling=i,t.child=g,s=t.child,s.memoizedState=Ws(a),s.childLanes=Fs(e,h,a),t.memoizedState=Ps,i):(aa(t),Is(t,g))}if(y=e.memoizedState,y!==null&&(g=y.dehydrated,g!==null)){if(c)t.flags&256?(aa(t),t.flags&=-257,t=ec(e,t,a)):t.memoizedState!==null?(la(),t.child=e.child,t.flags|=128,t=null):(la(),s=i.fallback,g=t.mode,i=or({mode:"visible",children:i.children},g),s=Ra(s,g,a,null),s.flags|=2,i.return=t,s.return=t,i.sibling=s,t.child=i,Al(t,e.child,null,a),i=t.child,i.memoizedState=Ws(a),i.childLanes=Fs(e,h,a),t.memoizedState=Ps,t=s);else if(aa(t),Uc(g)){if(h=g.nextSibling&&g.nextSibling.dataset,h)var T=h.dgst;h=T,i=Error(r(419)),i.stack="",i.digest=h,pi({value:i,source:null,stack:null}),t=ec(e,t,a)}else if(et||gi(e,t,a,!1),h=(a&e.childLanes)!==0,et||h){if(h=Re,h!==null&&(i=a&-a,i=(i&42)!==0?1:Zo(i),i=(i&(h.suspendedLanes|a))!==0?0:i,i!==0&&i!==y.retryLane))throw y.retryLane=i,vl(e,i),Dt(h,e,i),ov;g.data==="$?"||_c(),t=ec(e,t,a)}else g.data==="$?"?(t.flags|=192,t.child=e.child,t=null):(e=y.treeContext,He=It(g.nextSibling),ht=t,ye=!0,ja=null,sn=!1,e!==null&&(kt[qt++]=Tn,kt[qt++]=wn,kt[qt++]=Da,Tn=e.id,wn=e.overflow,Da=t),t=Is(t,i.children),t.flags|=4096);return t}return s?(la(),s=i.fallback,g=t.mode,y=e.child,T=y.sibling,i=An(y,{mode:"hidden",children:i.children}),i.subtreeFlags=y.subtreeFlags&65011712,T!==null?s=An(T,s):(s=Ra(s,g,a,null),s.flags|=2),s.return=t,i.return=t,i.sibling=s,t.child=i,i=s,s=t.child,g=e.child.memoizedState,g===null?g=Ws(a):(y=g.cachePool,y!==null?(T=Pe._currentValue,y=y.parent!==T?{parent:T,pool:T}:y):y=ah(),g={baseLanes:g.baseLanes|a,cachePool:y}),s.memoizedState=g,s.childLanes=Fs(e,h,a),t.memoizedState=Ps,i):(aa(t),a=e.child,e=a.sibling,a=An(a,{mode:"visible",children:i.children}),a.return=t,a.sibling=null,e!==null&&(h=t.deletions,h===null?(t.deletions=[e],t.flags|=16):h.push(e)),t.child=a,t.memoizedState=null,a)}function Is(e,t){return t=or({mode:"visible",children:t},e.mode),t.return=e,e.child=t}function or(e,t){return e=At(22,e,null,t),e.lanes=0,e.stateNode={_visibility:1,_pendingMarkers:null,_retryCache:null,_transitions:null},e}function ec(e,t,a){return Al(t,e.child,null,a),e=Is(t,t.pendingProps.children),e.flags|=2,t.memoizedState=null,e}function _v(e,t,a){e.lanes|=t;var i=e.alternate;i!==null&&(i.lanes|=t),_s(e.return,t,a)}function tc(e,t,a,i,s){var c=e.memoizedState;c===null?e.memoizedState={isBackwards:t,rendering:null,renderingStartTime:0,last:i,tail:a,tailMode:s}:(c.isBackwards=t,c.rendering=null,c.renderingStartTime=0,c.last=i,c.tail=a,c.tailMode=s)}function yv(e,t,a){var i=t.pendingProps,s=i.revealOrder,c=i.tail;if(it(e,t,i.children,a),i=We.current,(i&2)!==0)i=i&1|2,t.flags|=128;else{if(e!==null&&(e.flags&128)!==0)e:for(e=t.child;e!==null;){if(e.tag===13)e.memoizedState!==null&&_v(e,a,t);else if(e.tag===19)_v(e,a,t);else if(e.child!==null){e.child.return=e,e=e.child;continue}if(e===t)break e;for(;e.sibling===null;){if(e.return===null||e.return===t)break e;e=e.return}e.sibling.return=e.return,e=e.sibling}i&=1}switch(Y(We,i),s){case"forwards":for(a=t.child,s=null;a!==null;)e=a.alternate,e!==null&&lr(e)===null&&(s=a),a=a.sibling;a=s,a===null?(s=t.child,t.child=null):(s=a.sibling,a.sibling=null),tc(t,!1,s,a,c);break;case"backwards":for(a=null,s=t.child,t.child=null;s!==null;){if(e=s.alternate,e!==null&&lr(e)===null){t.child=s;break}e=s.sibling,s.sibling=a,a=s,s=e}tc(t,!0,a,null,c);break;case"together":tc(t,!1,null,null,void 0);break;default:t.memoizedState=null}return t.child}function jn(e,t,a){if(e!==null&&(t.dependencies=e.dependencies),sa|=t.lanes,(a&t.childLanes)===0)if(e!==null){if(gi(e,t,a,!1),(a&t.childLanes)===0)return null}else return null;if(e!==null&&t.child!==e.child)throw Error(r(153));if(t.child!==null){for(e=t.child,a=An(e,e.pendingProps),t.child=a,a.return=t;e.sibling!==null;)e=e.sibling,a=a.sibling=An(e,e.pendingProps),a.return=t;a.sibling=null}return t.child}function nc(e,t){return(e.lanes&t)!==0?!0:(e=e.dependencies,!!(e!==null&&qu(e)))}function c0(e,t,a){switch(t.tag){case 3:je(t,t.stateNode.containerInfo),Fn(t,Pe,e.memoizedState.cache),mi();break;case 27:case 5:No(t);break;case 4:je(t,t.stateNode.containerInfo);break;case 10:Fn(t,t.type,t.memoizedProps.value);break;case 13:var i=t.memoizedState;if(i!==null)return i.dehydrated!==null?(aa(t),t.flags|=128,null):(a&t.child.childLanes)!==0?gv(e,t,a):(aa(t),e=jn(e,t,a),e!==null?e.sibling:null);aa(t);break;case 19:var s=(e.flags&128)!==0;if(i=(a&t.childLanes)!==0,i||(gi(e,t,a,!1),i=(a&t.childLanes)!==0),s){if(i)return yv(e,t,a);t.flags|=128}if(s=t.memoizedState,s!==null&&(s.rendering=null,s.tail=null,s.lastEffect=null),Y(We,We.current),i)break;return null;case 22:case 23:return t.lanes=0,dv(e,t,a);case 24:Fn(t,Pe,e.memoizedState.cache)}return jn(e,t,a)}function bv(e,t,a){if(e!==null)if(e.memoizedProps!==t.pendingProps)et=!0;else{if(!nc(e,a)&&(t.flags&128)===0)return et=!1,c0(e,t,a);et=(e.flags&131072)!==0}else et=!1,ye&&(t.flags&1048576)!==0&&Pd(t,ku,t.index);switch(t.lanes=0,t.tag){case 16:e:{e=t.pendingProps;var i=t.elementType,s=i._init;if(i=s(i._payload),t.type=i,typeof i=="function")fs(i)?(e=La(i,e),t.tag=1,t=mv(null,t,i,e,a)):(t.tag=0,t=Js(null,t,i,e,a));else{if(i!=null){if(s=i.$$typeof,s===W){t.tag=11,t=sv(null,t,i,e,a);break e}else if(s===xe){t.tag=14,t=cv(null,t,i,e,a);break e}}throw t=Kn(i)||i,Error(r(306,t,""))}}return t;case 0:return Js(e,t,t.type,t.pendingProps,a);case 1:return i=t.type,s=La(i,t.pendingProps),mv(e,t,i,s,a);case 3:e:{if(je(t,t.stateNode.containerInfo),e===null)throw Error(r(387));i=t.pendingProps;var c=t.memoizedState;s=c.element,As(e,t),Oi(t,i,null,a);var h=t.memoizedState;if(i=h.cache,Fn(t,Pe,i),i!==c.cache&&ys(t,[Pe],a,!0),Ei(),i=h.element,c.isDehydrated)if(c={element:i,isDehydrated:!1,cache:h.cache},t.updateQueue.baseState=c,t.memoizedState=c,t.flags&256){t=pv(e,t,i,a);break e}else if(i!==s){s=$t(Error(r(424)),t),pi(s),t=pv(e,t,i,a);break e}else{switch(e=t.stateNode.containerInfo,e.nodeType){case 9:e=e.body;break;default:e=e.nodeName==="HTML"?e.ownerDocument.body:e}for(He=It(e.firstChild),ht=t,ye=!0,ja=null,sn=!0,a=Fh(t,null,i,a),t.child=a;a;)a.flags=a.flags&-3|4096,a=a.sibling}else{if(mi(),i===s){t=jn(e,t,a);break e}it(e,t,i,a)}t=t.child}return t;case 26:return rr(e,t),e===null?(a=Om(t.type,null,t.pendingProps,null))?t.memoizedState=a:ye||(a=t.type,e=t.pendingProps,i=xr(re.current).createElement(a),i[ct]=t,i[vt]=e,rt(i,a,e),Ie(i),t.stateNode=i):t.memoizedState=Om(t.type,e.memoizedProps,t.pendingProps,e.memoizedState),null;case 27:return No(t),e===null&&ye&&(i=t.stateNode=Sm(t.type,t.pendingProps,re.current),ht=t,sn=!0,s=He,ha(t.type)?(Zc=s,He=It(i.firstChild)):He=s),it(e,t,t.pendingProps.children,a),rr(e,t),e===null&&(t.flags|=4194304),t.child;case 5:return e===null&&ye&&((s=i=He)&&(i=L0(i,t.type,t.pendingProps,sn),i!==null?(t.stateNode=i,ht=t,He=It(i.firstChild),sn=!1,s=!0):s=!1),s||Ma(t)),No(t),s=t.type,c=t.pendingProps,h=e!==null?e.memoizedProps:null,i=c.children,jc(s,c)?i=null:h!==null&&jc(s,h)&&(t.flags|=32),t.memoizedState!==null&&(s=Ns(e,t,n0,null,null,a),Xi._currentValue=s),rr(e,t),it(e,t,i,a),t.child;case 6:return e===null&&ye&&((e=a=He)&&(a=$0(a,t.pendingProps,sn),a!==null?(t.stateNode=a,ht=t,He=null,e=!0):e=!1),e||Ma(t)),null;case 13:return gv(e,t,a);case 4:return je(t,t.stateNode.containerInfo),i=t.pendingProps,e===null?t.child=Al(t,null,i,a):it(e,t,i,a),t.child;case 11:return sv(e,t,t.type,t.pendingProps,a);case 7:return it(e,t,t.pendingProps,a),t.child;case 8:return it(e,t,t.pendingProps.children,a),t.child;case 12:return it(e,t,t.pendingProps.children,a),t.child;case 10:return i=t.pendingProps,Fn(t,t.type,i.value),it(e,t,i.children,a),t.child;case 9:return s=t.type._context,i=t.pendingProps.children,Ua(t),s=ft(s),i=i(s),t.flags|=1,it(e,t,i,a),t.child;case 14:return cv(e,t,t.type,t.pendingProps,a);case 15:return fv(e,t,t.type,t.pendingProps,a);case 19:return yv(e,t,a);case 31:return i=t.pendingProps,a=t.mode,i={mode:i.mode,children:i.children},e===null?(a=or(i,a),a.ref=t.ref,t.child=a,a.return=t,t=a):(a=An(e.child,i),a.ref=t.ref,t.child=a,a.return=t,t=a),t;case 22:return dv(e,t,a);case 24:return Ua(t),i=ft(Pe),e===null?(s=xs(),s===null&&(s=Re,c=bs(),s.pooledCache=c,c.refCount++,c!==null&&(s.pooledCacheLanes|=a),s=c),t.memoizedState={parent:i,cache:s},Os(t),Fn(t,Pe,s)):((e.lanes&a)!==0&&(As(e,t),Oi(t,null,null,a),Ei()),s=e.memoizedState,c=t.memoizedState,s.parent!==i?(s={parent:i,cache:i},t.memoizedState=s,t.lanes===0&&(t.memoizedState=t.updateQueue.baseState=s),Fn(t,Pe,i)):(i=c.cache,Fn(t,Pe,i),i!==s.cache&&ys(t,[Pe],a,!0))),it(e,t,t.pendingProps.children,a),t.child;case 29:throw t.pendingProps}throw Error(r(156,t.tag))}function Mn(e){e.flags|=4}function Sv(e,t){if(t.type!=="stylesheet"||(t.state.loading&4)!==0)e.flags&=-16777217;else if(e.flags|=16777216,!Rm(t)){if(t=Vt.current,t!==null&&((ve&4194048)===ve?cn!==null:(ve&62914560)!==ve&&(ve&536870912)===0||t!==cn))throw Si=Es,lh;e.flags|=8192}}function sr(e,t){t!==null&&(e.flags|=4),e.flags&16384&&(t=e.tag!==22?If():536870912,e.lanes|=t,Rl|=t)}function Ni(e,t){if(!ye)switch(e.tailMode){case"hidden":t=e.tail;for(var a=null;t!==null;)t.alternate!==null&&(a=t),t=t.sibling;a===null?e.tail=null:a.sibling=null;break;case"collapsed":a=e.tail;for(var i=null;a!==null;)a.alternate!==null&&(i=a),a=a.sibling;i===null?t||e.tail===null?e.tail=null:e.tail.sibling=null:i.sibling=null}}function Be(e){var t=e.alternate!==null&&e.alternate.child===e.child,a=0,i=0;if(t)for(var s=e.child;s!==null;)a|=s.lanes|s.childLanes,i|=s.subtreeFlags&65011712,i|=s.flags&65011712,s.return=e,s=s.sibling;else for(s=e.child;s!==null;)a|=s.lanes|s.childLanes,i|=s.subtreeFlags,i|=s.flags,s.return=e,s=s.sibling;return e.subtreeFlags|=i,e.childLanes=a,t}function f0(e,t,a){var i=t.pendingProps;switch(ms(t),t.tag){case 31:case 16:case 15:case 0:case 11:case 7:case 8:case 12:case 9:case 14:return Be(t),null;case 1:return Be(t),null;case 3:return a=t.stateNode,i=null,e!==null&&(i=e.memoizedState.cache),t.memoizedState.cache!==i&&(t.flags|=2048),Rn(Pe),Qn(),a.pendingContext&&(a.context=a.pendingContext,a.pendingContext=null),(e===null||e.child===null)&&(vi(t)?Mn(t):e===null||e.memoizedState.isDehydrated&&(t.flags&256)===0||(t.flags|=1024,Id())),Be(t),null;case 26:return a=t.memoizedState,e===null?(Mn(t),a!==null?(Be(t),Sv(t,a)):(Be(t),t.flags&=-16777217)):a?a!==e.memoizedState?(Mn(t),Be(t),Sv(t,a)):(Be(t),t.flags&=-16777217):(e.memoizedProps!==i&&Mn(t),Be(t),t.flags&=-16777217),null;case 27:bu(t),a=re.current;var s=t.type;if(e!==null&&t.stateNode!=null)e.memoizedProps!==i&&Mn(t);else{if(!i){if(t.stateNode===null)throw Error(r(166));return Be(t),null}e=ee.current,vi(t)?Wd(t):(e=Sm(s,i,a),t.stateNode=e,Mn(t))}return Be(t),null;case 5:if(bu(t),a=t.type,e!==null&&t.stateNode!=null)e.memoizedProps!==i&&Mn(t);else{if(!i){if(t.stateNode===null)throw Error(r(166));return Be(t),null}if(e=ee.current,vi(t))Wd(t);else{switch(s=xr(re.current),e){case 1:e=s.createElementNS("http://www.w3.org/2000/svg",a);break;case 2:e=s.createElementNS("http://www.w3.org/1998/Math/MathML",a);break;default:switch(a){case"svg":e=s.createElementNS("http://www.w3.org/2000/svg",a);break;case"math":e=s.createElementNS("http://www.w3.org/1998/Math/MathML",a);break;case"script":e=s.createElement("div"),e.innerHTML=" - + +
- \ No newline at end of file + diff --git a/vite-app/docs/runtime-configuration.md b/vite-app/docs/runtime-configuration.md new file mode 100644 index 00000000..72f5d0ba --- /dev/null +++ b/vite-app/docs/runtime-configuration.md @@ -0,0 +1,73 @@ +# Vite App with Runtime Configuration + +This Vite app is designed to work with the Python evaluation protocol server and automatically discovers its configuration at runtime. + +## Runtime Configuration + +The app automatically discovers the server configuration in the following order: + +1. **Server-Injected Configuration** (Recommended): The Python server injects configuration directly into the HTML +2. **Location-Based Discovery**: Falls back to discovering configuration from the current URL +3. **Default Values**: Uses localhost:8000 as a last resort + +## How It Works + +### Server-Side Injection +The Python server (`logs_server.py`) automatically injects configuration into the HTML response: + +```html + +``` + +### Frontend Discovery +The frontend automatically reads this configuration and uses it for WebSocket connections: + +```typescript +// First, check if server injected configuration is available +if (window.SERVER_CONFIG) { + const serverConfig = window.SERVER_CONFIG; + config.websocket.host = serverConfig.host; + config.websocket.port = serverConfig.port; + // ... etc +} +``` + +## Usage + +### Starting the Server +```bash +# Default port 8000 +python -m eval_protocol.utils.logs_server + +# Custom port +python -m eval_protocol.utils.logs_server --port 9000 + +# Custom host and port +python -m eval_protocol.utils.logs_server --host 0.0.0.0 --port 9000 + +# Custom build directory +python -m eval_protocol.utils.logs_server --build-dir /path/to/dist +``` + +### Building the Frontend +```bash +cd vite-app +pnpm install +pnpm build +``` + +The built files will be in the `dist/` directory and automatically served by the Python server. + +## Benefits + +- **No hard-coded ports**: The frontend automatically adapts to whatever port the server runs on +- **Flexible deployment**: Can run on any port without rebuilding the frontend +- **Automatic discovery**: Works whether served from the same origin or different origins +- **Fallback support**: Gracefully handles cases where server injection isn't available diff --git a/vite-app/src/App.tsx b/vite-app/src/App.tsx index a5e86e63..9eeaae0f 100644 --- a/vite-app/src/App.tsx +++ b/vite-app/src/App.tsx @@ -8,6 +8,7 @@ import { EvaluationRowSchema, type EvaluationRow } from "./types/eval-protocol"; import { WebSocketServerMessageSchema } from "./types/websocket"; import { GlobalState } from "./GlobalState"; import logoLight from "./assets/logo-light.png"; +import { getWebSocketUrl, discoverServerConfig } from "./config"; export const state = new GlobalState(); @@ -26,7 +27,7 @@ const App = observer(() => { return; // Already connected } - const ws = new WebSocket("ws://localhost:8000/ws"); + const ws = new WebSocket(getWebSocketUrl()); wsRef.current = ws; ws.onopen = () => { @@ -109,7 +110,13 @@ const App = observer(() => { }; useEffect(() => { - connectWebSocket(); + // Discover server configuration first, then connect + const initializeApp = async () => { + await discoverServerConfig(); + connectWebSocket(); + }; + + initializeApp(); return () => { if (reconnectTimeoutRef.current) { diff --git a/vite-app/src/config.ts b/vite-app/src/config.ts new file mode 100644 index 00000000..96626f06 --- /dev/null +++ b/vite-app/src/config.ts @@ -0,0 +1,63 @@ +// Configuration for the application +export const config = { + // WebSocket connection settings + websocket: { + host: 'localhost', // Will be discovered at runtime + port: '8000', // Will be discovered at runtime + protocol: 'ws', + }, + // API settings + api: { + host: 'localhost', // Will be discovered at runtime + port: '8000', // Will be discovered at runtime + protocol: 'http', + }, +}; + +// Helper function to build WebSocket URL +export const getWebSocketUrl = (): string => { + const { protocol, host, port } = config.websocket; + return `${protocol}://${host}:${port}/ws`; +}; + +// Helper function to build API URL +export const getApiUrl = (): string => { + const { protocol, host, port } = config.api; + return `${protocol}://${host}:${port}`; +}; + +// Runtime configuration discovery +export const discoverServerConfig = async (): Promise => { + try { + // First, check if server injected configuration is available + if (window.SERVER_CONFIG) { + const serverConfig = window.SERVER_CONFIG; + config.websocket.host = serverConfig.host; + config.websocket.port = serverConfig.port; + config.websocket.protocol = serverConfig.protocol; + config.api.host = serverConfig.host; + config.api.port = serverConfig.port; + config.api.protocol = serverConfig.apiProtocol; + console.log('Using server-injected config:', config); + return; + } + + // Fallback: Try to discover server configuration from the current location + const currentHost = window.location.hostname; + const currentPort = window.location.port; + const currentProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + + // Update config with discovered values + config.websocket.host = currentHost; + config.websocket.port = currentPort || (currentProtocol === 'wss:' ? '443' : '80'); + config.websocket.protocol = currentProtocol; + + config.api.host = currentHost; + config.api.port = currentPort || (currentProtocol === 'wss:' ? '443' : '80'); + config.api.protocol = window.location.protocol === 'https:' ? 'https:' : 'http:'; + + console.log('Using discovered config from location:', config); + } catch (error) { + console.warn('Failed to discover server config, using defaults:', error); + } +}; diff --git a/vite-app/src/types/global.d.ts b/vite-app/src/types/global.d.ts new file mode 100644 index 00000000..a8ccce4a --- /dev/null +++ b/vite-app/src/types/global.d.ts @@ -0,0 +1,13 @@ +// Global type declarations +declare global { + interface Window { + SERVER_CONFIG?: { + host: string; + port: string; + protocol: string; + apiProtocol: string; + }; + } +} + +export {}; From 3c36fabc21e258d0439e8c8ec74c6f5afc2cd3e8 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 11 Aug 2025 10:35:13 -0700 Subject: [PATCH 02/43] Fix WebSocketManager to reset broadcast task after cancellation --- eval_protocol/utils/logs_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eval_protocol/utils/logs_server.py b/eval_protocol/utils/logs_server.py index 815d33ef..e2a655cd 100644 --- a/eval_protocol/utils/logs_server.py +++ b/eval_protocol/utils/logs_server.py @@ -109,6 +109,7 @@ def stop_broadcast_loop(self): """Stop the broadcast loop.""" if self._broadcast_task and not self._broadcast_task.done(): self._broadcast_task.cancel() + self._broadcast_task = None class EvaluationWatcher: From 2e0be2401066424036a247ba776743d1c9b76830 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 11 Aug 2025 10:35:28 -0700 Subject: [PATCH 03/43] simple tests work --- tests/test_logs_server_simple.py | 88 ++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/test_logs_server_simple.py diff --git a/tests/test_logs_server_simple.py b/tests/test_logs_server_simple.py new file mode 100644 index 00000000..98bc3e47 --- /dev/null +++ b/tests/test_logs_server_simple.py @@ -0,0 +1,88 @@ +import json +import tempfile +from pathlib import Path +from unittest.mock import AsyncMock, Mock, patch + +import pytest + +from eval_protocol.utils.logs_server import EvaluationWatcher, WebSocketManager + + +class TestWebSocketManagerBasic: + """Basic tests for WebSocketManager without starting real loops.""" + + def test_initialization(self): + """Test WebSocketManager initialization.""" + manager = WebSocketManager() + assert len(manager.active_connections) == 0 + assert manager._broadcast_queue is not None + assert manager._broadcast_task is None + + @pytest.mark.asyncio + async def test_connect_disconnect(self): + """Test WebSocket connection and disconnection.""" + manager = WebSocketManager() + mock_websocket = AsyncMock() + + # Test connection + await manager.connect(mock_websocket) + assert len(manager.active_connections) == 1 + assert mock_websocket in manager.active_connections + mock_websocket.accept.assert_called_once() + + # Test disconnection + manager.disconnect(mock_websocket) + assert len(manager.active_connections) == 0 + assert mock_websocket not in manager.active_connections + + def test_broadcast_row_upserted(self): + """Test broadcasting row upsert events.""" + manager = WebSocketManager() + + # Create a simple mock row + mock_row = Mock() + mock_row.model_dump.return_value = {"id": "test-123", "content": "test"} + + # Test that broadcast doesn't fail when no connections + manager.broadcast_row_upserted(mock_row) + + # Test that message is queued + assert not manager._broadcast_queue.empty() + queued_message = manager._broadcast_queue.get_nowait() + assert "type" in queued_message + assert "row" in queued_message + json_message = json.loads(queued_message) + assert json_message["row"]["id"] == "test-123" + assert json_message["row"]["content"] == "test" + + +class TestEvaluationWatcherBasic: + """Basic tests for EvaluationWatcher without starting real threads.""" + + def test_initialization(self): + """Test EvaluationWatcher initialization.""" + mock_manager = Mock() + watcher = EvaluationWatcher(mock_manager) + assert watcher.websocket_manager == mock_manager + assert watcher._thread is None + assert watcher._stop_event is not None + + def test_start_stop(self): + """Test starting and stopping the watcher.""" + mock_manager = Mock() + watcher = EvaluationWatcher(mock_manager) + + # Test start + watcher.start() + assert watcher._thread is not None + assert watcher._thread.is_alive() + + # Test stop + watcher.stop() + assert watcher._stop_event.is_set() + if watcher._thread: + watcher._thread.join(timeout=1.0) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 6cdadf3ce8206628b14377535caff9903fd3d1ec Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 11 Aug 2025 10:49:23 -0700 Subject: [PATCH 04/43] TODO: TestLogsServer --- eval_protocol/utils/logs_server.py | 26 +- tests/test_logs_server.py | 496 +++++++++++++++++++++++++++++ 2 files changed, 516 insertions(+), 6 deletions(-) create mode 100644 tests/test_logs_server.py diff --git a/eval_protocol/utils/logs_server.py b/eval_protocol/utils/logs_server.py index e2a655cd..e2a675c7 100644 --- a/eval_protocol/utils/logs_server.py +++ b/eval_protocol/utils/logs_server.py @@ -87,18 +87,32 @@ async def _send_text_to_all_connections(self, text: str): return tasks = [] + failed_connections = [] + for connection in connections: try: tasks.append(connection.send_text(text)) except Exception as e: logger.error(f"Failed to send text to WebSocket: {e}") - with self._lock: - try: - self.active_connections.remove(connection) - except ValueError: - pass + failed_connections.append(connection) + + # Execute all sends in parallel if tasks: - await asyncio.gather(*tasks, return_exceptions=True) + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Check for any exceptions that occurred during execution + for i, result in enumerate(results): + if isinstance(result, Exception): + logger.error(f"Failed to send text to WebSocket: {result}") + failed_connections.append(connections[i]) + + # Remove all failed connections + with self._lock: + for connection in failed_connections: + try: + self.active_connections.remove(connection) + except ValueError: + pass def start_broadcast_loop(self): """Start the broadcast loop in the current event loop.""" diff --git a/tests/test_logs_server.py b/tests/test_logs_server.py new file mode 100644 index 00000000..10e5dd6c --- /dev/null +++ b/tests/test_logs_server.py @@ -0,0 +1,496 @@ +import asyncio +import json +import tempfile +import threading +import time +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +import psutil +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from eval_protocol.dataset_logger import default_logger +from eval_protocol.dataset_logger.dataset_logger import LOG_EVENT_TYPE +from eval_protocol.event_bus import event_bus +from eval_protocol.models import EvalMetadata, EvaluationRow, InputMetadata, Message +from eval_protocol.utils.logs_server import ( + EvaluationWatcher, + LogsServer, + WebSocketManager, + create_app, + serve_logs, +) + + +class TestWebSocketManager: + """Test WebSocketManager class.""" + + def test_initialization(self): + """Test WebSocketManager initialization.""" + manager = WebSocketManager() + assert len(manager.active_connections) == 0 + assert manager._broadcast_queue is not None + assert manager._broadcast_task is None + + @pytest.mark.asyncio + async def test_connect_disconnect(self): + """Test WebSocket connection and disconnection.""" + manager = WebSocketManager() + mock_websocket = AsyncMock() + + # Test connection + with patch.object(default_logger, "read", return_value=[]): + await manager.connect(mock_websocket) + assert len(manager.active_connections) == 1 + assert mock_websocket in manager.active_connections + mock_websocket.accept.assert_called_once() + + # Test disconnection + manager.disconnect(mock_websocket) + assert len(manager.active_connections) == 0 + assert mock_websocket not in manager.active_connections + + @pytest.mark.asyncio + async def test_connect_sends_initial_logs(self): + """Test that connecting sends initial logs.""" + manager = WebSocketManager() + mock_websocket = AsyncMock() + + # Mock default_logger.read() + mock_logs = [ + EvaluationRow( + messages=[Message(role="user", content="test")], + input_metadata=InputMetadata(row_id="test-123"), + ) + ] + + with patch.object(default_logger, "read", return_value=mock_logs): + await manager.connect(mock_websocket) + + # Verify that initial logs were sent + mock_websocket.send_text.assert_called_once() + sent_data = json.loads(mock_websocket.send_text.call_args[0][0]) + assert sent_data["type"] == "initialize_logs" + assert len(sent_data["logs"]) == 1 + + def test_broadcast_row_upserted(self): + """Test broadcasting row upsert events.""" + manager = WebSocketManager() + test_row = EvaluationRow( + messages=[Message(role="user", content="test")], + input_metadata=InputMetadata(row_id="test-123"), + ) + + # Test that broadcast doesn't fail when no connections + manager.broadcast_row_upserted(test_row) + + # Test that message is queued + assert not manager._broadcast_queue.empty() + queued_message = manager._broadcast_queue.get_nowait() + data = json.loads(queued_message) + assert data["type"] == "log" + assert "row" in data + assert data["row"]["messages"][0]["content"] == "test" + assert data["row"]["input_metadata"]["row_id"] == "test-123" + + @pytest.mark.asyncio + async def test_broadcast_loop(self): + """Test the broadcast loop functionality.""" + manager = WebSocketManager() + mock_websocket = AsyncMock() + await manager.connect(mock_websocket) + + # Test that broadcast loop can be started and stopped + manager.start_broadcast_loop() + assert manager._broadcast_task is not None + + # Stop broadcast loop + manager.stop_broadcast_loop() + assert manager._broadcast_task is None + + @pytest.mark.asyncio + async def test_send_text_to_all_connections(self): + """Test sending text to all connections.""" + manager = WebSocketManager() + mock_websocket1 = AsyncMock() + mock_websocket2 = AsyncMock() + + # Mock default_logger.read() to return empty logs + with patch.object(default_logger, "read", return_value=[]): + await manager.connect(mock_websocket1) + await manager.connect(mock_websocket2) + + test_message = "test message" + await manager._send_text_to_all_connections(test_message) + + # Check that the test message was sent to both websockets + mock_websocket1.send_text.assert_any_call(test_message) + mock_websocket2.send_text.assert_any_call(test_message) + + @pytest.mark.asyncio + async def test_send_text_handles_failed_connections(self): + """Test that failed connections are handled gracefully.""" + manager = WebSocketManager() + mock_websocket1 = AsyncMock() + mock_websocket2 = AsyncMock() + + # Mock default_logger.read() to return empty logs + with patch.object(default_logger, "read", return_value=[]): + await manager.connect(mock_websocket1) + await manager.connect(mock_websocket2) + + # Make the second websocket fail AFTER connection is established + # We need to make send_text raise an exception when awaited + async def failing_send_text(text): + raise Exception("Connection failed") + + mock_websocket2.send_text = failing_send_text + + test_message = "test message" + await manager._send_text_to_all_connections(test_message) + + # First websocket should receive the message + mock_websocket1.send_text.assert_any_call(test_message) + # Second websocket should have been removed due to failure + assert len(manager.active_connections) == 1 + assert mock_websocket1 in manager.active_connections + + +class TestEvaluationWatcher: + """Test EvaluationWatcher class.""" + + def test_initialization(self): + """Test EvaluationWatcher initialization.""" + mock_manager = Mock() + watcher = EvaluationWatcher(mock_manager) + assert watcher.websocket_manager == mock_manager + assert watcher._thread is None + assert watcher._stop_event is not None + + def test_start_stop(self): + """Test starting and stopping the watcher.""" + mock_manager = Mock() + watcher = EvaluationWatcher(mock_manager) + + # Test start + watcher.start() + assert watcher._thread is not None + assert watcher._thread.is_alive() + + # Test stop + watcher.stop() + assert watcher._stop_event.is_set() + if watcher._thread: + watcher._thread.join(timeout=1.0) + + @patch("psutil.Process") + def test_should_update_status_running_process(self, mock_process): + """Test status update for running process.""" + mock_manager = Mock() + watcher = EvaluationWatcher(mock_manager) + + # Mock a running process + mock_process_instance = Mock() + mock_process_instance.is_running.return_value = True + mock_process.return_value = mock_process_instance + + test_row = EvaluationRow( + messages=[Message(role="user", content="test")], + input_metadata=InputMetadata(row_id="test-123"), + eval_metadata=EvalMetadata(name="test_eval", num_runs=1, aggregation_method="mean", status="running"), + pid=12345, + ) + + # Process is running, should not update + assert watcher._should_update_status(test_row) is False + + @patch("psutil.Process") + def test_should_update_status_stopped_process(self, mock_process): + """Test status update for stopped process.""" + mock_manager = Mock() + watcher = EvaluationWatcher(mock_manager) + + # Mock a stopped process + mock_process_instance = Mock() + mock_process_instance.is_running.return_value = False + mock_process.return_value = mock_process_instance + + test_row = EvaluationRow( + messages=[Message(role="user", content="test")], + input_metadata=InputMetadata(row_id="test-123"), + eval_metadata=EvalMetadata(name="test_eval", num_runs=1, aggregation_method="mean", status="running"), + pid=12345, + ) + + # Process is stopped, should update + assert watcher._should_update_status(test_row) is True + + @patch("psutil.Process") + def test_should_update_status_no_such_process(self, mock_process): + """Test status update for non-existent process.""" + mock_manager = Mock() + watcher = EvaluationWatcher(mock_manager) + + # Mock process not found + mock_process.side_effect = psutil.NoSuchProcess(pid=999) + + test_row = EvaluationRow( + messages=[Message(role="user", content="test")], + input_metadata=InputMetadata(row_id="test-123"), + eval_metadata=EvalMetadata(name="test_eval", num_runs=1, aggregation_method="mean", status="running"), + pid=999, + ) + + # Process doesn't exist, should update + assert watcher._should_update_status(test_row) is True + + def test_should_update_status_not_running(self): + """Test status update for non-running evaluation.""" + mock_manager = Mock() + watcher = EvaluationWatcher(mock_manager) + + test_row = EvaluationRow( + messages=[Message(role="user", content="test")], + input_metadata=InputMetadata(row_id="test-123"), + eval_metadata=EvalMetadata(name="test_eval", num_runs=1, aggregation_method="mean", status="finished"), + pid=12345, + ) + + # Not running status, should not update + assert watcher._should_update_status(test_row) is False + + def test_should_update_status_no_pid(self): + """Test status update for evaluation without PID.""" + mock_manager = Mock() + watcher = EvaluationWatcher(mock_manager) + + test_row = EvaluationRow( + messages=[Message(role="user", content="test")], + input_metadata=InputMetadata(row_id="test-123"), + eval_metadata=EvalMetadata(name="test_eval", num_runs=1, aggregation_method="mean", status="running"), + pid=None, + ) + + # No PID, should not update + assert watcher._should_update_status(test_row) is False + + +class TestLogsServer: + """Test LogsServer class.""" + + @pytest.fixture + def temp_build_dir(self): + """Create a temporary build directory for testing.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + # Create a minimal index.html file + (temp_path / "index.html").write_text("Test") + # Create assets directory (required by ViteServer) + (temp_path / "assets").mkdir(exist_ok=True) + yield temp_path + + def test_initialization(self, temp_build_dir): + """Test LogsServer initialization.""" + server = LogsServer(build_dir=str(temp_build_dir)) + assert server.build_dir == str(temp_build_dir) + assert server.websocket_manager is not None + assert server.evaluation_watcher is not None + + def test_initialization_invalid_build_dir(self): + """Test LogsServer initialization with invalid build directory.""" + with pytest.raises(ValueError, match="Build directory does not exist"): + LogsServer(build_dir="/nonexistent/path") + + def test_websocket_routes(self, temp_build_dir): + """Test that WebSocket routes are properly set up.""" + server = LogsServer(build_dir=str(temp_build_dir)) + + # Check that the WebSocket endpoint exists + websocket_routes = [route for route in server.app.routes if hasattr(route, "endpoint")] + assert len(websocket_routes) > 0 + + @pytest.mark.asyncio + async def test_handle_event(self, temp_build_dir): + """Test event handling.""" + server = LogsServer(build_dir=str(temp_build_dir)) + + # Test handling a log event + test_row = EvaluationRow( + messages=[Message(role="user", content="test")], + input_metadata=InputMetadata(row_id="test-123"), + ) + + server._handle_event(LOG_EVENT_TYPE, test_row) + # The event should be queued for broadcasting + assert not server.websocket_manager._broadcast_queue.empty() + + def test_create_app_factory(self, temp_build_dir): + """Test the create_app factory function.""" + app = create_app(build_dir=str(temp_build_dir)) + assert isinstance(app, FastAPI) + + def test_serve_logs_convenience_function(self, temp_build_dir): + """Test the serve_logs convenience function.""" + # This should not raise an error + serve_logs(port=8001) + + @pytest.mark.asyncio + async def test_run_async_lifecycle(self, temp_build_dir): + """Test the async lifecycle of the server.""" + server = LogsServer(build_dir=str(temp_build_dir)) + + # Mock the uvicorn.Server to avoid actually starting a server + with patch("uvicorn.Server") as mock_uvicorn_server: + mock_server = Mock() + mock_server.serve = AsyncMock() + mock_uvicorn_server.return_value = mock_server + + # Start the server + start_task = asyncio.create_task(server.run_async()) + + # Wait a bit for it to start + await asyncio.sleep(0.1) + + # Cancel the task instead of calling non-existent stop method + start_task.cancel() + + # Wait for the task to complete + try: + await start_task + except asyncio.CancelledError: + pass + + +class TestLogsServerIntegration: + """Integration tests for LogsServer.""" + + @pytest.fixture + def temp_build_dir_with_files(self): + """Create a temporary build directory with various test files.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create index.html + (temp_path / "index.html").write_text("Test") + + # Create some static assets + (temp_path / "static").mkdir() + (temp_path / "static" / "app.js").write_text("console.log('test');") + (temp_path / "static" / "style.css").write_text("body { color: black; }") + + # Create a nested directory structure + (temp_path / "nested").mkdir() + (temp_path / "nested" / "file.txt").write_text("nested content") + + yield temp_path + + def test_static_file_serving(self, temp_build_dir_with_files): + """Test that static files are served correctly.""" + server = LogsServer(build_dir=str(temp_build_dir_with_files)) + client = TestClient(server.app) + + # Test serving index.html + response = client.get("/") + assert response.status_code == 200 + assert "Test" in response.text + + # Test serving static files + response = client.get("/static/app.js") + assert response.status_code == 200 + assert "console.log('test')" in response.text + + response = client.get("/static/style.css") + assert response.status_code == 200 + assert "color: black" in response.text + + def test_spa_routing(self, temp_build_dir_with_files): + """Test SPA routing fallback.""" + server = LogsServer(build_dir=str(temp_build_dir_with_files)) + client = TestClient(server.app) + + # Test that non-existent routes fall back to index.html + response = client.get("/some/nonexistent/route") + assert response.status_code == 200 + assert "Test" in response.text + + def test_root_endpoint(self, temp_build_dir_with_files): + """Test the root endpoint.""" + server = LogsServer(build_dir=str(temp_build_dir_with_files)) + client = TestClient(server.app) + + response = client.get("/") + assert response.status_code == 200 + assert "Test" in response.text + + def test_health_endpoint(self, temp_build_dir_with_files): + """Test the health endpoint.""" + server = LogsServer(build_dir=str(temp_build_dir_with_files)) + client = TestClient(server.app) + + response = client.get("/health") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + + +@pytest.mark.asyncio +class TestAsyncWebSocketOperations: + """Test async WebSocket operations.""" + + async def test_websocket_connection_lifecycle(self): + """Test complete WebSocket connection lifecycle.""" + manager = WebSocketManager() + + # Create mock WebSocket + mock_websocket = AsyncMock() + + # Test connection + with patch.object(default_logger, "read", return_value=[]): + await manager.connect(mock_websocket) + assert len(manager.active_connections) == 1 + + # Test broadcasting without starting the loop + test_row = EvaluationRow( + messages=[Message(role="user", content="test")], + input_metadata=InputMetadata(row_id="test-123"), + ) + manager.broadcast_row_upserted(test_row) + + # Verify message was queued + assert not manager._broadcast_queue.empty() + + # Test disconnection + manager.disconnect(mock_websocket) + assert len(manager.active_connections) == 0 + + async def test_multiple_websocket_connections(self): + """Test handling multiple WebSocket connections.""" + manager = WebSocketManager() + + # Create multiple mock WebSockets + mock_websocket1 = AsyncMock() + mock_websocket2 = AsyncMock() + mock_websocket3 = AsyncMock() + + # Connect all + with patch.object(default_logger, "read", return_value=[]): + await manager.connect(mock_websocket1) + await manager.connect(mock_websocket2) + await manager.connect(mock_websocket3) + assert len(manager.active_connections) == 3 + + # Test broadcasting to all without starting the loop + test_row = EvaluationRow( + messages=[Message(role="user", content="test")], + input_metadata=InputMetadata(row_id="test-123"), + ) + manager.broadcast_row_upserted(test_row) + + # Verify message was queued + assert not manager._broadcast_queue.empty() + + # Disconnect one + manager.disconnect(mock_websocket2) + assert len(manager.active_connections) == 2 From 88f0f3a6507658d29bcc99ab39433fa604c49b8e Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 11 Aug 2025 10:56:20 -0700 Subject: [PATCH 05/43] TODO: TestLogsServerIntegration --- tests/test_logs_server.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tests/test_logs_server.py b/tests/test_logs_server.py index 10e5dd6c..5516f070 100644 --- a/tests/test_logs_server.py +++ b/tests/test_logs_server.py @@ -9,6 +9,7 @@ import psutil import pytest from fastapi import FastAPI +from fastapi.routing import APIWebSocketRoute from fastapi.testclient import TestClient from eval_protocol.dataset_logger import default_logger @@ -291,16 +292,16 @@ def temp_build_dir(self): (temp_path / "assets").mkdir(exist_ok=True) yield temp_path - def test_initialization(self, temp_build_dir): + def test_initialization(self, temp_build_dir: Path): """Test LogsServer initialization.""" server = LogsServer(build_dir=str(temp_build_dir)) - assert server.build_dir == str(temp_build_dir) + assert server.build_dir == temp_build_dir assert server.websocket_manager is not None assert server.evaluation_watcher is not None def test_initialization_invalid_build_dir(self): """Test LogsServer initialization with invalid build directory.""" - with pytest.raises(ValueError, match="Build directory does not exist"): + with pytest.raises(FileNotFoundError, match="Build directory '/nonexistent/path' does not exist"): LogsServer(build_dir="/nonexistent/path") def test_websocket_routes(self, temp_build_dir): @@ -308,8 +309,13 @@ def test_websocket_routes(self, temp_build_dir): server = LogsServer(build_dir=str(temp_build_dir)) # Check that the WebSocket endpoint exists - websocket_routes = [route for route in server.app.routes if hasattr(route, "endpoint")] - assert len(websocket_routes) > 0 + if not server.app.routes: + raise ValueError("No routes found") + for route in server.app.routes: + if isinstance(route, APIWebSocketRoute) and route.path == "/ws": + break + else: + raise ValueError("WebSocket route not found") @pytest.mark.asyncio async def test_handle_event(self, temp_build_dir): @@ -317,10 +323,10 @@ async def test_handle_event(self, temp_build_dir): server = LogsServer(build_dir=str(temp_build_dir)) # Test handling a log event - test_row = EvaluationRow( - messages=[Message(role="user", content="test")], - input_metadata=InputMetadata(row_id="test-123"), - ) + test_row = { + "messages": [{"role": "user", "content": "test"}], + "input_metadata": {"row_id": "test-123"}, + } server._handle_event(LOG_EVENT_TYPE, test_row) # The event should be queued for broadcasting @@ -333,8 +339,12 @@ def test_create_app_factory(self, temp_build_dir): def test_serve_logs_convenience_function(self, temp_build_dir): """Test the serve_logs convenience function.""" - # This should not raise an error - serve_logs(port=8001) + # Mock the LogsServer.run method to avoid actually starting a server + with patch("eval_protocol.utils.logs_server.LogsServer.run") as mock_run: + # This should not raise an error + serve_logs(port=8001) + # Verify that the run method was called + mock_run.assert_called_once() @pytest.mark.asyncio async def test_run_async_lifecycle(self, temp_build_dir): From 1b421799e6fed80095268217c583cd6499842088 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 11 Aug 2025 11:06:04 -0700 Subject: [PATCH 06/43] TODO: test HTML injection - also test TestAsyncWebSocketOperations --- eval_protocol/utils/vite_server.py | 24 ++++++++++++------------ tests/test_logs_server.py | 23 ++++++++++++----------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/eval_protocol/utils/vite_server.py b/eval_protocol/utils/vite_server.py index 4c3143e2..02eef31d 100644 --- a/eval_protocol/utils/vite_server.py +++ b/eval_protocol/utils/vite_server.py @@ -97,7 +97,17 @@ def _setup_routes(self): # Mount static files self.app.mount("/assets", StaticFiles(directory=self.build_dir / "assets"), name="assets") - # Serve other static files from build directory + @self.app.get("/") + async def root(): + """Serve the main index.html file with injected configuration.""" + return self._serve_index_with_config() + + @self.app.get("/health") + async def health(): + """Health check endpoint.""" + return {"status": "ok", "build_dir": str(self.build_dir)} + + # Serve other static files from build directory - this must be last @self.app.get("/{path:path}") async def serve_spa(path: str): """ @@ -114,22 +124,12 @@ async def serve_spa(path: str): # For SPA routing, serve index.html for non-existent routes # but exclude API routes and asset requests - if not path.startswith(("api/", "assets/")): + if not path.startswith(("api/", "assets/", "health")): return self._serve_index_with_config() # If we get here, the file doesn't exist and it's not a SPA route raise HTTPException(status_code=404, detail="File not found") - @self.app.get("/") - async def root(): - """Serve the main index.html file with injected configuration.""" - return self._serve_index_with_config() - - @self.app.get("/health") - async def health(): - """Health check endpoint.""" - return {"status": "ok", "build_dir": str(self.build_dir)} - def run(self): """ Run the Vite server. diff --git a/tests/test_logs_server.py b/tests/test_logs_server.py index 5516f070..48db912b 100644 --- a/tests/test_logs_server.py +++ b/tests/test_logs_server.py @@ -378,21 +378,22 @@ class TestLogsServerIntegration: @pytest.fixture def temp_build_dir_with_files(self): - """Create a temporary build directory with various test files.""" + """Create a temporary build directory with index.html and assets/ directory.""" with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) # Create index.html (temp_path / "index.html").write_text("Test") - # Create some static assets - (temp_path / "static").mkdir() - (temp_path / "static" / "app.js").write_text("console.log('test');") - (temp_path / "static" / "style.css").write_text("body { color: black; }") + # Create assets directory and some files inside it + assets_dir = temp_path / "assets" + assets_dir.mkdir() + (assets_dir / "app.js").write_text("console.log('test');") + (assets_dir / "style.css").write_text("body { color: black; }") - # Create a nested directory structure - (temp_path / "nested").mkdir() - (temp_path / "nested" / "file.txt").write_text("nested content") + # Optionally, create a nested directory inside assets + (assets_dir / "nested").mkdir() + (assets_dir / "nested" / "file.txt").write_text("nested content") yield temp_path @@ -407,11 +408,11 @@ def test_static_file_serving(self, temp_build_dir_with_files): assert "Test" in response.text # Test serving static files - response = client.get("/static/app.js") + response = client.get("/assets/app.js") assert response.status_code == 200 assert "console.log('test')" in response.text - response = client.get("/static/style.css") + response = client.get("/assets/style.css") assert response.status_code == 200 assert "color: black" in response.text @@ -442,7 +443,7 @@ def test_health_endpoint(self, temp_build_dir_with_files): response = client.get("/health") assert response.status_code == 200 data = response.json() - assert data["status"] == "healthy" + assert data["status"] == "ok" @pytest.mark.asyncio From b10d403dcb2a60765778270db54173b0d6a36b45 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 11 Aug 2025 11:15:51 -0700 Subject: [PATCH 07/43] add logs server tests --- tests/test_vite_server.py | 224 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 tests/test_vite_server.py diff --git a/tests/test_vite_server.py b/tests/test_vite_server.py new file mode 100644 index 00000000..eb4a59df --- /dev/null +++ b/tests/test_vite_server.py @@ -0,0 +1,224 @@ +import tempfile +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from eval_protocol.utils.vite_server import ViteServer + + +class TestViteServer: + """Test ViteServer class.""" + + @pytest.fixture + def temp_build_dir_with_files(self): + """Create a temporary build directory with index.html and assets/ directory.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create index.html + (temp_path / "index.html").write_text("Test") + + # Create assets directory and some files inside it + assets_dir = temp_path / "assets" + assets_dir.mkdir() + (assets_dir / "app.js").write_text("console.log('test');") + (assets_dir / "style.css").write_text("body { color: black; }") + + # Optionally, create a nested directory inside assets + (assets_dir / "nested").mkdir() + (assets_dir / "nested" / "file.txt").write_text("nested content") + + yield temp_path + + def test_initialization(self, temp_build_dir_with_files): + """Test ViteServer initialization.""" + vite_server = ViteServer(build_dir=str(temp_build_dir_with_files), host="localhost", port=8000) + + assert vite_server.build_dir == temp_build_dir_with_files + assert vite_server.host == "localhost" + assert vite_server.port == 8000 + assert vite_server.index_file == "index.html" + assert vite_server.app is not None + + def test_initialization_invalid_build_dir(self): + """Test ViteServer initialization with invalid build directory.""" + with pytest.raises(FileNotFoundError): + ViteServer(build_dir="nonexistent_dir") + + def test_initialization_invalid_index_file(self, temp_build_dir_with_files): + """Test ViteServer initialization with invalid index file.""" + # Remove the index.html file + (temp_build_dir_with_files / "index.html").unlink() + + with pytest.raises(FileNotFoundError): + ViteServer(build_dir=str(temp_build_dir_with_files)) + + def test_html_injection_in_vite_server(self, temp_build_dir_with_files): + """Test that ViteServer injects server configuration into HTML.""" + # Create a more complex HTML file for testing injection + index_html = """ + + + + + Test App + + + +
Test Application
+ + +""" + + # Write the test HTML + (temp_build_dir_with_files / "index.html").write_text(index_html) + + # Create ViteServer instance + vite_server = ViteServer(build_dir=str(temp_build_dir_with_files), host="localhost", port=8000) + + # Test the HTML injection method directly + injected_html = vite_server._inject_config_into_html(index_html) + + # Verify server configuration is injected + assert "window.SERVER_CONFIG" in injected_html + assert 'host: "localhost"' in injected_html + assert 'port: "8000"' in injected_html + assert 'protocol: "ws"' in injected_html + assert 'apiProtocol: "http"' in injected_html + + # Verify injection happens before closing tag + head_end_index = injected_html.find("") + config_script_index = injected_html.find("window.SERVER_CONFIG") + assert config_script_index < head_end_index + + # Verify the original HTML structure is preserved + assert '
Test Application
' in injected_html + assert '' in injected_html + + # Test that the injected config is valid JavaScript + assert "window.SERVER_CONFIG = {" in injected_html + assert "};" in injected_html + + def test_html_injection_without_head_tag(self, temp_build_dir_with_files): + """Test HTML injection when no tag is present.""" + # Create HTML without tag + simple_html = """ + + +

Simple App

+

No head tag

+ +""" + + (temp_build_dir_with_files / "index.html").write_text(simple_html) + + vite_server = ViteServer(build_dir=str(temp_build_dir_with_files), host="127.0.0.1", port=9000) + + injected_html = vite_server._inject_config_into_html(simple_html) + + # Verify config is injected at the beginnin + assert injected_html.strip().startswith(" +
- + \ No newline at end of file From adae8f62463d937dca631376580fd24066d59983 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 11 Aug 2025 15:34:32 -0700 Subject: [PATCH 18/43] Update model path in pytest configuration to use gpt-oss-120b for improved performance --- tests/pytest/test_pytest_mcp_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/test_pytest_mcp_config.py b/tests/pytest/test_pytest_mcp_config.py index 475d74e2..d9b46c58 100644 --- a/tests/pytest/test_pytest_mcp_config.py +++ b/tests/pytest/test_pytest_mcp_config.py @@ -20,7 +20,7 @@ ] ], rollout_processor=default_agent_rollout_processor, - model=["fireworks_ai/accounts/fireworks/models/kimi-k2-instruct"], + model=["fireworks_ai/accounts/fireworks/models/gpt-oss-120b"], mode="pointwise", mcp_config_path="tests/pytest/mcp_configurations/mock_discord_mcp_config.json", ) From f15542c5f861067c320e05851cef89be1a53adcb Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 11 Aug 2025 15:45:24 -0700 Subject: [PATCH 19/43] move ids under execution_metadata --- .../sqlite_evaluation_row_store.py | 6 +-- eval_protocol/models.py | 49 +++++++++++-------- eval_protocol/pytest/evaluation_test.py | 8 +-- .../test_sqlite_dataset_logger_adapter.py | 4 +- tests/pytest/test_pytest_ids.py | 12 ++--- vite-app/src/GlobalState.tsx | 4 +- vite-app/src/components/EvaluationRow.tsx | 12 ++--- vite-app/src/components/EvaluationTable.tsx | 2 +- vite-app/src/types/eval-protocol.ts | 12 +++-- vite-app/src/util/pivot.test.ts | 2 +- 10 files changed, 62 insertions(+), 49 deletions(-) diff --git a/eval_protocol/dataset_logger/sqlite_evaluation_row_store.py b/eval_protocol/dataset_logger/sqlite_evaluation_row_store.py index 1a6618e5..a8f149a8 100644 --- a/eval_protocol/dataset_logger/sqlite_evaluation_row_store.py +++ b/eval_protocol/dataset_logger/sqlite_evaluation_row_store.py @@ -37,9 +37,9 @@ def db_path(self) -> str: return self._db_path def upsert_row(self, data: dict) -> None: - rollout_id = data["rollout_id"] - if "rollout_id" not in data: - raise ValueError("rollout_id is required to upsert a row") + rollout_id = data["execution_metadata"]["rollout_id"] + if rollout_id is None: + raise ValueError("execution_metadata.rollout_id is required to upsert a row") if self._EvaluationRow.select().where(self._EvaluationRow.rollout_id == rollout_id).exists(): self._EvaluationRow.update(data=data).where(self._EvaluationRow.rollout_id == rollout_id).execute() else: diff --git a/eval_protocol/models.py b/eval_protocol/models.py index b3d48b6c..79c4490d 100644 --- a/eval_protocol/models.py +++ b/eval_protocol/models.py @@ -237,6 +237,30 @@ class EvalMetadata(BaseModel): passed: Optional[bool] = Field(None, description="Whether the evaluation passed based on the threshold") +class ExecutionMetadata(BaseModel): + """Metadata about the execution of the evaluation.""" + + invocation_id: Optional[str] = Field( + default_factory=generate_id, + description="The ID of the invocation that this row belongs to.", + ) + + experiment_id: Optional[str] = Field( + default_factory=generate_id, + description="The ID of the experiment that this row belongs to.", + ) + + rollout_id: Optional[str] = Field( + default_factory=generate_id, + description="The ID of the rollout that this row belongs to.", + ) + + run_id: Optional[str] = Field( + None, + description=("The ID of the run that this row belongs to."), + ) + + class RolloutStatus(BaseModel): """Status of the rollout.""" @@ -281,26 +305,6 @@ class EvaluationRow(BaseModel): description="The status of the rollout.", ) - invocation_id: Optional[str] = Field( - default_factory=generate_id, - description="The ID of the invocation that this row belongs to.", - ) - - experiment_id: Optional[str] = Field( - default_factory=generate_id, - description="The ID of the experiment that this row belongs to.", - ) - - rollout_id: Optional[str] = Field( - default_factory=generate_id, - description="The ID of the rollout that this row belongs to.", - ) - - run_id: Optional[str] = Field( - None, - description=("The ID of the run that this row belongs to."), - ) - # Ground truth reference (moved from EvaluateResult to top level) ground_truth: Optional[str] = Field( default=None, description="Optional ground truth reference for this evaluation." @@ -311,6 +315,11 @@ class EvaluationRow(BaseModel): default=None, description="The evaluation result for this row/trajectory." ) + execution_metadata: ExecutionMetadata = Field( + default_factory=ExecutionMetadata, + description="Metadata about the execution of the evaluation.", + ) + # LLM usage statistics usage: Optional[CompletionUsage] = Field( default=None, description="Token usage statistics from LLM calls during execution." diff --git a/eval_protocol/pytest/evaluation_test.py b/eval_protocol/pytest/evaluation_test.py index f36fee9b..ef516f6b 100644 --- a/eval_protocol/pytest/evaluation_test.py +++ b/eval_protocol/pytest/evaluation_test.py @@ -383,8 +383,8 @@ def _log_eval_error( row.input_metadata.session_data["mode"] = mode # Initialize eval_metadata for each row row.eval_metadata = eval_metadata - row.experiment_id = experiment_id - row.invocation_id = invocation_id + row.execution_metadata.experiment_id = experiment_id + row.execution_metadata.invocation_id = invocation_id # has to be done in the pytest main process since it's # used to determine whether this eval has stopped @@ -409,11 +409,11 @@ def _log_eval_error( # apply new run_id to fresh_dataset for row in fresh_dataset: - row.run_id = run_id + row.execution_metadata.run_id = run_id # generate new rollout_id for each row for row in fresh_dataset: - row.rollout_id = generate_id() + row.execution_metadata.rollout_id = generate_id() # log the fresh_dataset for row in fresh_dataset: diff --git a/tests/dataset_logger/test_sqlite_dataset_logger_adapter.py b/tests/dataset_logger/test_sqlite_dataset_logger_adapter.py index 897691b2..7689d085 100644 --- a/tests/dataset_logger/test_sqlite_dataset_logger_adapter.py +++ b/tests/dataset_logger/test_sqlite_dataset_logger_adapter.py @@ -24,7 +24,7 @@ def test_update_log_and_read(): logger = SqliteDatasetLoggerAdapter(store=store) logger.log(row) - saved = logger.read(row.rollout_id)[0] + saved = logger.read(row.execution_metadata.rollout_id)[0] assert row.messages == saved.messages assert row.input_metadata == saved.input_metadata @@ -42,7 +42,7 @@ def test_create_log_and_read(): row = EvaluationRow(input_metadata=input_metadata, messages=messages) logger.log(row) - saved = logger.read(rollout_id=row.rollout_id)[0] + saved = logger.read(rollout_id=row.execution_metadata.rollout_id)[0] assert row.messages == saved.messages assert row.input_metadata == saved.input_metadata diff --git a/tests/pytest/test_pytest_ids.py b/tests/pytest/test_pytest_ids.py index 4deda27c..0131bcbe 100644 --- a/tests/pytest/test_pytest_ids.py +++ b/tests/pytest/test_pytest_ids.py @@ -12,8 +12,8 @@ def __init__(self): self._rows: dict[str, EvaluationRow] = {} def log(self, row: EvaluationRow): - print(row.run_id, row.rollout_id) - self._rows[row.rollout_id] = row + print(row.execution_metadata.run_id, row.execution_metadata.rollout_id) + self._rows[row.execution_metadata.rollout_id] = row def read(self): return list(self._rows.values()) @@ -76,10 +76,10 @@ def test_evaluation_test_decorator_ids_single(monkeypatch): logger=InMemoryLogger(), ) def eval_fn(row: EvaluationRow) -> EvaluationRow: - unique_run_ids.add(row.run_id) - unique_experiment_ids.add(row.experiment_id) - unique_rollout_ids.add(row.rollout_id) - unique_invocation_ids.add(row.invocation_id) + unique_run_ids.add(row.execution_metadata.run_id) + unique_experiment_ids.add(row.execution_metadata.experiment_id) + unique_rollout_ids.add(row.execution_metadata.rollout_id) + unique_invocation_ids.add(row.execution_metadata.invocation_id) unique_row_ids.add(row.input_metadata.row_id) return row diff --git a/vite-app/src/GlobalState.tsx b/vite-app/src/GlobalState.tsx index 9669fe4c..67fa7fc0 100644 --- a/vite-app/src/GlobalState.tsx +++ b/vite-app/src/GlobalState.tsx @@ -14,10 +14,10 @@ export class GlobalState { upsertRows(dataset: EvaluationRow[]) { dataset.forEach((row) => { - if (!row.rollout_id) { + if (!row.execution_metadata?.rollout_id) { return; } - this.dataset[row.rollout_id] = row; + this.dataset[row.execution_metadata.rollout_id] = row; }); } diff --git a/vite-app/src/components/EvaluationRow.tsx b/vite-app/src/components/EvaluationRow.tsx index d16b06d2..fdeaf03c 100644 --- a/vite-app/src/components/EvaluationRow.tsx +++ b/vite-app/src/components/EvaluationRow.tsx @@ -133,10 +133,10 @@ const IdSection = observer(({ data }: { data: EvaluationRowType }) => ( )); @@ -197,7 +197,7 @@ const ExpandedContent = observer( export const EvaluationRow = observer( ({ row }: { row: EvaluationRowType; index: number }) => { - const rolloutId = row.rollout_id; + const rolloutId = row.execution_metadata?.rollout_id; const isExpanded = state.isRowExpanded(rolloutId); const toggleExpanded = () => state.toggleRowExpansion(rolloutId); @@ -226,7 +226,7 @@ export const EvaluationRow = observer( {/* Rollout ID */} - + {/* Model */} diff --git a/vite-app/src/components/EvaluationTable.tsx b/vite-app/src/components/EvaluationTable.tsx index f45f16c5..fb470b9e 100644 --- a/vite-app/src/components/EvaluationTable.tsx +++ b/vite-app/src/components/EvaluationTable.tsx @@ -20,7 +20,7 @@ const TableBody = observer( {paginatedData.map((row, index) => ( diff --git a/vite-app/src/types/eval-protocol.ts b/vite-app/src/types/eval-protocol.ts index 3fb00d5c..b18697f1 100644 --- a/vite-app/src/types/eval-protocol.ts +++ b/vite-app/src/types/eval-protocol.ts @@ -94,15 +94,19 @@ export const RolloutStatusSchema = z.object({ error_message: z.string().optional().describe('Error message if the rollout failed.') }); +export const ExecutionMetadataSchema = z.object({ + invocation_id: z.string().optional().describe('The ID of the invocation that this row belongs to.'), + experiment_id: z.string().optional().describe('The ID of the experiment that this row belongs to.'), + rollout_id: z.string().optional().describe('The ID of the rollout that this row belongs to.'), + run_id: z.string().optional().describe('The ID of the run that this row belongs to.'), +}); + export const EvaluationRowSchema = z.object({ messages: z.array(MessageSchema).describe('List of messages in the conversation/trajectory.'), tools: z.array(z.record(z.string(), z.any())).optional().describe('Available tools/functions that were provided to the agent.'), input_metadata: InputMetadataSchema.describe('Metadata related to the input (dataset info, model config, session data, etc.).'), rollout_status: RolloutStatusSchema.default({ status: 'finished' }).describe('The status of the rollout.'), - invocation_id: z.string().optional().describe('The ID of the invocation that this row belongs to.'), - experiment_id: z.string().optional().describe('The ID of the experiment that this row belongs to.'), - rollout_id: z.string().optional().describe('The ID of the rollout that this row belongs to.'), - run_id: z.string().optional().describe('The ID of the run that this row belongs to.'), + execution_metadata: ExecutionMetadataSchema.optional().describe('Metadata about the execution of the evaluation.'), ground_truth: z.string().optional().describe('Optional ground truth reference for this evaluation.'), evaluation_result: EvaluateResultSchema.optional().describe('The evaluation result for this row/trajectory.'), usage: CompletionUsageSchema.optional().describe('Token usage statistics from LLM calls during execution.'), diff --git a/vite-app/src/util/pivot.test.ts b/vite-app/src/util/pivot.test.ts index ec26063e..4875e7d3 100644 --- a/vite-app/src/util/pivot.test.ts +++ b/vite-app/src/util/pivot.test.ts @@ -189,7 +189,7 @@ describe('computePivot', () => { const res = computePivot({ data: rows, - rowFields: ['$.eval_metadata.name', '$.experiment_id'], + rowFields: ['$.eval_metadata.name', '$.execution_metadata.experiment_id'], columnFields: ['$.input_metadata.completion_params.model'], valueField: '$.evaluation_result.score', aggregator: 'avg', From 082ff797b8c8cf62a55eb4efb9b40ebbecac28ed Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 11 Aug 2025 21:50:08 -0700 Subject: [PATCH 20/43] move pivot tab into its own component --- vite-app/src/components/Dashboard.tsx | 26 ++------------------- vite-app/src/components/PivotTab.tsx | 33 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 24 deletions(-) create mode 100644 vite-app/src/components/PivotTab.tsx diff --git a/vite-app/src/components/Dashboard.tsx b/vite-app/src/components/Dashboard.tsx index 59868622..fa3ca49d 100644 --- a/vite-app/src/components/Dashboard.tsx +++ b/vite-app/src/components/Dashboard.tsx @@ -4,7 +4,7 @@ import { useLocation, useNavigate } from "react-router-dom"; import { state } from "../App"; import Button from "./Button"; import { EvaluationTable } from "./EvaluationTable"; -import PivotTable from "./PivotTable"; +import PivotTab from "./PivotTab"; import TabButton from "./TabButton"; import flattenJson from "../util/flatten-json"; @@ -132,29 +132,7 @@ const Dashboard = observer(({ onRefresh }: DashboardProps) => { {activeTab === "table" ? ( ) : ( -
-
- Showing pivot of flattened rows (JSONPath keys). Defaults: - rows by eval name and status; columns by model; values average - score. -
- -
+ )} diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx new file mode 100644 index 00000000..3da09640 --- /dev/null +++ b/vite-app/src/components/PivotTab.tsx @@ -0,0 +1,33 @@ +import { observer } from "mobx-react"; +import PivotTable from "./PivotTable"; + +interface PivotTabProps { + data: any[]; +} + +const PivotTab = observer(({ data }: PivotTabProps) => { + return ( +
+
+ Showing pivot of flattened rows (JSONPath keys). Defaults: rows by eval + name and status; columns by model; values average score. +
+ +
+ ); +}); + +export default PivotTab; From fc6c320947dd386c64402efc4056e56b3d0e7390 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:12:48 -0700 Subject: [PATCH 21/43] put flattened dataset into globalstate --- vite-app/src/GlobalState.tsx | 9 +++++++++ vite-app/src/components/Dashboard.tsx | 10 ++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/vite-app/src/GlobalState.tsx b/vite-app/src/GlobalState.tsx index 67fa7fc0..ceca4d1c 100644 --- a/vite-app/src/GlobalState.tsx +++ b/vite-app/src/GlobalState.tsx @@ -1,5 +1,6 @@ import { makeAutoObservable } from "mobx"; import type { EvaluationRow } from "./types/eval-protocol"; +import flattenJson from "./util/flatten-json"; export class GlobalState { isConnected: boolean = false; @@ -53,6 +54,14 @@ export class GlobalState { ); } + get flattenedDataset() { + return this.sortedDataset.map((row) => flattenJson(row)); + } + + get flattenedDatasetKeys() { + return this.flattenedDataset.map((row) => Object.keys(row)); + } + get totalCount() { return Object.keys(this.dataset).length; } diff --git a/vite-app/src/components/Dashboard.tsx b/vite-app/src/components/Dashboard.tsx index fa3ca49d..75ec4e2b 100644 --- a/vite-app/src/components/Dashboard.tsx +++ b/vite-app/src/components/Dashboard.tsx @@ -1,12 +1,11 @@ import { observer } from "mobx-react"; -import { useMemo, useState, useEffect } from "react"; +import { useState, useEffect } from "react"; import { useLocation, useNavigate } from "react-router-dom"; import { state } from "../App"; import Button from "./Button"; import { EvaluationTable } from "./EvaluationTable"; import PivotTab from "./PivotTab"; import TabButton from "./TabButton"; -import flattenJson from "../util/flatten-json"; interface DashboardProps { onRefresh: () => void; @@ -68,11 +67,6 @@ const Dashboard = observer(({ onRefresh }: DashboardProps) => { setActiveTab(deriveTabFromPath(location.pathname)); }, [location.pathname]); - const flattened = useMemo(() => { - const flattenedDataset = state.sortedDataset.map((row) => flattenJson(row)); - return flattenedDataset; - }, [state.sortedDataset]); - return (
{/* Summary */} @@ -132,7 +126,7 @@ const Dashboard = observer(({ onRefresh }: DashboardProps) => { {activeTab === "table" ? ( ) : ( - + )}
From 641a7044db6d9b714c5c88cb5c86142e3269618a Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:23:08 -0700 Subject: [PATCH 22/43] parameterize pivottab --- vite-app/src/components/Dashboard.tsx | 6 +- vite-app/src/components/PivotTab.tsx | 188 ++++++++++++++++++++++++-- 2 files changed, 174 insertions(+), 20 deletions(-) diff --git a/vite-app/src/components/Dashboard.tsx b/vite-app/src/components/Dashboard.tsx index 75ec4e2b..5911279c 100644 --- a/vite-app/src/components/Dashboard.tsx +++ b/vite-app/src/components/Dashboard.tsx @@ -123,11 +123,7 @@ const Dashboard = observer(({ onRefresh }: DashboardProps) => { {/* Tab content */}
- {activeTab === "table" ? ( - - ) : ( - - )} + {activeTab === "table" ? : }
)} diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx index 3da09640..f47c1097 100644 --- a/vite-app/src/components/PivotTab.tsx +++ b/vite-app/src/components/PivotTab.tsx @@ -1,27 +1,185 @@ import { observer } from "mobx-react"; import PivotTable from "./PivotTable"; +import Select from "./Select"; +import { state } from "../App"; +import { useState } from "react"; -interface PivotTabProps { - data: any[]; +interface FieldSelectorProps { + title: string; + fields: string[]; + onFieldChange: (index: number, value: string) => void; + onAddField: () => void; + onRemoveField: (index: number) => void; + availableKeys: string[]; } -const PivotTab = observer(({ data }: PivotTabProps) => { +const FieldSelector = ({ + title, + fields, + onFieldChange, + onAddField, + onRemoveField, + availableKeys, +}: FieldSelectorProps) => ( +
+
{title}:
+
+ {fields.map((field, index) => ( +
+ + {fields.length > 0 && ( + + )} +
+ ))} + {fields.length < 3 && ( + + )} +
+
+); + +const SingleFieldSelector = ({ + title, + field, + onFieldChange, + availableKeys, +}: { + title: string; + field: string; + onFieldChange: (value: string) => void; + availableKeys: string[]; +}) => ( +
+
{title}:
+ +
+); + +const PivotTab = observer(() => { + const [selectedRowFields, setSelectedRowFields] = useState([ + "$.eval_metadata.name", + ]); + const [selectedColumnFields, setSelectedColumnFields] = useState([ + "$.input_metadata.completion_params.model", + ]); + const [selectedValueField, setSelectedValueField] = useState( + "$.evaluation_result.score" + ); + + const createFieldHandler = ( + setter: React.Dispatch> + ) => { + return (index: number, value: string) => { + setter((prev) => { + const newFields = [...prev]; + newFields[index] = value; + return newFields; + }); + }; + }; + + const createAddHandler = ( + setter: React.Dispatch> + ) => { + return () => { + setter((prev) => (prev.length < 3 ? [...prev, ""] : prev)); + }; + }; + + const createRemoveHandler = ( + setter: React.Dispatch> + ) => { + return (index: number) => { + setter((prev) => prev.filter((_, i) => i !== index)); + }; + }; + + const availableKeys = state.flattenedDatasetKeys[0] || []; + return (
-
- Showing pivot of flattened rows (JSONPath keys). Defaults: rows by eval - name and status; columns by model; values average score. +
+ Configure your pivot table by selecting fields for rows, columns, and + values. Use the dropdowns below to choose from available flattened + JSONPath keys. You can add/remove fields and change the value field to + pivot on different metrics.
+ + + + + + + field !== "" + ) as (keyof (typeof state.flattenedDataset)[number])[] + } + columnFields={ + selectedColumnFields.filter( + (field) => field !== "" + ) as (keyof (typeof state.flattenedDataset)[number])[] + } + valueField={ + selectedValueField as keyof (typeof state.flattenedDataset)[number] + } aggregator="avg" showRowTotals showColumnTotals From 137eedfb79cba6ecdd8319b9403d95e46ddb534b Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:29:42 -0700 Subject: [PATCH 23/43] styling + add parameterization for aggregate method --- vite-app/src/components/PivotTab.tsx | 121 +++++++++++++++++-------- vite-app/src/components/PivotTable.tsx | 99 +++++++++++++++----- vite-app/src/util/pivot.ts | 36 +++++++- 3 files changed, 189 insertions(+), 67 deletions(-) diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx index f47c1097..6e4f790c 100644 --- a/vite-app/src/components/PivotTab.tsx +++ b/vite-app/src/components/PivotTab.tsx @@ -20,46 +20,55 @@ const FieldSelector = ({ onAddField, onRemoveField, availableKeys, -}: FieldSelectorProps) => ( -
-
{title}:
-
- {fields.map((field, index) => ( -
- - {fields.length > 0 && ( - + )} +
+ ))} + {fields.length < 3 && ( + + )} +
-
-); + ); +}; const SingleFieldSelector = ({ title, @@ -90,6 +99,32 @@ const SingleFieldSelector = ({
); +const AggregatorSelector = ({ + aggregator, + onAggregatorChange, +}: { + aggregator: string; + onAggregatorChange: (value: string) => void; +}) => ( +
+
+ Aggregation Method: +
+ +
+); + const PivotTab = observer(() => { const [selectedRowFields, setSelectedRowFields] = useState([ "$.eval_metadata.name", @@ -100,6 +135,7 @@ const PivotTab = observer(() => { const [selectedValueField, setSelectedValueField] = useState( "$.evaluation_result.score" ); + const [selectedAggregator, setSelectedAggregator] = useState("avg"); const createFieldHandler = ( setter: React.Dispatch> @@ -147,6 +183,7 @@ const PivotTab = observer(() => { onAddField={createAddHandler(setSelectedRowFields)} onRemoveField={createRemoveHandler(setSelectedRowFields)} availableKeys={availableKeys} + variant="row" /> { onAddField={createAddHandler(setSelectedColumnFields)} onRemoveField={createRemoveHandler(setSelectedColumnFields)} availableKeys={availableKeys} + variant="column" /> { availableKeys={availableKeys} /> + + { valueField={ selectedValueField as keyof (typeof state.flattenedDataset)[number] } - aggregator="avg" + aggregator={selectedAggregator as any} showRowTotals showColumnTotals /> diff --git a/vite-app/src/components/PivotTable.tsx b/vite-app/src/components/PivotTable.tsx index a5b2e9e9..11ef5516 100644 --- a/vite-app/src/components/PivotTable.tsx +++ b/vite-app/src/components/PivotTable.tsx @@ -104,28 +104,48 @@ export function PivotTable>({ - {/* Row header labels */} + {/* Row header labels with enhanced styling */} {rowFields.map((f) => ( - {String(f)} + +
+ {String(f)} +
+ ))} - {/* Column headers (flattened) */} + {/* Column headers with enhanced styling */} {colKeyTuples.map((tuple, idx) => ( - - {tuple.map((v) => String(v ?? "")).join(" / ")} + +
+ {tuple.map((v) => String(v ?? "")).join(" / ")} +
))} - {showRowTotals && Total} + {showRowTotals && ( + +
Total
+ + )} {rowKeyTuples.map((rTuple, rIdx) => { const rKey = toKey(rTuple); return ( - - {/* Row header cells */} + + {/* Row header cells with enhanced styling */} {rTuple.map((value, i) => ( - - {String(value ?? "")} + +
+ {String(value ?? "")} +
))} {/* Data cells */} @@ -134,39 +154,70 @@ export function PivotTable>({ const cell = cells[rKey]?.[cKey]; const content = cell ? formatter(cell.value) : emptyValue; return ( - - {content} + +
{content}
 ); })} - {/* Row total */} + {/* Row total with enhanced styling */} {showRowTotals && ( - - {formatter(rowTotals[rKey] ?? 0)} + +
+ {formatter(rowTotals[rKey] ?? 0)} +
)} ); })} {showColumnTotals && ( - + {/* Total label spanning row header columns */} - - Total + +
Total
 - {/* Column totals */} + {/* Column totals with enhanced styling */} {colKeyTuples.map((cTuple, cIdx) => { const cKey = toKey(cTuple); return ( - - {formatter(colTotals[cKey] ?? 0)} + +
+ {formatter(colTotals[cKey] ?? 0)} +
); })} - {/* Grand total */} + {/* Grand total with enhanced styling */} {showRowTotals && ( - - {formatter(grandTotal)} + +
+ {formatter(grandTotal)} +
)} diff --git a/vite-app/src/util/pivot.ts b/vite-app/src/util/pivot.ts index 8ac87c64..eb9a3ff9 100644 --- a/vite-app/src/util/pivot.ts +++ b/vite-app/src/util/pivot.ts @@ -4,6 +4,8 @@ * - "count": number of records in the cell (ignores valueField) * - "sum": sum of numeric values extracted from valueField * - "avg": average of numeric values extracted from valueField + * - "min": minimum of numeric values extracted from valueField + * - "max": maximum of numeric values extracted from valueField * - custom: user function receiving the numeric values (from valueField if provided) * and the raw records in the cell. Return the aggregated number to display. */ @@ -11,6 +13,8 @@ export type Aggregator = | "count" | "sum" | "avg" + | "min" + | "max" | ((values: number[], records: T[]) => number); export interface PivotComputationResult { @@ -50,6 +54,8 @@ function aggregate>( return values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length; + if (agg === "min") return values.length === 0 ? 0 : Math.min(...values); + if (agg === "max") return values.length === 0 ? 0 : Math.max(...values); // default: count return records.length; } @@ -134,7 +140,29 @@ export interface ComputePivotParams> { * }) * ``` * - * 4) Multiple column fields (composite columns) + * 4) Minimum amounts per region Ɨ product + * ```ts + * const res = computePivot({ + * data: rows, + * rowFields: ['region'], + * columnFields: ['product'], + * valueField: 'amount', + * aggregator: 'min', + * }) + * ``` + * + * 5) Maximum amounts per region Ɨ product + * ```ts + * const res = computePivot({ + * data: rows, + * rowFields: ['region'], + * columnFields: ['product'], + * valueField: 'amount', + * aggregator: 'max', + * }) + * ``` + * + * 6) Multiple column fields (composite columns) * ```ts * const res = computePivot({ * data: rows, @@ -146,7 +174,7 @@ export interface ComputePivotParams> { * // Each column is the tuple [product, quarter] * ``` * - * 5) Custom aggregator (e.g., max) + * 7) Custom aggregator (e.g., max) * ```ts * const res = computePivot({ * data: rows, @@ -157,7 +185,7 @@ export interface ComputePivotParams> { * }) * ``` * - * 6) Single grand total (no rows/cols) + * 8) Single grand total (no rows/cols) * ```ts * const res = computePivot({ * data: rows, @@ -169,7 +197,7 @@ export interface ComputePivotParams> { * // res.grandTotal is the total sum * ``` * - * 7) Excel-style: multiple value fields alongside multiple column fields (recipe) + * 9) Excel-style: multiple value fields alongside multiple column fields (recipe) * - Run computePivot once per metric (valueField + aggregator) and read values side-by-side. * ```ts * const metrics = [ From 0581145640b021a437930b97ff1b0bf4fab087b2 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:30:58 -0700 Subject: [PATCH 24/43] add tests for min and max aggregate methods --- vite-app/src/util/pivot.test.ts | 73 +++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/vite-app/src/util/pivot.test.ts b/vite-app/src/util/pivot.test.ts index 4875e7d3..63c21012 100644 --- a/vite-app/src/util/pivot.test.ts +++ b/vite-app/src/util/pivot.test.ts @@ -86,6 +86,79 @@ describe('computePivot', () => { expect(res.cells[rKeyWest][cKeyGadget].value).toBe(90) }) + it('computes minimum aggregator', () => { + const res = computePivot({ + data: rows, + rowFields: ['region'], + columnFields: ['product'], + valueField: 'amount', + aggregator: 'min', + }) + + const rKeyEast = 'East' + const rKeyWest = 'West' + const cKeyGadget = 'Gadget' + const cKeyWidget = 'Widget' + + // East Gadget: values -> [10] => min 10 + expect(res.cells[rKeyEast][cKeyGadget].value).toBe(10) + // West Gadget: values -> [90] => min 90 + expect(res.cells[rKeyWest][cKeyGadget].value).toBe(90) + // East Widget: values -> [200] => min 200 + expect(res.cells[rKeyEast][cKeyWidget].value).toBe(200) + // West Widget: values -> [120] => min 120 + expect(res.cells[rKeyWest][cKeyWidget].value).toBe(120) + }) + + it('computes maximum aggregator', () => { + const res = computePivot({ + data: rows, + rowFields: ['region'], + columnFields: ['product'], + valueField: 'amount', + aggregator: 'max', + }) + + const rKeyEast = 'East' + const rKeyWest = 'West' + const cKeyGadget = 'Gadget' + const cKeyWidget = 'Widget' + + // East Gadget: values -> [10] => max 10 + expect(res.cells[rKeyEast][cKeyGadget].value).toBe(10) + // West Gadget: values -> [90] => max 90 + expect(res.cells[rKeyWest][cKeyGadget].value).toBe(90) + // East Widget: values -> [200] => max 200 + expect(res.cells[rKeyEast][cKeyWidget].value).toBe(200) + // West Widget: values -> [120] => max 120 + expect(res.cells[rKeyWest][cKeyWidget].value).toBe(120) + }) + + it('handles empty cells for min/max aggregators', () => { + // Add a row with no valid numeric values + const rowsWithEmpty = [ + ...rows, + { region: 'North', rep: 'C', product: 'Widget', amount: 'not-a-number' }, + { region: 'North', rep: 'D', product: 'Gadget', amount: 'also-not-a-number' }, + ] + + const res = computePivot({ + data: rowsWithEmpty, + rowFields: ['region'], + columnFields: ['product'], + valueField: 'amount', + aggregator: 'min', + }) + + const rKeyNorth = 'North' + const cKeyWidget = 'Widget' + const cKeyGadget = 'Gadget' + + // North region has no valid numeric values, should return 0 for min + expect(res.cells[rKeyNorth][cKeyWidget].value).toBe(0) + expect(res.cells[rKeyNorth][cKeyGadget].value).toBe(0) + }) + it('supports custom aggregator', () => { const maxAgg: Aggregator = (values) => values.length ? Math.max(...values) : 0 From 8a868278e9e1fd94fc421e59313d5b4f49d38af3 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:35:26 -0700 Subject: [PATCH 25/43] add filter functionality --- vite-app/src/GlobalState.tsx | 3 +- vite-app/src/components/PivotTab.tsx | 167 ++++++++++++++++++++++++- vite-app/src/components/PivotTable.tsx | 7 ++ vite-app/src/util/pivot.test.ts | 28 +++++ vite-app/src/util/pivot.ts | 13 +- 5 files changed, 215 insertions(+), 3 deletions(-) diff --git a/vite-app/src/GlobalState.tsx b/vite-app/src/GlobalState.tsx index ceca4d1c..78fadf6a 100644 --- a/vite-app/src/GlobalState.tsx +++ b/vite-app/src/GlobalState.tsx @@ -59,7 +59,8 @@ export class GlobalState { } get flattenedDatasetKeys() { - return this.flattenedDataset.map((row) => Object.keys(row)); + if (this.flattenedDataset.length === 0) return []; + return Object.keys(this.flattenedDataset[0]); } get totalCount() { diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx index 6e4f790c..420f0d3e 100644 --- a/vite-app/src/components/PivotTab.tsx +++ b/vite-app/src/components/PivotTab.tsx @@ -125,6 +125,120 @@ const AggregatorSelector = ({ ); +const FilterSelector = ({ + filters, + onFiltersChange, + availableKeys, +}: { + filters: Array<{ field: string; operator: string; value: string }>; + onFiltersChange: ( + filters: Array<{ field: string; operator: string; value: string }> + ) => void; + availableKeys: string[]; +}) => { + const addFilter = () => { + onFiltersChange([...filters, { field: "", operator: "==", value: "" }]); + }; + + const removeFilter = (index: number) => { + onFiltersChange(filters.filter((_, i) => i !== index)); + }; + + const updateFilter = ( + index: number, + field: string, + operator: string, + value: string + ) => { + const newFilters = [...filters]; + newFilters[index] = { field, operator, value }; + onFiltersChange(newFilters); + }; + + const operators = [ + { value: "==", label: "equals" }, + { value: "!=", label: "not equals" }, + { value: ">", label: "greater than" }, + { value: "<", label: "less than" }, + { value: ">=", label: "greater than or equal" }, + { value: "<=", label: "less than or equal" }, + { value: "contains", label: "contains" }, + { value: "!contains", label: "not contains" }, + ]; + + return ( +
+
Filters:
+
+ {filters.map((filter, index) => ( +
+ + + + updateFilter( + index, + filter.field, + filter.operator, + e.target.value + ) + } + placeholder="Value" + className="px-2 py-1 text-xs border border-gray-300 rounded focus:outline-none focus:border-gray-500 min-w-32" + /> + +
+ ))} + +
+
+ ); +}; + const PivotTab = observer(() => { const [selectedRowFields, setSelectedRowFields] = useState([ "$.eval_metadata.name", @@ -136,6 +250,9 @@ const PivotTab = observer(() => { "$.evaluation_result.score" ); const [selectedAggregator, setSelectedAggregator] = useState("avg"); + const [filters, setFilters] = useState< + Array<{ field: string; operator: string; value: string }> + >([]); const createFieldHandler = ( setter: React.Dispatch> @@ -165,7 +282,48 @@ const PivotTab = observer(() => { }; }; - const availableKeys = state.flattenedDatasetKeys[0] || []; + const availableKeys = state.flattenedDatasetKeys; + + // Create filter function from filter configuration + const createFilterFunction = ( + filters: Array<{ field: string; operator: string; value: string }> + ) => { + if (filters.length === 0) return undefined; + + return (record: any) => { + return filters.every((filter) => { + if (!filter.field || !filter.value) return true; // Skip incomplete filters + + const fieldValue = record[filter.field]; + const filterValue = filter.value; + + switch (filter.operator) { + case "==": + return String(fieldValue) === filterValue; + case "!=": + return String(fieldValue) !== filterValue; + case ">": + return Number(fieldValue) > Number(filterValue); + case "<": + return Number(fieldValue) < Number(filterValue); + case ">=": + return Number(fieldValue) >= Number(filterValue); + case "<=": + return Number(fieldValue) <= Number(filterValue); + case "contains": + return String(fieldValue) + .toLowerCase() + .includes(filterValue.toLowerCase()); + case "!contains": + return !String(fieldValue) + .toLowerCase() + .includes(filterValue.toLowerCase()); + default: + return true; + } + }); + }; + }; return (
@@ -208,6 +366,12 @@ const PivotTab = observer(() => { onAggregatorChange={setSelectedAggregator} /> + + { aggregator={selectedAggregator as any} showRowTotals showColumnTotals + filter={createFilterFunction(filters)} />
); diff --git a/vite-app/src/components/PivotTable.tsx b/vite-app/src/components/PivotTable.tsx index 11ef5516..192f56a0 100644 --- a/vite-app/src/components/PivotTable.tsx +++ b/vite-app/src/components/PivotTable.tsx @@ -60,6 +60,11 @@ export interface PivotTableProps> { * Default: "-". */ emptyValue?: React.ReactNode; + /** + * Optional filter function to apply to records before pivoting. + * Return true to include the record, false to exclude it. + */ + filter?: (record: T) => boolean; } function toKey(parts: unknown[]): string { @@ -83,6 +88,7 @@ export function PivotTable>({ className = "", formatter = (v) => v.toLocaleString(undefined, { maximumFractionDigits: 3 }), emptyValue = "-", + filter, }: PivotTableProps) { const { rowKeyTuples, @@ -97,6 +103,7 @@ export function PivotTable>({ columnFields, valueField, aggregator, + filter, }); return ( diff --git a/vite-app/src/util/pivot.test.ts b/vite-app/src/util/pivot.test.ts index 63c21012..7949ccda 100644 --- a/vite-app/src/util/pivot.test.ts +++ b/vite-app/src/util/pivot.test.ts @@ -159,6 +159,34 @@ describe('computePivot', () => { expect(res.cells[rKeyNorth][cKeyGadget].value).toBe(0) }) + it('applies filter before pivoting', () => { + const res = computePivot({ + data: rows, + rowFields: ['region'], + columnFields: ['product'], + valueField: 'amount', + aggregator: 'sum', + filter: (record) => record.region === 'East', // Only include East region + }) + + // Should only have East region rows + expect(res.rowKeyTuples.map((t) => String(t))).toEqual(['East']) + + // Should still have all product columns + expect(res.colKeyTuples.map((t) => String(t))).toEqual(['Gadget', 'Widget']) + + // East Gadget: 10 (string convertible) + invalid -> 10 + expect(res.cells['East']['Gadget'].value).toBe(10) + // East Widget: 200 + expect(res.cells['East']['Widget'].value).toBe(200) + + // West region should not be present + expect(res.cells['West']).toBeUndefined() + + // Grand total should only include East region data + expect(res.grandTotal).toBe(210) // 10 + 200 + }) + it('supports custom aggregator', () => { const maxAgg: Aggregator = (values) => values.length ? Math.max(...values) : 0 diff --git a/vite-app/src/util/pivot.ts b/vite-app/src/util/pivot.ts index eb9a3ff9..b2a68cf2 100644 --- a/vite-app/src/util/pivot.ts +++ b/vite-app/src/util/pivot.ts @@ -103,6 +103,13 @@ export interface ComputePivotParams> { * @default "count" */ aggregator?: Aggregator; + + /** + * Optional filter function to apply to records before pivoting. + * Return true to include the record, false to exclude it. + * This is useful for focusing analysis on specific subsets of data. + */ + filter?: (record: T) => boolean; } /** @@ -236,11 +243,15 @@ export function computePivot>({ columnFields, valueField, aggregator = "count", + filter, }: ComputePivotParams): PivotComputationResult { + // Apply filter first if provided + const filteredData = filter ? data.filter(filter) : data; + // Filter out records that do not have defined values for all rowFields. // This avoids creating a row key of "undefined" and ensures such records // are not returned as part of the cells/row totals. - const dataWithDefinedRows = data.filter((rec) => + const dataWithDefinedRows = filteredData.filter((rec) => rowFields.every((f) => rec[f] !== undefined) ); const rowKeyTuples: unknown[][] = []; From 013991a99e1da22c29d87d2f89f5809d7f1e52b5 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:37:56 -0700 Subject: [PATCH 26/43] Enhance pivot functionality by adding support for column totals using various aggregation methods (avg, count) and updating tests to validate these changes. --- vite-app/src/util/pivot.test.ts | 49 +++++++++++++++++++++++++++++++++ vite-app/src/util/pivot.ts | 21 +++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/vite-app/src/util/pivot.test.ts b/vite-app/src/util/pivot.test.ts index 7949ccda..a9966657 100644 --- a/vite-app/src/util/pivot.test.ts +++ b/vite-app/src/util/pivot.test.ts @@ -187,6 +187,55 @@ describe('computePivot', () => { expect(res.grandTotal).toBe(210) // 10 + 200 }) + it('column totals use same aggregation method as cells', () => { + const res = computePivot({ + data: rows, + rowFields: ['region'], + columnFields: ['product'], + valueField: 'amount', + aggregator: 'avg', + }) + + // Row totals should sum the cell values (for display purposes) + expect(res.rowTotals['East']).toBe(105) // (10 + 200) / 2 = 105 + expect(res.rowTotals['West']).toBe(105) // (90 + 120) / 2 = 105 + + // Column totals should use the same aggregation method (avg) over all records in that column + // Gadget column: values [90, 10] -> avg = 50 + expect(res.colTotals['Gadget']).toBe(50) + // Widget column: values [120, 200] -> avg = 160 + expect(res.colTotals['Widget']).toBe(160) + + // Grand total should also use avg over all records + expect(res.grandTotal).toBe(105) // (90 + 10 + 120 + 200) / 4 = 105 + }) + + it('column totals with count aggregator', () => { + const res = computePivot({ + data: rows, + rowFields: ['region'], + columnFields: ['product'], + aggregator: 'count', // No valueField, just count records + }) + + // Each cell should count records + expect(res.cells['East']['Gadget'].value).toBe(2) // 2 records + expect(res.cells['East']['Widget'].value).toBe(1) // 1 record + expect(res.cells['West']['Gadget'].value).toBe(1) // 1 record + expect(res.cells['West']['Widget'].value).toBe(1) // 1 record + + // Row totals should sum the counts + expect(res.rowTotals['East']).toBe(3) // 2 + 1 + expect(res.rowTotals['West']).toBe(2) // 1 + 1 + + // Column totals should count total records in each column + expect(res.colTotals['Gadget']).toBe(3) // 2 + 1 = 3 records + expect(res.colTotals['Widget']).toBe(2) // 1 + 1 = 2 records + + // Grand total should count all records + expect(res.grandTotal).toBe(5) // Total records + }) + it('supports custom aggregator', () => { const maxAgg: Aggregator = (values) => values.length ? Math.max(...values) : 0 diff --git a/vite-app/src/util/pivot.ts b/vite-app/src/util/pivot.ts index b2a68cf2..f48ea7bc 100644 --- a/vite-app/src/util/pivot.ts +++ b/vite-app/src/util/pivot.ts @@ -316,10 +316,29 @@ export function computePivot>({ const value = aggregate(values, records, aggregator); cells[rKey][cKey] = { value, records }; rowTotals[rKey] += value; - colTotals[cKey] += value; } } + // Calculate column totals using the same aggregation method + for (const cKey of Object.keys(colTotals)) { + const columnRecords: T[] = []; + const columnValues: number[] = []; + + for (const rKey of Object.keys(cellRecords)) { + if (cellRecords[rKey][cKey]) { + columnRecords.push(...cellRecords[rKey][cKey]); + if (valueField != null) { + for (const rec of cellRecords[rKey][cKey]) { + const v = getNumber(rec[valueField]); + if (v != null) columnValues.push(v); + } + } + } + } + + colTotals[cKey] = aggregate(columnValues, columnRecords, aggregator); + } + // Grand total should follow the same aggregation semantics over the entire dataset // rather than summing per-row/per-column aggregates (which can be incorrect for // non-additive aggregations like "avg"). From 832a9f5d69c70081d3c8253ccf57e6b2dceb9d20 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:45:56 -0700 Subject: [PATCH 27/43] styling --- vite-app/src/components/PivotTable.tsx | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/vite-app/src/components/PivotTable.tsx b/vite-app/src/components/PivotTable.tsx index 192f56a0..330db95d 100644 --- a/vite-app/src/components/PivotTable.tsx +++ b/vite-app/src/components/PivotTable.tsx @@ -133,8 +133,11 @@ export function PivotTable>({ ))} {showRowTotals && ( - -
Total
+ +
Total
 )} @@ -177,9 +180,9 @@ export function PivotTable>({ align="right" nowrap medium - className="bg-gray-100" + className="bg-gray-100 border-l-2 border-l-gray-300" > -
+
{formatter(rowTotals[rKey] ?? 0)}
@@ -188,14 +191,14 @@ export function PivotTable>({ ); })} {showColumnTotals && ( - + {/* Total label spanning row header columns */} -
Total
+
Total
 {/* Column totals with enhanced styling */} {colKeyTuples.map((cTuple, cIdx) => { @@ -206,9 +209,9 @@ export function PivotTable>({ align="right" nowrap medium - className="bg-gray-100" + className="bg-gray-100 border-t-2 border-gray-300" > -
+
{formatter(colTotals[cKey] ?? 0)}
@@ -220,7 +223,7 @@ export function PivotTable>({ align="right" nowrap semibold - className="bg-gray-200" + className="bg-gray-200 border-t-0" >
{formatter(grandTotal)} From e283eea735219a203ddc68823830f00c5e25a35c Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:50:03 -0700 Subject: [PATCH 28/43] support date in flatten json --- .../__snapshots__/flatten-json.test.ts.snap | 9048 +++++++++++++++++ vite-app/src/util/flatten-json.test.ts | 232 +- vite-app/src/util/flatten-json.ts | 10 +- 3 files changed, 9280 insertions(+), 10 deletions(-) diff --git a/vite-app/src/util/__snapshots__/flatten-json.test.ts.snap b/vite-app/src/util/__snapshots__/flatten-json.test.ts.snap index 1498f63b..71f12e88 100644 --- a/vite-app/src/util/__snapshots__/flatten-json.test.ts.snap +++ b/vite-app/src/util/__snapshots__/flatten-json.test.ts.snap @@ -1,5 +1,9053 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html +exports[`flattenJson > against logs.json > flattens each entry in logs.json.logs and matches snapshot 1`] = ` +[ + { + "$.created_at": "2025-08-08T09:52:30.335005", + "$.input_metadata.row_id": "1", + "$.messages[0].content": "Hello", + "$.messages[0].role": "user", + "$.messages[1].content": "Hello", + "$.messages[1].role": "assistant", + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T09:47:43.816761", + "$.input_metadata.row_id": "row_0", + "$.messages[0].content": "Hello 0", + "$.messages[0].role": "user", + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T09:41:55.015216", + "$.input_metadata.row_id": "row_1", + "$.messages[0].content": "Hello 1", + "$.messages[0].role": "user", + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T09:41:55.015767", + "$.input_metadata.row_id": "row_2", + "$.messages[0].content": "Hello 2", + "$.messages[0].role": "user", + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T10:00:08.522553", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev12+g2dc40d5.dirty", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "describe-use-entire-person-education", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.pid": 67635, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T10:01:12.825337", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev12+g2dc40d5.dirty", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "meet-solid-big-picture-car", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.pid": 68229, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:06:51.152819", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev15+g6486dbd", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "leave-ask-logical-face-way", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.messages[2].content": "I'll help you learn about managing deployments on Fireworks AI. Let me search for comprehensive information about deployment management.", + "$.messages[2].role": "assistant", + "$.messages[2].tool_calls[0].function.arguments": "{"query": "deployment management Fireworks AI"}", + "$.messages[2].tool_calls[0].function.name": "search", + "$.messages[2].tool_calls[0].id": "functions.search:0", + "$.messages[2].tool_calls[0].type": "function", + "$.messages[3].content[0].text": "Title: Fireworks AI Developer Platform +Link: https://docs.fireworks.ai/getting-started/introduction +Content: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Start building with open source AI models + +", + "$.messages[3].content[0].type": "text", + "$.messages[3].content[1].text": "Title: How do I close my Fireworks.ai account? +Link: https://docs.fireworks.ai/faq-new/account-access/how-do-i-close-my-fireworksai-account +Content: To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +How do I close my Fireworks.ai account? +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +", + "$.messages[3].content[1].type": "text", + "$.messages[3].content[2].text": "Title: Do you host your deployments in the EU or Asia? +Link: https://docs.fireworks.ai/faq-new/support-general/do-you-host-your-deployments-in-the-eu-or-asia +Content: We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +Do you host your deployments in the EU or Asia? +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +", + "$.messages[3].content[2].type": "text", + "$.messages[3].content[3].text": "Title: Introducing the Fireworks Build SDK +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-introduction +Content: The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +", + "$.messages[3].content[3].type": "text", + "$.messages[3].content[4].text": "Title: On-demand deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +On-demand deployments +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +", + "$.messages[3].content[4].type": "text", + "$.messages[3].content[5].text": "Title: I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +Link: https://docs.fireworks.ai/faq-new/account-access/i-have-multiple-fireworks-accounts-when-i-try-to-login-with-google-on-fireworks +Content: If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +", + "$.messages[3].content[5].type": "text", + "$.messages[3].content[6].text": "Title: Introduction +Link: https://docs.fireworks.ai/api-reference/introduction +Content: Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Introduction +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +", + "$.messages[3].content[6].type": "text", + "$.messages[3].content[7].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Create a Deployment on Fireworks AI platform + +Create a Deployment on Fireworks AI platform + +Creates a new deployment. + +", + "$.messages[3].content[7].type": "text", + "$.messages[3].content[8].text": "Title: How much does Fireworks cost? +Link: https://docs.fireworks.ai/faq-new/billing-pricing/how-much-does-fireworks-cost +Content: Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +How much does Fireworks cost? +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +", + "$.messages[3].content[8].type": "text", + "$.messages[3].content[9].text": "Title: Python (Fireworks) +Link: https://docs.fireworks.ai/getting-started/quickstart +Content: This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +Set up your developer environment & call the Fireworks API +This is the recommended way to get started : This tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: Sending the first API Request You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. Fireworks AI platform offers drop-in replacement with OpenAI Python Client. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $OPENAI_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx OPENAI_API_BASE "https://api.fireworks.ai/inference/v1" +setx OPENAI_API_KEY "" + +", + "$.messages[3].content[9].type": "text", + "$.messages[3].role": "tool", + "$.messages[3].tool_call_id": "functions.search:0", + "$.messages[4].content": "Let me search for more specific information about deployment management tools and commands.", + "$.messages[4].role": "assistant", + "$.messages[4].tool_calls[0].function.arguments": "{"query": "firectl deployment management commands CLI"}", + "$.messages[4].tool_calls[0].function.name": "search", + "$.messages[4].tool_calls[0].id": "functions.search:1", + "$.messages[4].tool_calls[0].type": "function", + "$.messages[5].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[5].content[0].type": "text", + "$.messages[5].content[1].text": "Title: Installing the CLI +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: To create and manage on-demand deployments, use our CLI tool firectl . Install the tool by following the installation guide . Verify that your installation is working and you are logged into the correct account by running: firectl whoami + +To create and manage on-demand deployments, use our CLI tool firectl . Install the tool by following the installation guide . Verify that your installation is working and you are logged into the correct account by running: firectl whoami + +To create and manage on-demand deployments, use our CLI tool firectl . Install the tool by following the installation guide . Verify that your installation is working and you are logged into the correct account by running: + +", + "$.messages[5].content[1].type": "text", + "$.messages[5].content[2].text": "Title: Cli overview +Link: https://docs.fireworks.ai/evaluators/cli_reference/cli_overview +Content: Command Line Interface Reference +The Reward Kit provides a command-line interface (CLI) for common operations like previewing evaluations, deploying reward functions, and running agent evaluations. + +Command Line Interface Reference +The Reward Kit provides a command-line interface (CLI) for common operations like previewing evaluations, deploying reward functions, and running agent evaluations. + +Command Overview +The Reward Kit CLI supports the following main commands: run : Run a local evaluation pipeline using a Hydra configuration. preview : Preview evaluation results or re-evaluate generated outputs. deploy : Deploy a reward function as an evaluator. agent-eval : Run agent evaluations on task bundles. list : List existing evaluators (coming soon). delete : Delete an evaluator (coming soon). + +", + "$.messages[5].content[2].type": "text", + "$.messages[5].content[3].text": "Title: Using the CLI (reward-kit deploy) +Link: https://docs.fireworks.ai/evaluators/developer_guide/evaluation_workflows +Content: The reward-kit deploy command is suitable for deploying reward functions defined in script files. The --metrics-folders argument should point to the directory containing your reward function script (e.g., a main.py with the @reward_function decorator). For more details on reward-kit deploy , see the CLI Reference. + +The reward-kit deploy command is suitable for deploying reward functions defined in script files. The --metrics-folders argument should point to the directory containing your reward function script (e.g., a main.py with the @reward_function decorator). For more details on reward-kit deploy , see the CLI Reference. + +The reward-kit deploy command is suitable for deploying reward functions defined in script files. The --metrics-folders argument should point to the directory containing your reward function script (e.g., a main.py with the @reward_function decorator). # Deploy with the CLI +reward-kit deploy \\ + --id helpfulness-evaluator \\ + --metrics-folders "helpfulness=./path/to/your/metric_script_directory" \\ + --display-name "Helpfulness Evaluator" \\ + --description "Evaluates the helpfulness of responses" \\ + --force + +", + "$.messages[5].content[3].type": "text", + "$.messages[5].content[4].text": "Title: Does Fireworks support custom base models? +Link: https://docs.fireworks.ai/faq-new/models-inference/does-fireworks-support-custom-base-models +Content: Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Does Fireworks support custom base models? +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +", + "$.messages[5].content[4].type": "text", + "$.messages[5].content[5].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[5].content[5].type": "text", + "$.messages[5].content[6].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[5].content[6].type": "text", + "$.messages[5].content[7].text": "Title: 1. Setup +Link: https://docs.fireworks.ai/tools-sdks/python-client/the-tutorial +Content: To get started with the Fireworks AI Python SDK, you need to install the firectl CLI tool and create an API key. Install our CLI tool firectl to interact with the Fireworks AI platform. Sign in to Fireworks by running the following command: firectl signin + +To get started with the Fireworks AI Python SDK, you need to install the firectl CLI tool and create an API key. Install our CLI tool firectl to interact with the Fireworks AI platform. Sign in to Fireworks by running the following command: firectl signin + +To get started with the Fireworks AI Python SDK, you need to install the firectl CLI tool and create an API key. Install our CLI tool firectl to interact with the Fireworks AI platform. Sign in to Fireworks by running the following command: A browser window will open to the Fireworks AI login page. Once you login, your machine will be authenticated. Create an API key by running the following command: $ firectl create api-key --key-name "quick-start" +Key Id: key_42vAYeb7rwt9zzg1 +Display Name: quick-start +Key: fw_3ZLd.... +Secure: true +Be sure to save this key. It will not be shown again. + +", + "$.messages[5].content[7].type": "text", + "$.messages[5].content[8].text": "Title: Deploying a fine-tuned model using an on-demand deployment +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-models +Content: Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Deploying a fine-tuned model serverlessly +Not all base models support serverless addons. Please check this list for the full list of serverless models that support LoRA add-ons. If your base model supports serverless addons, you can run the following command to load the fine-tuned model into a serverless deployment: firectl load-lora + +", + "$.messages[5].content[8].type": "text", + "$.messages[5].content[9].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. firectl create deployment [flags] + +Example +firectl create deployment falcon-7b + +", + "$.messages[5].content[9].type": "text", + "$.messages[5].role": "tool", + "$.messages[5].tool_call_id": "functions.search:1", + "$.pid": 83882, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:10:27.844496", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev15+g6486dbd", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "investigate-recent-dangerous-hope-magazine", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.pid": 84847, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:12:00.215991", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev15+g6486dbd.dirty", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "seem-environmental-media-television-concert", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.pid": 85545, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:12:34.382113", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev15+g6486dbd.dirty", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "treat-suppose-salty-week-plan", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.pid": 85754, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:12:46.750720", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev15+g6486dbd.dirty", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "contain-recent-bad-plan-change", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.pid": 85851, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:15:45.037115", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "finished", + "$.eval_metadata.version": "v0.2.6.dev15+g6486dbd.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "At least one tool call was made", + "$.evaluation_result.score": 1, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "move-little-tidy-pain-work", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.messages[2].content": "I'll help you learn about managing deployments on Fireworks! Let me search for comprehensive information about deployment management.", + "$.messages[2].role": "assistant", + "$.messages[2].tool_calls[0].function.arguments": "{"query": "deployments management create update delete scale"}", + "$.messages[2].tool_calls[0].function.name": "search", + "$.messages[2].tool_calls[0].id": "functions.search:0", + "$.messages[2].tool_calls[0].type": "function", + "$.messages[3].content[0].text": "Title: Delete Deployment +Link: https://docs.fireworks.ai/api-reference/delete-deployment +Content: + + + +REST Endpoint DELETE /v1/accounts/{account_id}/deployments/{deployment_id} + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" +deployment_id: + string + Description: The Deployment Id + Example: "" + +query params: +hard: + boolean + Description: If true, this will perform a hard deletion. + Example: true +ignoreChecks: + boolean + Description: If true, this will ignore checks and force the deletion of a deployment that is currently +deployed and is in use. + Example: true + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {} + +", + "$.messages[3].content[0].type": "text", + "$.messages[3].content[1].text": "Title: How does billing and scaling work for on-demand GPU deployments? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/how-does-billing-and-scaling-work-for-on-demand-gpu-deployments +Content: On-demand GPU deployments have unique billing and scaling characteristics compared to serverless deployments: Billing: Charges start when the server begins accepting requests Billed by GPU-second for each active instance Costs accumulate even if there are no active API calls Scaling options: Supports autoscaling from 0 to multiple GPUs Each additional GPU adds to the billing rate Can handle unlimited requests within the GPUā€™s capacity Management requirements: Not fully serverless; requires some manual management Manually delete deployments when no longer needed Or configure autoscaling to scale down to 0 during inactive periods Cost control tips: Regularly monitor active deployments Delete unused deployments to avoid unnecessary costs Consider serverless options for intermittent usage Use autoscaling to 0 to optimize costs during low-demand times + +On-demand GPU deployments have unique billing and scaling characteristics compared to serverless deployments: Billing: Charges start when the server begins accepting requests Billed by GPU-second for each active instance Costs accumulate even if there are no active API calls Scaling options: Supports autoscaling from 0 to multiple GPUs Each additional GPU adds to the billing rate Can handle unlimited requests within the GPUā€™s capacity Management requirements: Not fully serverless; requires some manual management Manually delete deployments when no longer needed Or configure autoscaling to scale down to 0 during inactive periods Cost control tips: Regularly monitor active deployments Delete unused deployments to avoid unnecessary costs Consider serverless options for intermittent usage Use autoscaling to 0 to optimize costs during low-demand times + +How does billing and scaling work for on-demand GPU deployments? +On-demand GPU deployments have unique billing and scaling characteristics compared to serverless deployments: Billing: Charges start when the server begins accepting requests Billed by GPU-second for each active instance Costs accumulate even if there are no active API calls Scaling options: Supports autoscaling from 0 to multiple GPUs Each additional GPU adds to the billing rate Can handle unlimited requests within the GPUā€™s capacity Management requirements: Not fully serverless; requires some manual management Manually delete deployments when no longer needed Or configure autoscaling to scale down to 0 during inactive periods Cost control tips: Regularly monitor active deployments Delete unused deployments to avoid unnecessary costs Consider serverless options for intermittent usage Use autoscaling to 0 to optimize costs during low-demand times + +", + "$.messages[3].content[1].type": "text", + "$.messages[3].content[2].text": "Title: Managing default deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: To delete a default deployment you must delete all other deployments for the same model first, or designate a different deployed model as the default as described above. This is to ensure that querying by model name will always route to an unambiguous default deployment as long as deployments for the model exist. + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: To delete a default deployment you must delete all other deployments for the same model first, or designate a different deployed model as the default as described above. This is to ensure that querying by model name will always route to an unambiguous default deployment as long as deployments for the model exist. + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: firectl update deployed-model --default + +", + "$.messages[3].content[2].type": "text", + "$.messages[3].content[3].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. + +Creates a new deployment. + +Creates a new deployment. firectl create deployment [flags] + +", + "$.messages[3].content[3].type": "text", + "$.messages[3].content[4].text": "Title: Delete a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/delete-resources +Content: Deletes an model deployment. + +Deletes an model deployment. + +Deletes an model deployment. firectl delete deployment [flags] + +", + "$.messages[3].content[4].type": "text", + "$.messages[3].content[5].text": "Title: Update Deployment +Link: https://docs.fireworks.ai/api-reference/update-deployment +Content: + + + +REST Endpoint PATCH /v1/accounts/{account_id}/deployments/{deployment_id} + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" +deployment_id: + string + Description: The Deployment Id + Example: "" + + +Request body: object + Description: N/A + Example: {"displayName":"","description":"","expireTime":"2023-11-07T05:31:56Z","minReplicaCount":123,"maxReplicaCount":123,"autoscalingPolicy":{"scaleUpWindow":"","scaleDownWindow":"","scaleToZeroWindow":"","loadTargets":{}},"baseModel":"","acceleratorCount":123,"acceleratorType":"ACCELERATOR_TYPE_UNSPECIFIED","precision":"PRECISION_UNSPECIFIED","enableAddons":true,"draftTokenCount":123,"draftModel":"","ngramSpeculationLength":123,"deploymentTemplate":"","autoTune":{"longPrompt":true},"placement":{"region":"REGION_UNSPECIFIED","multiRegion":"MULTI_REGION_UNSPECIFIED","regions":["REGION_UNSPECIFIED"]},"disableDeploymentSizeValidation":true} + + +Responses: +200: object + Description: N/A + Example: {"name":"","displayName":"","description":"","createTime":"2023-11-07T05:31:56Z","expireTime":"2023-11-07T05:31:56Z","purgeTime":"2023-11-07T05:31:56Z","deleteTime":"2023-11-07T05:31:56Z","state":"STATE_UNSPECIFIED","status":{"code":"OK","message":""},"minReplicaCount":123,"maxReplicaCount":123,"replicaCount":123,"autoscalingPolicy":{"scaleUpWindow":"","scaleDownWindow":"","scaleToZeroWindow":"","loadTargets":{}},"baseModel":"","acceleratorCount":123,"acceleratorType":"ACCELERATOR_TYPE_UNSPECIFIED","precision":"PRECISION_UNSPECIFIED","cluster":"","enableAddons":true,"draftTokenCount":123,"draftModel":"","ngramSpeculationLength":123,"numPeftDeviceCached":123,"deploymentTemplate":"","autoTune":{"longPrompt":true},"placement":{"region":"REGION_UNSPECIFIED","multiRegion":"MULTI_REGION_UNSPECIFIED","regions":["REGION_UNSPECIFIED"]},"region":"REGION_UNSPECIFIED","updateTime":"2023-11-07T05:31:56Z","disableDeploymentSizeValidation":true} + +", + "$.messages[3].content[5].type": "text", + "$.messages[3].content[6].text": "Title: What should I expect for deployment and scaling performance? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/what-should-i-expect-for-deployment-and-scaling-performance +Content: Initial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long + +Initial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long + +What should I expect for deployment and scaling performance? +Initial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long + +", + "$.messages[3].content[6].type": "text", + "$.messages[3].content[7].text": "Title: delete_deployment(ignore_checks: bool = False, wait: bool = True) +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-reference +Content: Deletes the deployment associated with this LLM instance if one exists. Arguments: ignore_checks bool, optional - Whether to ignore safety checks. Defaults to False. wait bool, optional - Whether to wait for deletion to complete. Defaults to True. llm.delete_deployment(ignore_checks=True) + +Deletes the deployment associated with this LLM instance if one exists. Arguments: ignore_checks bool, optional - Whether to ignore safety checks. Defaults to False. wait bool, optional - Whether to wait for deletion to complete. Defaults to True. + +scale_to_zero() +Sends a request to scale the deployment to 0 replicas but does not wait for it to complete. Returns: The deployment object, or None if no deployment exists deployment = llm.scale_to_zero() + +", + "$.messages[3].content[7].type": "text", + "$.messages[3].content[8].text": "Title: How does billing work for on-demand deployments? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/how-does-billing-work-for-on-demand-deployments +Content: On-demand deployments come with automatic cost optimization features: Default autoscaling : Automatically scales to 0 replicas when not in use Pay for what you use : Charged only for GPU time when replicas are active Flexible configuration : Customize autoscaling behavior to match your needs Best practices for cost management: Leverage default autoscaling : The system automatically scales down deployments when not in use Customize carefully : While you can modify autoscaling behavior using our configuration options , note that preventing scale-to-zero will result in continuous GPU charges Consider your use case : For intermittent or low-frequency usage, serverless deployments might be more cost-effective For detailed configuration options, see our deployment guide. + +On-demand deployments come with automatic cost optimization features: Default autoscaling : Automatically scales to 0 replicas when not in use Pay for what you use : Charged only for GPU time when replicas are active Flexible configuration : Customize autoscaling behavior to match your needs Best practices for cost management: Leverage default autoscaling : The system automatically scales down deployments when not in use Customize carefully : While you can modify autoscaling behavior using our configuration options , note that preventing scale-to-zero will result in continuous GPU charges Consider your use case : For intermittent or low-frequency usage, serverless deployments might be more cost-effective For detailed configuration options, see our deployment guide. + +How does billing work for on-demand deployments? +On-demand deployments come with automatic cost optimization features: Default autoscaling : Automatically scales to 0 replicas when not in use Pay for what you use : Charged only for GPU time when replicas are active Flexible configuration : Customize autoscaling behavior to match your needs Best practices for cost management: Leverage default autoscaling : The system automatically scales down deployments when not in use Customize carefully : While you can modify autoscaling behavior using our configuration options , note that preventing scale-to-zero will result in continuous GPU charges Consider your use case : For intermittent or low-frequency usage, serverless deployments might be more cost-effective For detailed configuration options, see our deployment guide. + +", + "$.messages[3].content[8].type": "text", + "$.messages[3].content[9].text": "Title: Create Deployment +Link: https://docs.fireworks.ai/api-reference/create-deployment +Content: + + + +REST Endpoint POST /v1/accounts/{account_id}/deployments + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" + +query params: +disableAutoDeploy: + boolean + Description: By default, a deployment created with a currently undeployed base model +will be deployed to this deployment. If true, this auto-deploy function +is disabled. + Example: true +disableSpeculativeDecoding: + boolean + Description: By default, a deployment will use the speculative decoding settings from +the base model. If true, this will disable speculative decoding. + Example: true +deploymentId: + string + Description: The ID of the deployment. If not specified, a random ID will be generated. + Example: "" + + +Request body: object + Description: N/A + Example: {"displayName":"","description":"","expireTime":"2023-11-07T05:31:56Z","minReplicaCount":123,"maxReplicaCount":123,"autoscalingPolicy":{"scaleUpWindow":"","scaleDownWindow":"","scaleToZeroWindow":"","loadTargets":{}},"baseModel":"","acceleratorCount":123,"acceleratorType":"ACCELERATOR_TYPE_UNSPECIFIED","precision":"PRECISION_UNSPECIFIED","enableAddons":true,"draftTokenCount":123,"draftModel":"","ngramSpeculationLength":123,"deploymentTemplate":"","autoTune":{"longPrompt":true},"placement":{"region":"REGION_UNSPECIFIED","multiRegion":"MULTI_REGION_UNSPECIFIED","regions":["REGION_UNSPECIFIED"]},"disableDeploymentSizeValidation":true} + + +Responses: +200: object + Description: N/A + Example: {"name":"","displayName":"","description":"","createTime":"2023-11-07T05:31:56Z","expireTime":"2023-11-07T05:31:56Z","purgeTime":"2023-11-07T05:31:56Z","deleteTime":"2023-11-07T05:31:56Z","state":"STATE_UNSPECIFIED","status":{"code":"OK","message":""},"minReplicaCount":123,"maxReplicaCount":123,"replicaCount":123,"autoscalingPolicy":{"scaleUpWindow":"","scaleDownWindow":"","scaleToZeroWindow":"","loadTargets":{}},"baseModel":"","acceleratorCount":123,"acceleratorType":"ACCELERATOR_TYPE_UNSPECIFIED","precision":"PRECISION_UNSPECIFIED","cluster":"","enableAddons":true,"draftTokenCount":123,"draftModel":"","ngramSpeculationLength":123,"numPeftDeviceCached":123,"deploymentTemplate":"","autoTune":{"longPrompt":true},"placement":{"region":"REGION_UNSPECIFIED","multiRegion":"MULTI_REGION_UNSPECIFIED","regions":["REGION_UNSPECIFIED"]},"region":"REGION_UNSPECIFIED","updateTime":"2023-11-07T05:31:56Z","disableDeploymentSizeValidation":true} + +", + "$.messages[3].content[9].type": "text", + "$.messages[3].role": "tool", + "$.messages[3].tool_call_id": "functions.search:0", + "$.messages[4].content": "Let me search for more specific information about deployment creation and management workflows.", + "$.messages[4].role": "assistant", + "$.messages[4].tool_calls[0].function.arguments": "{"query": "firectl deployment management commands create update list"}", + "$.messages[4].tool_calls[0].function.name": "search", + "$.messages[4].tool_calls[0].id": "functions.search:1", + "$.messages[4].tool_calls[0].type": "function", + "$.messages[5].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[5].content[0].type": "text", + "$.messages[5].content[1].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. + +", + "$.messages[5].content[1].type": "text", + "$.messages[5].content[2].text": "Title: Creating an on-demand deployment +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: The state field should show READY. The deployment ID is the last part of the deployment name: accounts//deployments/. You can also list all deployments in your account at any time by running: firectl list deployments + +Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: The state field should show READY. The deployment ID is the last part of the deployment name: accounts//deployments/. You can also list all deployments in your account at any time by running: firectl list deployments + +Publishing a deployed model +By default, models can only be queried by the account that owns them. To make a deployment public so anyone with a valid Fireworks API key can query it, update the deployed model with the --public flag. Find the Deployed Model ID Every model running on a deployment receives a unique deployed model ID. You can find this ID if you know the model name and deployment name using the following command: firectl list deployed-models --filter 'model="" AND deployment=""' + +", + "$.messages[5].content[2].type": "text", + "$.messages[5].content[3].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[5].content[3].type": "text", + "$.messages[5].content[4].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[5].content[4].type": "text", + "$.messages[5].content[5].text": "Title: Deploying a fine-tuned model using an on-demand deployment +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-models +Content: Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +firectl +firectl create dataset /path/to/jsonl/file + +", + "$.messages[5].content[5].type": "text", + "$.messages[5].content[6].text": "Title: Does Fireworks support custom base models? +Link: https://docs.fireworks.ai/faq-new/models-inference/does-fireworks-support-custom-base-models +Content: Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Does Fireworks support custom base models? +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +", + "$.messages[5].content[6].type": "text", + "$.messages[5].content[7].text": "Title: firectl +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-vlm +Content: # Create a deployment for your fine-tuned VLM +firectl create deployment my-custom-vlm + +# Check deployment status +firectl get deployment accounts/your-account/deployment/deployment-id + +# Create a deployment for your fine-tuned VLM +firectl create deployment my-custom-vlm + +# Check deployment status +firectl get deployment accounts/your-account/deployment/deployment-id + +firectl create sftj \\ + --base-model accounts/fireworks/models/qwen2p5-vl-32b-instruct \\ + --dataset my-vlm-dataset \\ + --output-model my-custom-vlm \\ + --epochs 3 For additional parameters like learning rates, evaluation datasets, and batch sizes, see Additional SFT job settings . + +", + "$.messages[5].content[7].type": "text", + "$.messages[5].content[8].text": "Title: Uploading the model locally (firectl) +Link: https://docs.fireworks.ai/models/uploading-custom-models +Content: To upload a custom base model, run the following command. firectl create model /path/to/files/ + +To upload a custom base model, run the following command. firectl create model /path/to/files/ + +Publishing +By default, all models you create are only visible to and deployable by users within your account. To publish a model so anyone with a Fireworks account can deploy it, you can create it with the --public flag. This will allow it to show up in public model lists. To unpublish the model, just run firectl update model --public=false + +", + "$.messages[5].content[8].type": "text", + "$.messages[5].content[9].text": "Title: firectl +Link: https://docs.fireworks.ai/models/quantization +Content: firectl create deployment --accelerator-type NVIDIA_H100_80GB --precision FP8 + +firectl create deployment --accelerator-type NVIDIA_H100_80GB --precision FP8 + +Creating an FP8 deployment +By default, creating a dedicated deployment will use the FP16 checkpoint. To see what precisions are available for a +model, run: get model + +", + "$.messages[5].content[9].type": "text", + "$.messages[5].role": "tool", + "$.messages[5].tool_call_id": "functions.search:1", + "$.messages[6].content": "Based on my research, here's a comprehensive guide to managing deployments on Fireworks: + +## Deployment Management on Fireworks + +### 1. Creating Deployments + +**Basic deployment creation:** +\`\`\`bash +# Create a deployment from the model library +firectl create deployment accounts/fireworks/models/ + +# Create a deployment for a custom model +firectl create deployment + +# Create with specific configuration +firectl create deployment --wait +\`\`\` + +**Advanced deployment options:** +\`\`\`bash +# Deploy with specific hardware requirements +firectl create deployment --accelerator-type NVIDIA_H100_80GB --precision FP8 + +# Deploy LoRA fine-tuned models (single command) +firectl create deployment "accounts/fireworks/models/" + +# Deploy with addons enabled (for multi-LoRA) +firectl create deployment --enable-addons +\`\`\` + +### 2. Monitoring Deployments + +**Check deployment status:** +\`\`\`bash +# List all deployments +firectl list deployments + +# Get specific deployment details +firectl get deployment accounts//deployments/ + +# Check if deployment is ready +# The state field should show "READY" +\`\`\` + +**Verify deployed models:** +\`\`\`bash +# List deployed models +firectl list deployed-models + +# Find specific deployed model +firectl list deployed-models --filter 'model="" AND deployment=""' +\`\`\` + +### 3. Managing Default Deployments + +**Check default deployment:** +\`\`\`bash +# See which deployment is marked as default +firectl list deployed-models +# Look for "Default: true" in the output +\`\`\` + +**Update default deployment:** +\`\`\`bash +# Update which deployment is the default +firectl update deployed-model --default +\`\`\` + +**Important note:** To delete a default deployment, you must either: +- Delete all other deployments for the same model first, OR +- Designate a different deployed model as the default + +### 4. Scaling and Cost Management + +**Autoscaling configuration:** +- **Default behavior:** Automatically scales to 0 replicas when not in use +- **Billing:** Pay only for GPU time when replicas are active +- **Manual scaling:** Use autoscaling policies to customize behavior + +**Cost optimization commands:** +\`\`\`bash +# Scale deployment to zero (cost saving) +firectl update deployment --min-replica-count 0 + +# Or using the Python SDK +from fireworks.client import Fireworks +llm = Fireworks(model="") +llm.scale_to_zero() +\`\`\` + +### 5. Updating Deployments + +**Update deployment configuration:** +\`\`\`bash +# Update display name, description, or scaling policies +firectl update deployment \\ + --display-name "New Name" \\ + --min-replica-count 1 \\ + --max-replica-count 5 +\`\`\` + +**Available update options:** +- Display name and description +- Replica counts (min/max) +- Autoscaling policies +- Expiration time +- Hardware configuration + +### 6. Deleting Deployments + +**Basic deletion:** +\`\`\`bash +# Delete a deployment +firectl delete deployment + +# Force deletion (ignoring safety checks) +firectl delete deployment --ignore-checks + +# Using Python SDK +llm.delete_deployment(ignore_checks=True) +\`\`\` + +**API deletion:** +\`\`\`bash +# Via REST API +curl -X DELETE \\ + https://api.fireworks.ai/v1/accounts/{account_id}/deployments/{deployment_id} \\ + -H "Authorization: Bearer YOUR_API_KEY" +\`\`\` + +### 7. Deployment Performance Expectations + +**Timeline expectations:** +- **Initial deployment:** Should complete within minutes +- **Scaling from zero:** Brief availability delays while scaling up +- **Troubleshooting:** If deployment takes >1 hour, likely indicates a crash + +### 8. Best Practices + +**Cost management:** +1. **Monitor regularly:** Check active deployments frequently +2. **Delete unused deployments:** Avoid unnecessary GPU charges +3. **Use autoscaling:** Leverage scale-to-zero for intermittent usage +4. **Consider serverless:** For low-frequency usage patterns + +**Configuration tips:** +- Use \`--wait\` flag for synchronous deployment creation +- Configure appropriate min/max replica counts for your use case +- Set expiration times for temporary deployments +- Monitor deployment status during initial creation + +### 9. Common Workflows + +**Complete deployment lifecycle:** +\`\`\`bash +# 1. Create deployment +firectl create deployment --wait + +# 2. Verify it's ready +firectl get deployment + +# 3. Use the deployment +# Make API calls to the deployment + +# 4. Monitor usage +firectl list deployments + +# 5. Scale down when not needed +firectl update deployment --min-replica-count 0 + +# 6. Delete when completely done +firectl delete deployment +\`\`\` + +This comprehensive guide covers everything from basic deployment creation to advanced cost optimization strategies. The key is understanding the balance between performance needs and cost efficiency through proper scaling configuration.", + "$.messages[6].role": "assistant", + "$.pid": 86994, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:19:37.022552", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "finished", + "$.eval_metadata.version": "v0.2.6.dev15+g6486dbd.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "At least one tool call was made", + "$.evaluation_result.score": 1, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "call-stand-low-night-conference", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.messages[2].content": "I'll help you learn about managing deployments on Fireworks AI. Let me search for comprehensive documentation about deployment management.", + "$.messages[2].role": "assistant", + "$.messages[2].tool_calls[0].function.arguments": "{"query": "deployment management Fireworks AI"}", + "$.messages[2].tool_calls[0].function.name": "search", + "$.messages[2].tool_calls[0].id": "functions.search:0", + "$.messages[2].tool_calls[0].type": "function", + "$.messages[3].content[0].text": "Title: Fireworks AI Developer Platform +Link: https://docs.fireworks.ai/getting-started/introduction +Content: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Start building with open source AI models + +", + "$.messages[3].content[0].type": "text", + "$.messages[3].content[1].text": "Title: How do I close my Fireworks.ai account? +Link: https://docs.fireworks.ai/faq-new/account-access/how-do-i-close-my-fireworksai-account +Content: To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +How do I close my Fireworks.ai account? +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +", + "$.messages[3].content[1].type": "text", + "$.messages[3].content[2].text": "Title: Do you host your deployments in the EU or Asia? +Link: https://docs.fireworks.ai/faq-new/support-general/do-you-host-your-deployments-in-the-eu-or-asia +Content: We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +Do you host your deployments in the EU or Asia? +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +", + "$.messages[3].content[2].type": "text", + "$.messages[3].content[3].text": "Title: Introducing the Fireworks Build SDK +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-introduction +Content: The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +", + "$.messages[3].content[3].type": "text", + "$.messages[3].content[4].text": "Title: On-demand deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +On-demand deployments +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +", + "$.messages[3].content[4].type": "text", + "$.messages[3].content[5].text": "Title: I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +Link: https://docs.fireworks.ai/faq-new/account-access/i-have-multiple-fireworks-accounts-when-i-try-to-login-with-google-on-fireworks +Content: If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +", + "$.messages[3].content[5].type": "text", + "$.messages[3].content[6].text": "Title: Introduction +Link: https://docs.fireworks.ai/api-reference/introduction +Content: Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Introduction +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +", + "$.messages[3].content[6].type": "text", + "$.messages[3].content[7].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Create a Deployment on Fireworks AI platform + +Create a Deployment on Fireworks AI platform + +Creates a new deployment. + +", + "$.messages[3].content[7].type": "text", + "$.messages[3].content[8].text": "Title: How much does Fireworks cost? +Link: https://docs.fireworks.ai/faq-new/billing-pricing/how-much-does-fireworks-cost +Content: Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +How much does Fireworks cost? +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +", + "$.messages[3].content[8].type": "text", + "$.messages[3].content[9].text": "Title: Python (Fireworks) +Link: https://docs.fireworks.ai/getting-started/quickstart +Content: This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +Set up your developer environment & call the Fireworks API +This is the recommended way to get started : This tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: Sending the first API Request You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. Fireworks AI platform offers drop-in replacement with OpenAI Python Client. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $OPENAI_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx OPENAI_API_BASE "https://api.fireworks.ai/inference/v1" +setx OPENAI_API_KEY "" + +", + "$.messages[3].content[9].type": "text", + "$.messages[3].role": "tool", + "$.messages[3].tool_call_id": "functions.search:0", + "$.messages[4].content": "Let me search for more specific information about deployment management commands and workflows.", + "$.messages[4].role": "assistant", + "$.messages[4].tool_calls[0].function.arguments": "{"query": "firectl deployment commands create manage update delete"}", + "$.messages[4].tool_calls[0].function.name": "search", + "$.messages[4].tool_calls[0].id": "functions.search:1", + "$.messages[4].tool_calls[0].type": "function", + "$.messages[5].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[5].content[0].type": "text", + "$.messages[5].content[1].text": "Title: Delete a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/delete-resources +Content: Deletes an model deployment. firectl delete deployment [flags] + +Deletes an model deployment. firectl delete deployment [flags] + +Example +firectl delete deployment my-deployment + +", + "$.messages[5].content[1].type": "text", + "$.messages[5].content[2].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. firectl create deployment [flags] + +Create a Deployment on Fireworks AI platform + +", + "$.messages[5].content[2].type": "text", + "$.messages[5].content[3].text": "Title: Managing default deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: firectl update deployed-model --default + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: firectl update deployed-model --default + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: To delete a default deployment you must delete all other deployments for the same model first, or designate a different deployed model as the default as described above. This is to ensure that querying by model name will always route to an unambiguous default deployment as long as deployments for the model exist. + +", + "$.messages[5].content[3].type": "text", + "$.messages[5].content[4].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[5].content[4].type": "text", + "$.messages[5].content[5].text": "Title: Deployment Command +Link: https://docs.fireworks.ai/evaluators/examples/gcp_cloud_run_deployment_example +Content: It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: Run the deployment command: Command Explanation: --id my-dummy-gcp-evaluator : A unique ID for your evaluator on the Fireworks AI platform. --target gcp-cloud-run : Specifies deployment to GCP Cloud Run. --function-ref dummy_rewards.hello_world_reward : The Python import path to your reward function. If dummy_rewards.py is in the current directory, this reference works. --gcp-auth-mode api-key : Configures the Cloud Run service with API key authentication. reward-kit will generate a key, store it in GCP Secret Manager, and configure the service. The key is also saved to your local rewardkit.yaml under evaluator_endpoint_keys . This is the default if not specified. --verbose : Shows detailed output, including gcloud commands being executed. --force : (Optional) If an evaluator with the same --id already exists, this flag will delete the existing one before creating the new one. + +It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: Run the deployment command: Command Explanation: --id my-dummy-gcp-evaluator : A unique ID for your evaluator on the Fireworks AI platform. --target gcp-cloud-run : Specifies deployment to GCP Cloud Run. --function-ref dummy_rewards.hello_world_reward : The Python import path to your reward function. If dummy_rewards.py is in the current directory, this reference works. --gcp-auth-mode api-key : Configures the Cloud Run service with API key authentication. reward-kit will generate a key, store it in GCP Secret Manager, and configure the service. The key is also saved to your local rewardkit.yaml under evaluator_endpoint_keys . This is the default if not specified. --verbose : Shows detailed output, including gcloud commands being executed. --force : (Optional) If an evaluator with the same --id already exists, this flag will delete the existing one before creating the new one. + +It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: source .venv/bin/activate + +", + "$.messages[5].content[5].type": "text", + "$.messages[5].content[6].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[5].content[6].type": "text", + "$.messages[5].content[7].text": "Title: How do I close my Fireworks.ai account? +Link: https://docs.fireworks.ai/faq-new/account-access/how-do-i-close-my-fireworksai-account +Content: To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +How do I close my Fireworks.ai account? +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +", + "$.messages[5].content[7].type": "text", + "$.messages[5].content[8].text": "Title: Delete Deployment +Link: https://docs.fireworks.ai/api-reference/delete-deployment +Content: + + + +REST Endpoint DELETE /v1/accounts/{account_id}/deployments/{deployment_id} + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" +deployment_id: + string + Description: The Deployment Id + Example: "" + +query params: +hard: + boolean + Description: If true, this will perform a hard deletion. + Example: true +ignoreChecks: + boolean + Description: If true, this will ignore checks and force the deletion of a deployment that is currently +deployed and is in use. + Example: true + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {} + +", + "$.messages[5].content[8].type": "text", + "$.messages[5].content[9].text": "Title: Deploying a fine-tuned model using an on-demand deployment +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-models +Content: Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +firectl +firectl create dataset /path/to/jsonl/file + +", + "$.messages[5].content[9].type": "text", + "$.messages[5].role": "tool", + "$.messages[5].tool_call_id": "functions.search:1", + "$.messages[6].content": "Let me search for more information about listing and managing existing deployments.", + "$.messages[6].role": "assistant", + "$.messages[6].tool_calls[0].function.arguments": "{"query": "list deployments firectl commands deployment management"}", + "$.messages[6].tool_calls[0].function.name": "search", + "$.messages[6].tool_calls[0].id": "functions.search:0", + "$.messages[6].tool_calls[0].type": "function", + "$.messages[7].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[7].content[0].type": "text", + "$.messages[7].content[1].text": "Title: List deployments +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/list-resources +Content: Prints all deployments in the account. firectl list deployments [flags] + +Prints all deployments in the account. firectl list deployments [flags] + +List deployed models +Prints all deployed models in an account. firectl list deployed-models [flags] + +", + "$.messages[7].content[1].type": "text", + "$.messages[7].content[2].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[7].content[2].type": "text", + "$.messages[7].content[3].text": "Title: Managing default deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: firectl get model + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: firectl get model + +Creating an on-demand deployment +Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: The state field should show READY. The deployment ID is the last part of the deployment name: accounts//deployments/. You can also list all deployments in your account at any time by running: firectl list deployments + +", + "$.messages[7].content[3].type": "text", + "$.messages[7].content[4].text": "Title: Does Fireworks support custom base models? +Link: https://docs.fireworks.ai/faq-new/models-inference/does-fireworks-support-custom-base-models +Content: Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Does Fireworks support custom base models? +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +", + "$.messages[7].content[4].type": "text", + "$.messages[7].content[5].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[7].content[5].type": "text", + "$.messages[7].content[6].text": "Title: List Deployments +Link: https://docs.fireworks.ai/api-reference/list-deployments +Content: + + + +REST Endpoint GET /v1/accounts/{account_id}/deployments + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" + +query params: +pageSize: + integer + Description: The maximum number of deployments to return. The maximum page_size is 200, +values above 200 will be coerced to 200. +If unspecified, the default is 50. + Example: 123 +pageToken: + string + Description: A page token, received from a previous ListDeployments call. Provide this +to retrieve the subsequent page. When paginating, all other parameters +provided to ListDeployments must match the call that provided the page +token. + Example: "" +filter: + string + Description: Only deployment satisfying the provided filter (if specified) will be +returned. See https://google.aip.dev/160 for the filter grammar. + Example: "" +orderBy: + string + Description: A comma-separated list of fields to order by. e.g. "foo,bar" +The default sort order is ascending. To specify a descending order for a +field, append a " desc" suffix. e.g. "foo desc,bar" +Subfields are specified with a "." character. e.g. "foo.bar" +If not specified, the default order is by "create_time". + Example: "" +showDeleted: + boolean + Description: If set, DELETED deployments will be included. + Example: true +readMask: + string + Description: The fields to be returned in the response. If empty or "*", all fields will be returned. + Example: "" + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {"deployments":[{"name":"","displayName":"","description":"","createTime":"2023-11-07T05:31:56Z","expireTime":"2023-11-07T05:31:56Z","purgeTime":"2023-11-07T05:31:56Z","deleteTime":"2023-11-07T05:31:56Z","state":"STATE_UNSPECIFIED","status":{"code":"OK","message":""},"minReplicaCount":123,"maxReplicaCount":123,"replicaCount":123,"autoscalingPolicy":{"scaleUpWindow":"","scaleDownWindow":"","scaleToZeroWindow":"","loadTargets":{}},"baseModel":"","acceleratorCount":123,"acceleratorType":"ACCELERATOR_TYPE_UNSPECIFIED","precision":"PRECISION_UNSPECIFIED","cluster":"","enableAddons":true,"draftTokenCount":123,"draftModel":"","ngramSpeculationLength":123,"numPeftDeviceCached":123,"deploymentTemplate":"","autoTune":{"longPrompt":true},"placement":{"region":"REGION_UNSPECIFIED","multiRegion":"MULTI_REGION_UNSPECIFIED","regions":["REGION_UNSPECIFIED"]},"region":"REGION_UNSPECIFIED","updateTime":"2023-11-07T05:31:56Z","disableDeploymentSizeValidation":true}],"nextPageToken":"","totalSize":123} + +", + "$.messages[7].content[6].type": "text", + "$.messages[7].content[7].text": "Title: Deploy Command +Link: https://docs.fireworks.ai/evaluators/cli_reference/cli_overview +Content: The deploy command deploys a reward function as an evaluator on the Fireworks platform. + +The deploy command deploys a reward function as an evaluator on the Fireworks platform. + +Command Overview +The Reward Kit CLI supports the following main commands: run : Run a local evaluation pipeline using a Hydra configuration. preview : Preview evaluation results or re-evaluate generated outputs. deploy : Deploy a reward function as an evaluator. agent-eval : Run agent evaluations on task bundles. list : List existing evaluators (coming soon). delete : Delete an evaluator (coming soon). + +", + "$.messages[7].content[7].type": "text", + "$.messages[7].content[8].text": "Title: Deploying a fine-tuned model using an on-demand deployment +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-models +Content: Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Deploying a fine-tuned model serverlessly +Not all base models support serverless addons. Please check this list for the full list of serverless models that support LoRA add-ons. If your base model supports serverless addons, you can run the following command to load the fine-tuned model into a serverless deployment: firectl load-lora + +", + "$.messages[7].content[8].type": "text", + "$.messages[7].content[9].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. firectl create deployment [flags] + +Create a Deployment on Fireworks AI platform + +", + "$.messages[7].content[9].type": "text", + "$.messages[7].role": "tool", + "$.messages[7].tool_call_id": "functions.search:0", + "$.messages[8].content": "Based on my research, here's a comprehensive guide to managing deployments on Fireworks AI: + +## Overview of Deployment Types + +Fireworks AI offers several deployment options: + +1. **Serverless Models**: Pay-per-token, shared infrastructure +2. **On-demand Deployments**: Dedicated GPU instances billed by GPU-second +3. **Reserved Capacity**: Pre-purchased dedicated capacity for high-volume usage + +## Key Benefits of On-demand Deployments +- Predictable performance unaffected by other users +- No hard rate limits +- Cost-effective for high utilization +- Access to larger model selection +- Support for custom base models + +## Managing Deployments with firectl + +### Installation and Setup +First, install the Fireworks CLI tools: + +\`\`\`bash +pip install fireworks-ai +\`\`\` + +Set up your API key: +\`\`\`bash +export FIREWORKS_API_KEY="your-api-key-here" +\`\`\` + +### Basic Deployment Commands + +#### 1. List All Deployments +\`\`\`bash +firectl list deployments +\`\`\` + +#### 2. Create a New Deployment +\`\`\`bash +# Basic deployment +firectl create deployment + +# With custom configuration +firectl create deployment --min-replica-count 1 --max-replica-count 3 + +# Deploy a LoRA fine-tuned model +firectl create deployment "accounts/fireworks/models/" + +# Deploy with addons enabled (for multi-LoRA) +firectl create deployment --enable-addons +\`\`\` + +#### 3. Check Deployment Status +\`\`\`bash +firectl get deployment +\`\`\` + +#### 4. Delete a Deployment +\`\`\`bash +# Soft delete +firectl delete deployment + +# Force delete (ignores usage checks) +firectl delete deployment --ignore-checks +\`\`\` + +### Managing Default Deployments +When you have multiple deployments of the same model: + +\`\`\`bash +# Check which deployment is default +firectl get model + +# Update default deployment +firectl update deployed-model --default +\`\`\` + +## Using the Fireworks Build SDK (Python) + +The Fireworks Build SDK provides a higher-level interface: + +\`\`\`python +from fireworks import LLM + +# Auto-managed deployment +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +# Explicit on-demand deployment +llm = LLM(model="your-model-id", deployment_type="on-demand") + +# Make API calls +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Hello!"}] +) +\`\`\` + +## REST API Management + +### List Deployments via REST API +\`\`\`bash +curl -X GET \\ + "https://api.fireworks.ai/v1/accounts/{account_id}/deployments" \\ + -H "Authorization: Bearer $FIREWORKS_API_KEY" +\`\`\` + +### Delete Deployment via REST API +\`\`\`bash +curl -X DELETE \\ + "https://api.fireworks.ai/v1/accounts/{account_id}/deployments/{deployment_id}" \\ + -H "Authorization: Bearer $FIREWORKS_API_KEY" +\`\`\` + +## Deployment Workflow Examples + +### 1. Deploy a Fine-tuned Model +\`\`\`bash +# Single command deployment +firectl create deployment "accounts/fireworks/models/my-lora-model" + +# Wait for deployment to be ready +firectl get deployment +\`\`\` + +### 2. Multi-LoRA Deployment (Advanced) +\`\`\`bash +# Step 1: Create base deployment with addons +firectl create deployment "accounts/fireworks/models/base-model" --enable-addons + +# Step 2: Load LoRA models +firectl load-lora --deployment +firectl load-lora --deployment +\`\`\` + +### 3. Custom Model Deployment +\`\`\`bash +# First upload your custom model +# Then deploy it +firectl create deployment "accounts/{account_id}/models/custom-model-id" +\`\`\` + +## Best Practices + +1. **Monitor Costs**: On-demand deployments are billed by GPU-second +2. **Use Reserved Capacity**: For consistent high-volume usage +3. **Set Appropriate Scaling**: Configure min/max replica counts based on expected load +4. **Clean Up**: Delete unused deployments to avoid charges +5. **Check Status**: Always verify deployment state before using + +## Troubleshooting + +- **"Invalid ID" errors**: Ensure your account ID is properly configured +- **Deployment stuck**: Check resource quotas and account limits +- **Performance issues**: Verify deployment is in READY state and check replica counts + +Would you like me to dive deeper into any specific aspect of deployment management, such as scaling configurations, cost optimization, or advanced deployment patterns?", + "$.messages[8].role": "assistant", + "$.pid": 89723, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:27:14.528488", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "finished", + "$.eval_metadata.version": "v0.2.6.dev15+g6486dbd.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "At least one tool call was made", + "$.evaluation_result.score": 1, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "construct-do-safe-art-song", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.messages[2].content": "I'll help you learn about managing deployments on Fireworks! Let me search for comprehensive information about deployment management.", + "$.messages[2].role": "assistant", + "$.messages[2].tool_calls[0].function.arguments": "{"query": "deployment management Fireworks AI"}", + "$.messages[2].tool_calls[0].function.name": "search", + "$.messages[2].tool_calls[0].id": "functions.search:0", + "$.messages[2].tool_calls[0].type": "function", + "$.messages[3].content[0].text": "Title: Fireworks AI Developer Platform +Link: https://docs.fireworks.ai/getting-started/introduction +Content: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Start building with open source AI models + +", + "$.messages[3].content[0].type": "text", + "$.messages[3].content[1].text": "Title: How do I close my Fireworks.ai account? +Link: https://docs.fireworks.ai/faq-new/account-access/how-do-i-close-my-fireworksai-account +Content: To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +How do I close my Fireworks.ai account? +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +", + "$.messages[3].content[1].type": "text", + "$.messages[3].content[2].text": "Title: Do you host your deployments in the EU or Asia? +Link: https://docs.fireworks.ai/faq-new/support-general/do-you-host-your-deployments-in-the-eu-or-asia +Content: We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +Do you host your deployments in the EU or Asia? +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +", + "$.messages[3].content[2].type": "text", + "$.messages[3].content[3].text": "Title: Introducing the Fireworks Build SDK +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-introduction +Content: The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +", + "$.messages[3].content[3].type": "text", + "$.messages[3].content[4].text": "Title: On-demand deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +On-demand deployments +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +", + "$.messages[3].content[4].type": "text", + "$.messages[3].content[5].text": "Title: I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +Link: https://docs.fireworks.ai/faq-new/account-access/i-have-multiple-fireworks-accounts-when-i-try-to-login-with-google-on-fireworks +Content: If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +", + "$.messages[3].content[5].type": "text", + "$.messages[3].content[6].text": "Title: Introduction +Link: https://docs.fireworks.ai/api-reference/introduction +Content: Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Introduction +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +", + "$.messages[3].content[6].type": "text", + "$.messages[3].content[7].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Create a Deployment on Fireworks AI platform + +Create a Deployment on Fireworks AI platform + +Creates a new deployment. + +", + "$.messages[3].content[7].type": "text", + "$.messages[3].content[8].text": "Title: How much does Fireworks cost? +Link: https://docs.fireworks.ai/faq-new/billing-pricing/how-much-does-fireworks-cost +Content: Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +How much does Fireworks cost? +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +", + "$.messages[3].content[8].type": "text", + "$.messages[3].content[9].text": "Title: Python (Fireworks) +Link: https://docs.fireworks.ai/getting-started/quickstart +Content: This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +Set up your developer environment & call the Fireworks API +This is the recommended way to get started : This tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: Sending the first API Request You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. Fireworks AI platform offers drop-in replacement with OpenAI Python Client. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $OPENAI_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %OPENAI_API_KEY% + +", + "$.messages[3].content[9].type": "text", + "$.messages[3].role": "tool", + "$.messages[3].tool_call_id": "functions.search:0", + "$.messages[4].content": "Let me search for more specific information about deployment management commands and workflows.", + "$.messages[4].role": "assistant", + "$.messages[4].tool_calls[0].function.arguments": "{"query": "firectl deployment management commands create delete list"}", + "$.messages[4].tool_calls[0].function.name": "search", + "$.messages[4].tool_calls[0].id": "functions.search:1", + "$.messages[4].tool_calls[0].type": "function", + "$.messages[5].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[5].content[0].type": "text", + "$.messages[5].content[1].text": "Title: Delete a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/delete-resources +Content: Deletes an model deployment. firectl delete deployment [flags] + +Deletes an model deployment. firectl delete deployment [flags] + +Example +firectl delete deployment my-deployment + +", + "$.messages[5].content[1].type": "text", + "$.messages[5].content[2].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. + +", + "$.messages[5].content[2].type": "text", + "$.messages[5].content[3].text": "Title: Creating an on-demand deployment +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: The state field should show READY. The deployment ID is the last part of the deployment name: accounts//deployments/. You can also list all deployments in your account at any time by running: firectl list deployments + +Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: The state field should show READY. The deployment ID is the last part of the deployment name: accounts//deployments/. You can also list all deployments in your account at any time by running: firectl list deployments + +Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: firectl get deployment + +", + "$.messages[5].content[3].type": "text", + "$.messages[5].content[4].text": "Title: Delete Deployment +Link: https://docs.fireworks.ai/api-reference/delete-deployment +Content: + + + +REST Endpoint DELETE /v1/accounts/{account_id}/deployments/{deployment_id} + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" +deployment_id: + string + Description: The Deployment Id + Example: "" + +query params: +hard: + boolean + Description: If true, this will perform a hard deletion. + Example: true +ignoreChecks: + boolean + Description: If true, this will ignore checks and force the deletion of a deployment that is currently +deployed and is in use. + Example: true + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {} + +", + "$.messages[5].content[4].type": "text", + "$.messages[5].content[5].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[5].content[5].type": "text", + "$.messages[5].content[6].text": "Title: Deployment Command +Link: https://docs.fireworks.ai/evaluators/examples/gcp_cloud_run_deployment_example +Content: It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: Run the deployment command: Command Explanation: --id my-dummy-gcp-evaluator : A unique ID for your evaluator on the Fireworks AI platform. --target gcp-cloud-run : Specifies deployment to GCP Cloud Run. --function-ref dummy_rewards.hello_world_reward : The Python import path to your reward function. If dummy_rewards.py is in the current directory, this reference works. --gcp-auth-mode api-key : Configures the Cloud Run service with API key authentication. reward-kit will generate a key, store it in GCP Secret Manager, and configure the service. The key is also saved to your local rewardkit.yaml under evaluator_endpoint_keys . This is the default if not specified. --verbose : Shows detailed output, including gcloud commands being executed. --force : (Optional) If an evaluator with the same --id already exists, this flag will delete the existing one before creating the new one. + +It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: Run the deployment command: Command Explanation: --id my-dummy-gcp-evaluator : A unique ID for your evaluator on the Fireworks AI platform. --target gcp-cloud-run : Specifies deployment to GCP Cloud Run. --function-ref dummy_rewards.hello_world_reward : The Python import path to your reward function. If dummy_rewards.py is in the current directory, this reference works. --gcp-auth-mode api-key : Configures the Cloud Run service with API key authentication. reward-kit will generate a key, store it in GCP Secret Manager, and configure the service. The key is also saved to your local rewardkit.yaml under evaluator_endpoint_keys . This is the default if not specified. --verbose : Shows detailed output, including gcloud commands being executed. --force : (Optional) If an evaluator with the same --id already exists, this flag will delete the existing one before creating the new one. + +It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: source .venv/bin/activate + +", + "$.messages[5].content[6].type": "text", + "$.messages[5].content[7].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[5].content[7].type": "text", + "$.messages[5].content[8].text": "Title: Deploying a fine-tuned model using an on-demand deployment +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-models +Content: Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +firectl +firectl create dataset /path/to/jsonl/file + +", + "$.messages[5].content[8].type": "text", + "$.messages[5].content[9].text": "Title: Deleting Stored Responses +Link: https://docs.fireworks.ai/guides/response-api +Content: When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. from fireworks import LLM +import requests +import os + +llm = LLM(model="qwen3-235b-a22b", deployment_type="serverless") + +# Create a response +response = llm.responses.create( + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. from fireworks import LLM +import requests +import os + +llm = LLM(model="qwen3-235b-a22b", deployment_type="serverless") + +# Create a response +response = llm.responses.create( + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. import os +from openai import OpenAI +import requests + +client = OpenAI( + base_url="https://api.fireworks.ai/inference/v1", + api_key=os.getenv("FIREWORKS_API_KEY", "YOUR_FIREWORKS_API_KEY_HERE") +) + +# Create a response +response = client.responses.create( + model="accounts/fireworks/models/qwen3-235b-a22b", + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +", + "$.messages[5].content[9].type": "text", + "$.messages[5].role": "tool", + "$.messages[5].tool_call_id": "functions.search:1", + "$.messages[6].content": "Now let me search for information about listing and managing deployments to give you a complete picture.", + "$.messages[6].role": "assistant", + "$.messages[6].tool_calls[0].function.arguments": "{"query": "list deployments get deployment status firectl commands"}", + "$.messages[6].tool_calls[0].function.name": "search", + "$.messages[6].tool_calls[0].id": "functions.search:1", + "$.messages[6].tool_calls[0].type": "function", + "$.messages[7].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[7].content[0].type": "text", + "$.messages[7].content[1].text": "Title: List deployments +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/list-resources +Content: Prints all deployments in the account. firectl list deployments [flags] + +Prints all deployments in the account. firectl list deployments [flags] + +List deployed models +Prints all deployed models in an account. firectl list deployed-models [flags] + +", + "$.messages[7].content[1].type": "text", + "$.messages[7].content[2].text": "Title: Creating an on-demand deployment +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: The state field should show READY. The deployment ID is the last part of the deployment name: accounts//deployments/. You can also list all deployments in your account at any time by running: firectl list deployments + +Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: The state field should show READY. The deployment ID is the last part of the deployment name: accounts//deployments/. You can also list all deployments in your account at any time by running: firectl list deployments + +Publishing a deployed model +By default, models can only be queried by the account that owns them. To make a deployment public so anyone with a valid Fireworks API key can query it, update the deployed model with the --public flag. Find the Deployed Model ID Every model running on a deployment receives a unique deployed model ID. You can find this ID if you know the model name and deployment name using the following command: firectl list deployed-models --filter 'model="" AND deployment=""' + +", + "$.messages[7].content[2].type": "text", + "$.messages[7].content[3].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[7].content[3].type": "text", + "$.messages[7].content[4].text": "Title: firectl +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-vlm +Content: # Create a deployment for your fine-tuned VLM +firectl create deployment my-custom-vlm + +# Check deployment status +firectl get deployment accounts/your-account/deployment/deployment-id + +# Create a deployment for your fine-tuned VLM +firectl create deployment my-custom-vlm + +# Check deployment status +firectl get deployment accounts/your-account/deployment/deployment-id + +Try with an Example Dataset +To get a feel for how VLM fine-tuning works, you can use an example vision dataset: This is a classification dataset that contains images of food with tags for reasoning. Once downloaded, you can upload this dataset using the instructions in the next step. Upload your VLM dataset Upload your prepared JSONL dataset to Fireworks for training: Navigate to the Datasets tab in the Fireworks console, click "Create Dataset", and upload your JSONL file through the wizard. For larger datasets (>500MB), use as it handles large uploads more reliably than the web interface. For enhanced data control and security, we also support bring your own bucket (BYOB) configurations. See our External GCS Bucket Integration guide for setup details. Launch VLM fine-tuning job Create a supervised fine-tuning job for your VLM: For additional parameters like learning rates, evaluation datasets, and batch sizes, see Additional SFT job settings. Navigate to the Fine-tuning tab in the Fireworks console Click "Create Fine-tuning Job" Select your VLM base model (Qwen 2.5 VL) Choose your uploaded dataset Configure training parameters Launch the job VLM fine-tuning jobs typically take longer than text-only models due to the additional image processing. Expect training times of several hours depending on dataset size and model complexity. Monitor training progress Track your VLM fine-tuning job in the Fireworks console. Monitor key metrics: Training loss : Should generally decrease over time Evaluation loss : Monitor for overfitting if using evaluation dataset Training progress : Epochs completed and estimated time remaining Your VLM fine-tuning job is complete when the status shows COMPLETED and your custom model is ready for deployment. Deploy your fine-tuned VLM Once training is complete, deploy your custom VLM: # Create a deployment for your fine-tuned VLM +firectl create deployment my-custom-vlm + +# Check deployment status +firectl get deployment accounts/your-account/deployment/deployment-id + +", + "$.messages[7].content[4].type": "text", + "$.messages[7].content[5].text": "Title: firectl +Link: https://docs.fireworks.ai/guides/batch-inference +Content: # Get job status +firectl get batch-inference-job my-batch-job + +# List all batch jobs +firectl list batch-inference-jobs + +# Get job status +firectl get batch-inference-job my-batch-job + +# List all batch jobs +firectl list batch-inference-jobs + +firectl create dataset batch-input-dataset ./batch_input_data.jsonl + +", + "$.messages[7].content[5].type": "text", + "$.messages[7].content[6].text": "Title: Does Fireworks support custom base models? +Link: https://docs.fireworks.ai/faq-new/models-inference/does-fireworks-support-custom-base-models +Content: Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Does Fireworks support custom base models? +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +", + "$.messages[7].content[6].type": "text", + "$.messages[7].content[7].text": "Title: List Deployments +Link: https://docs.fireworks.ai/api-reference/list-deployments +Content: + + + +REST Endpoint GET /v1/accounts/{account_id}/deployments + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" + +query params: +pageSize: + integer + Description: The maximum number of deployments to return. The maximum page_size is 200, +values above 200 will be coerced to 200. +If unspecified, the default is 50. + Example: 123 +pageToken: + string + Description: A page token, received from a previous ListDeployments call. Provide this +to retrieve the subsequent page. When paginating, all other parameters +provided to ListDeployments must match the call that provided the page +token. + Example: "" +filter: + string + Description: Only deployment satisfying the provided filter (if specified) will be +returned. See https://google.aip.dev/160 for the filter grammar. + Example: "" +orderBy: + string + Description: A comma-separated list of fields to order by. e.g. "foo,bar" +The default sort order is ascending. To specify a descending order for a +field, append a " desc" suffix. e.g. "foo desc,bar" +Subfields are specified with a "." character. e.g. "foo.bar" +If not specified, the default order is by "create_time". + Example: "" +showDeleted: + boolean + Description: If set, DELETED deployments will be included. + Example: true +readMask: + string + Description: The fields to be returned in the response. If empty or "*", all fields will be returned. + Example: "" + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {"deployments":[{"name":"","displayName":"","description":"","createTime":"2023-11-07T05:31:56Z","expireTime":"2023-11-07T05:31:56Z","purgeTime":"2023-11-07T05:31:56Z","deleteTime":"2023-11-07T05:31:56Z","state":"STATE_UNSPECIFIED","status":{"code":"OK","message":""},"minReplicaCount":123,"maxReplicaCount":123,"replicaCount":123,"autoscalingPolicy":{"scaleUpWindow":"","scaleDownWindow":"","scaleToZeroWindow":"","loadTargets":{}},"baseModel":"","acceleratorCount":123,"acceleratorType":"ACCELERATOR_TYPE_UNSPECIFIED","precision":"PRECISION_UNSPECIFIED","cluster":"","enableAddons":true,"draftTokenCount":123,"draftModel":"","ngramSpeculationLength":123,"numPeftDeviceCached":123,"deploymentTemplate":"","autoTune":{"longPrompt":true},"placement":{"region":"REGION_UNSPECIFIED","multiRegion":"MULTI_REGION_UNSPECIFIED","regions":["REGION_UNSPECIFIED"]},"region":"REGION_UNSPECIFIED","updateTime":"2023-11-07T05:31:56Z","disableDeploymentSizeValidation":true}],"nextPageToken":"","totalSize":123} + +", + "$.messages[7].content[7].type": "text", + "$.messages[7].content[8].text": "Title: Deploy Command +Link: https://docs.fireworks.ai/evaluators/cli_reference/cli_overview +Content: The deploy command deploys a reward function as an evaluator on the Fireworks platform. + +The deploy command deploys a reward function as an evaluator on the Fireworks platform. + +Command Overview +The Reward Kit CLI supports the following main commands: run : Run a local evaluation pipeline using a Hydra configuration. preview : Preview evaluation results or re-evaluate generated outputs. deploy : Deploy a reward function as an evaluator. agent-eval : Run agent evaluations on task bundles. list : List existing evaluators (coming soon). delete : Delete an evaluator (coming soon). + +", + "$.messages[7].content[8].type": "text", + "$.messages[7].content[9].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[7].content[9].type": "text", + "$.messages[7].role": "tool", + "$.messages[7].tool_call_id": "functions.search:1", + "$.messages[8].content": "Perfect! Now I have comprehensive information about managing deployments on Fireworks. Let me provide you with a complete guide. + +# Managing Deployments on Fireworks: A Complete Guide + +Fireworks offers several ways to manage deployments, from command-line tools to REST APIs. Here's everything you need to know: + +## Types of Deployments + +### 1. On-Demand Deployments +- **Billed by GPU-second** (not per-token like serverless) +- **Dedicated resources** - no interference from other users +- **No hard rate limits** (subject to deployment capacity) +- **Access to more models** including custom ones +- **Better for high utilization** scenarios + +### 2. Serverless Deployments +- **Billed per-token** +- **Shared resources** with other users +- **Rate limits apply** +- **Instant availability** + +## Management Tools + +### Option 1: Fireworks Build SDK (Recommended) +\`\`\`python +from fireworks import LLM + +# Auto-deployment - SDK handles everything +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +# Manual deployment management +llm = LLM(model="your-model", deployment_type="on-demand") +\`\`\` + +### Option 2: firectl CLI Tool + +#### Installation +\`\`\`bash +# Install firectl CLI +curl -s https://fireworks.ai/install.sh | bash +\`\`\` + +#### Basic Commands + +**List all deployments:** +\`\`\`bash +firectl list deployments +\`\`\` + +**Create a deployment:** +\`\`\`bash +# From model library +firectl create deployment accounts/fireworks/models/llama-v3p1-8b-instruct --wait + +# Custom model +firectl create deployment accounts/your-account-id/models/your-custom-model --wait + +# LoRA fine-tuned model (single command) +firectl create deployment accounts/fireworks/models/your-lora-model-id +\`\`\` + +**Get deployment status:** +\`\`\`bash +firectl get deployment +\`\`\` + +**Delete a deployment:** +\`\`\`bash +firectl delete deployment +\`\`\` + +**List deployed models:** +\`\`\`bash +firectl list deployed-models +\`\`\` + +### Option 3: REST API + +#### List Deployments +\`\`\`bash +curl -X GET \\ + "https://api.fireworks.ai/v1/accounts/{account_id}/deployments" \\ + -H "Authorization: Bearer $FIREWORKS_API_KEY" +\`\`\` + +#### Create Deployment +\`\`\`bash +curl -X POST \\ + "https://api.fireworks.ai/v1/accounts/{account_id}/deployments" \\ + -H "Authorization: Bearer $FIREWORKS_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{ + "deployment": { + "displayName": "my-deployment", + "baseModel": "accounts/fireworks/models/llama-v3p1-8b-instruct" + } + }' +\`\`\` + +#### Delete Deployment +\`\`\`bash +curl -X DELETE \\ + "https://api.fireworks.ai/v1/accounts/{account_id}/deployments/{deployment_id}" \\ + -H "Authorization: Bearer $FIREWORKS_API_KEY" +\`\`\` + +## Deployment Workflow + +### 1. Choose Your Model +\`\`\`bash +# Check available models +firectl list models +\`\`\` + +### 2. Create Deployment +\`\`\`bash +# Basic deployment +firectl create deployment accounts/fireworks/models/llama-v3p1-8b-instruct --wait + +# With custom configuration +firectl create deployment accounts/fireworks/models/llama-v3p1-8b-instruct \\ + --min-replica-count 1 \\ + --max-replica-count 3 \\ + --wait +\`\`\` + +### 3. Monitor Status +\`\`\`bash +# Get deployment details +firectl get deployment + +# Watch logs +firectl logs deployment +\`\`\` + +### 4. Make Public (Optional) +\`\`\`bash +# Make deployment public +firectl update deployment --public +\`\`\` + +### 5. Clean Up +\`\`\`bash +# Delete when done +firectl delete deployment +\`\`\` + +## Advanced Features + +### Reserved Capacity +For high-volume usage, consider reserved capacity: +- More reliable capacity +- Higher GPU quotas +- Cost savings for consistent usage +- Contact Fireworks for enterprise pricing + +### Multi-LoRA Deployments +\`\`\`bash +# Create base deployment with addons +firectl create deployment accounts/fireworks/models/base-model --enable-addons --wait + +# Load LoRA models +firectl load-lora lora-model-1 --deployment +firectl load-lora lora-model-2 --deployment +\`\`\` + +### Custom Models +\`\`\`bash +# Upload custom model first +firectl upload model /path/to/model + +# Deploy custom model +firectl create deployment accounts/your-account-id/models/custom-model-name +\`\`\` + +## Best Practices + +1. **Use \`--wait\` flag** for important deployments to ensure they're ready +2. **Monitor costs** with \`firectl list deployments\` to see active deployments +3. **Set up alerts** for deployment health +4. **Use reserved capacity** for production workloads +5. **Clean up unused deployments** to avoid charges +6. **Test deployments** before making them public + +## Troubleshooting + +**"Invalid ID" errors:** +- Ensure your account ID is properly configured +- Check your API key and account settings + +**Deployment stuck:** +- Check resource availability +- Verify model compatibility +- Review deployment logs + +**Performance issues:** +- Consider scaling up replicas +- Check if reserved capacity is available +- Monitor GPU utilization + +## Cost Management +- **On-demand**: Pay per GPU-second used +- **Serverless**: Pay per token processed +- **Reserved capacity**: Fixed monthly cost for guaranteed capacity +- Use \`firectl list deployments\` to track active deployments and costs + +This comprehensive guide should give you everything you need to effectively manage deployments on Fireworks!", + "$.messages[8].role": "assistant", + "$.pid": 92995, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:38:53.968430", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "finished", + "$.eval_metadata.version": "v0.2.6.dev17+gf8398de.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "At least one tool call was made", + "$.evaluation_result.score": 1, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "have-disassemble-simple-final-experience", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[10].content": "Based on my research, I can now provide you with a comprehensive guide on managing deployments on Fireworks. Let me break this down into practical sections with commands and examples. + +## Managing Deployments on Fireworks: A Complete Guide + +Fireworks offers three deployment types with different management approaches: + +1. **Serverless**: Pre-configured, pay-per-token, automatic scaling +2. **On-demand**: Private GPUs, pay-per-second, manual/automatic scaling +3. **Enterprise Reserved**: Custom hardware, tailored setup, SLAs + +### 1. Basic Deployment Management Commands + +#### Using firectl CLI (Recommended) + +**List all deployments:** +\`\`\`bash +firectl list deployments +\`\`\` + +**Create a deployment:** +\`\`\`bash +# Basic deployment +firectl create deployment --wait + +# With custom configuration +firectl create deployment \\ + --min-replica-count 0 \\ + --max-replica-count 3 \\ + --wait +\`\`\` + +**Delete a deployment:** +\`\`\`bash +firectl delete deployment +\`\`\` + +**Update deployment configuration:** +\`\`\`bash +firectl update deployment \\ + --min-replica-count 2 \\ + --max-replica-count 5 +\`\`\` + +#### Using REST API + +**List deployments:** +\`\`\`bash +curl -X GET \\ + "https://api.fireworks.ai/v1/accounts/{account_id}/deployments" \\ + -H "Authorization: Bearer $FIREWORKS_API_KEY" +\`\`\` + +**Delete deployment via API:** +\`\`\`bash +curl -X DELETE \\ + "https://api.fireworks.ai/v1/accounts/{account_id}/deployments/{deployment_id}" \\ + -H "Authorization: Bearer $FIREWORKS_API_KEY" +\`\`\` + +### 2. Python SDK Management + +#### Installation and Setup +\`\`\`bash +pip install --upgrade fireworks-ai +export FIREWORKS_API_KEY="your-api-key" +\`\`\` + +#### Basic Usage +\`\`\`python +from fireworks import LLM + +# Serverless deployment (automatic) +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="serverless") + +# On-demand deployment with custom ID +llm = LLM( + model="llama4-maverick-instruct-basic", + deployment_type="on-demand", + id="my-custom-deployment" +) +llm.apply() # Required for on-demand deployments + +# Auto-selection (serverless if available, otherwise on-demand) +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") +\`\`\` + +#### Fine-tuned Model Deployment +\`\`\`python +# Single LoRA deployment (recommended) +lora_llm = LLM( + model="accounts/your-account/models/your-lora-model", + deployment_type="on-demand" +) +lora_llm.apply() + +# Multi-LoRA deployment +base_llm = LLM( + model="accounts/fireworks/models/llama4-maverick-instruct-basic", + deployment_type="on-demand-lora", + id="multi-lora-deployment" +) +base_llm.apply() +\`\`\` + +### 3. Deployment Configuration Options + +#### Scaling Configuration +\`\`\`bash +# Scale between 0-3 replicas based on demand +firectl create deployment \\ + --min-replica-count 0 \\ + --max-replica-count 3 + +# Fixed scaling (always have 2 replicas running) +firectl create deployment \\ + --min-replica-count 2 \\ + --max-replica-count 2 +\`\`\` + +#### Advanced Configuration +\`\`\`bash +# Custom model deployment +firectl create deployment accounts/your-account/models/custom-model \\ + --min-replica-count 0 \\ + --max-replica-count 5 \\ + --accelerator-type A100 \\ + --accelerator-count 2 +\`\`\` + +### 4. Cost Management Best Practices + +#### Automatic Cost Optimization +- **Scale-to-zero**: Deployments automatically scale to 0 when unused +- **No charges when scaled down**: Only pay for active GPU time +- **Autoscaling**: Automatically adjusts based on traffic + +#### Manual Cost Control +\`\`\`python +# Monitor deployment costs +import requests + +headers = {"Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}"} +response = requests.get( + "https://api.fireworks.ai/v1/accounts/{account_id}/deployments", + headers=headers +) + +# Delete unused deployments +for deployment in response.json()['deployments']: + if deployment['state'] == 'READY' and deployment['replicaCount'] == 0: + # Consider deleting if unused + pass +\`\`\` + +### 5. Monitoring and Troubleshooting + +#### Check Deployment Status +\`\`\`bash +# Get deployment details +firectl get deployment + +# Check deployed models +firectl list deployed-models +\`\`\` + +#### Web Dashboard +Track deployments at: \`https://app.fireworks.ai/dashboard/deployments\` + +### 6. Reserved Capacity (Enterprise) + +For high-volume usage, consider reserved capacity: +- More reliable capacity +- Higher GPU quotas +- Better pricing for consistent usage +- Contact: inquiries@fireworks.ai + +### 7. Common Management Workflows + +#### Daily Workflow Example +\`\`\`python +# Morning: Check active deployments +import subprocess +result = subprocess.run(['firectl', 'list', 'deployments'], + capture_output=True, text=True) +print(result.stdout) + +# Scale up for expected traffic +subprocess.run(['firectl', 'update', 'deployment', 'my-deployment', + '--min-replica-count', '2']) + +# Evening: Scale down +subprocess.run(['firectl', 'update', 'deployment', 'my-deployment', + '--min-replica-count', '0']) +\`\`\` + +#### Automated Cleanup Script +\`\`\`bash +#!/bin/bash +# cleanup_deployments.sh + +# List all deployments +deployments=$(firectl list deployments --format json | jq -r '.deployments[].name') + +# Delete deployments older than 7 days with 0 replicas +for deployment in $deployments; do + firectl delete deployment $deployment --ignore-checks +done +\`\`\` + +### 8. Important Considerations + +- **Billing**: On-demand deployments charge by GPU-second, even when idle +- **Rate limits**: Serverless has rate limits, on-demand does not +- **Cold starts**: Scaling from 0 may have brief delays +- **Monitoring**: Regularly check deployment status and costs +- **Cleanup**: Always delete unused deployments to avoid charges + +This comprehensive guide covers all aspects of deployment management on Fireworks. Start with serverless for simple use cases, move to on-demand for production workloads, and consider reserved capacity for enterprise needs.", + "$.messages[10].role": "assistant", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.messages[2].content": "I'll help you learn about managing deployments on Fireworks. Let me search for comprehensive information about deployment management.", + "$.messages[2].role": "assistant", + "$.messages[2].tool_calls[0].function.arguments": "{"query": "deployment management Fireworks AI"}", + "$.messages[2].tool_calls[0].function.name": "search", + "$.messages[2].tool_calls[0].id": "functions.search:0", + "$.messages[2].tool_calls[0].type": "function", + "$.messages[3].content[0].text": "Title: Fireworks AI Developer Platform +Link: https://docs.fireworks.ai/getting-started/introduction +Content: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Start building with open source AI models + +", + "$.messages[3].content[0].type": "text", + "$.messages[3].content[1].text": "Title: How do I close my Fireworks.ai account? +Link: https://docs.fireworks.ai/faq-new/account-access/how-do-i-close-my-fireworksai-account +Content: To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +How do I close my Fireworks.ai account? +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +", + "$.messages[3].content[1].type": "text", + "$.messages[3].content[2].text": "Title: Do you host your deployments in the EU or Asia? +Link: https://docs.fireworks.ai/faq-new/support-general/do-you-host-your-deployments-in-the-eu-or-asia +Content: We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +Do you host your deployments in the EU or Asia? +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +", + "$.messages[3].content[2].type": "text", + "$.messages[3].content[3].text": "Title: Introducing the Fireworks Build SDK +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-introduction +Content: The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +", + "$.messages[3].content[3].type": "text", + "$.messages[3].content[4].text": "Title: On-demand deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +On-demand deployments +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +", + "$.messages[3].content[4].type": "text", + "$.messages[3].content[5].text": "Title: I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +Link: https://docs.fireworks.ai/faq-new/account-access/i-have-multiple-fireworks-accounts-when-i-try-to-login-with-google-on-fireworks +Content: If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +", + "$.messages[3].content[5].type": "text", + "$.messages[3].content[6].text": "Title: Introduction +Link: https://docs.fireworks.ai/api-reference/introduction +Content: Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Introduction +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +", + "$.messages[3].content[6].type": "text", + "$.messages[3].content[7].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Create a Deployment on Fireworks AI platform + +Create a Deployment on Fireworks AI platform + +Creates a new deployment. + +", + "$.messages[3].content[7].type": "text", + "$.messages[3].content[8].text": "Title: How much does Fireworks cost? +Link: https://docs.fireworks.ai/faq-new/billing-pricing/how-much-does-fireworks-cost +Content: Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +How much does Fireworks cost? +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +", + "$.messages[3].content[8].type": "text", + "$.messages[3].content[9].text": "Title: Python (Fireworks) +Link: https://docs.fireworks.ai/getting-started/quickstart +Content: This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +Set up your developer environment & call the Fireworks API +This is the recommended way to get started : This tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: Sending the first API Request You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. Fireworks AI platform offers drop-in replacement with OpenAI Python Client. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $OPENAI_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %OPENAI_API_KEY% + +", + "$.messages[3].content[9].type": "text", + "$.messages[3].role": "tool", + "$.messages[3].tool_call_id": "functions.search:0", + "$.messages[4].content": "Let me search for more specific information about deployment management commands and workflows.", + "$.messages[4].role": "assistant", + "$.messages[4].tool_calls[0].function.arguments": "{"query": "firectl deployment management commands create delete list update"}", + "$.messages[4].tool_calls[0].function.name": "search", + "$.messages[4].tool_calls[0].id": "functions.search:1", + "$.messages[4].tool_calls[0].type": "function", + "$.messages[5].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[5].content[0].type": "text", + "$.messages[5].content[1].text": "Title: Delete a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/delete-resources +Content: Deletes an model deployment. firectl delete deployment [flags] + +Deletes an model deployment. firectl delete deployment [flags] + +Example +firectl delete deployment my-deployment + +", + "$.messages[5].content[1].type": "text", + "$.messages[5].content[2].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. + +", + "$.messages[5].content[2].type": "text", + "$.messages[5].content[3].text": "Title: Creating an on-demand deployment +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: The state field should show READY. The deployment ID is the last part of the deployment name: accounts//deployments/. You can also list all deployments in your account at any time by running: firectl list deployments + +Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: The state field should show READY. The deployment ID is the last part of the deployment name: accounts//deployments/. You can also list all deployments in your account at any time by running: firectl list deployments + +Publishing a deployed model +By default, models can only be queried by the account that owns them. To make a deployment public so anyone with a valid Fireworks API key can query it, update the deployed model with the --public flag. Find the Deployed Model ID Every model running on a deployment receives a unique deployed model ID. You can find this ID if you know the model name and deployment name using the following command: firectl list deployed-models --filter 'model="" AND deployment=""' + +", + "$.messages[5].content[3].type": "text", + "$.messages[5].content[4].text": "Title: Delete Deployment +Link: https://docs.fireworks.ai/api-reference/delete-deployment +Content: + + + +REST Endpoint DELETE /v1/accounts/{account_id}/deployments/{deployment_id} + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" +deployment_id: + string + Description: The Deployment Id + Example: "" + +query params: +hard: + boolean + Description: If true, this will perform a hard deletion. + Example: true +ignoreChecks: + boolean + Description: If true, this will ignore checks and force the deletion of a deployment that is currently +deployed and is in use. + Example: true + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {} + +", + "$.messages[5].content[4].type": "text", + "$.messages[5].content[5].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[5].content[5].type": "text", + "$.messages[5].content[6].text": "Title: Deployment Command +Link: https://docs.fireworks.ai/evaluators/examples/gcp_cloud_run_deployment_example +Content: It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: Run the deployment command: Command Explanation: --id my-dummy-gcp-evaluator : A unique ID for your evaluator on the Fireworks AI platform. --target gcp-cloud-run : Specifies deployment to GCP Cloud Run. --function-ref dummy_rewards.hello_world_reward : The Python import path to your reward function. If dummy_rewards.py is in the current directory, this reference works. --gcp-auth-mode api-key : Configures the Cloud Run service with API key authentication. reward-kit will generate a key, store it in GCP Secret Manager, and configure the service. The key is also saved to your local rewardkit.yaml under evaluator_endpoint_keys . This is the default if not specified. --verbose : Shows detailed output, including gcloud commands being executed. --force : (Optional) If an evaluator with the same --id already exists, this flag will delete the existing one before creating the new one. + +It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: Run the deployment command: Command Explanation: --id my-dummy-gcp-evaluator : A unique ID for your evaluator on the Fireworks AI platform. --target gcp-cloud-run : Specifies deployment to GCP Cloud Run. --function-ref dummy_rewards.hello_world_reward : The Python import path to your reward function. If dummy_rewards.py is in the current directory, this reference works. --gcp-auth-mode api-key : Configures the Cloud Run service with API key authentication. reward-kit will generate a key, store it in GCP Secret Manager, and configure the service. The key is also saved to your local rewardkit.yaml under evaluator_endpoint_keys . This is the default if not specified. --verbose : Shows detailed output, including gcloud commands being executed. --force : (Optional) If an evaluator with the same --id already exists, this flag will delete the existing one before creating the new one. + +It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: source .venv/bin/activate + +", + "$.messages[5].content[6].type": "text", + "$.messages[5].content[7].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[5].content[7].type": "text", + "$.messages[5].content[8].text": "Title: Deploying a fine-tuned model using an on-demand deployment +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-models +Content: Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +firectl +firectl create dataset /path/to/jsonl/file + +", + "$.messages[5].content[8].type": "text", + "$.messages[5].content[9].text": "Title: Deleting Stored Responses +Link: https://docs.fireworks.ai/guides/response-api +Content: When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. from fireworks import LLM +import requests +import os + +llm = LLM(model="qwen3-235b-a22b", deployment_type="serverless") + +# Create a response +response = llm.responses.create( + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. from fireworks import LLM +import requests +import os + +llm = LLM(model="qwen3-235b-a22b", deployment_type="serverless") + +# Create a response +response = llm.responses.create( + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. import os +from openai import OpenAI +import requests + +client = OpenAI( + base_url="https://api.fireworks.ai/inference/v1", + api_key=os.getenv("FIREWORKS_API_KEY", "YOUR_FIREWORKS_API_KEY_HERE") +) + +# Create a response +response = client.responses.create( + model="accounts/fireworks/models/qwen3-235b-a22b", + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +", + "$.messages[5].content[9].type": "text", + "$.messages[5].role": "tool", + "$.messages[5].tool_call_id": "functions.search:1", + "$.messages[6].content": "Let me search for more information about listing and managing deployments.", + "$.messages[6].role": "assistant", + "$.messages[6].tool_calls[0].function.arguments": "{"query": "list deployments firectl commands deployment management"}", + "$.messages[6].tool_calls[0].function.name": "search", + "$.messages[6].tool_calls[0].id": "functions.search:2", + "$.messages[6].tool_calls[0].type": "function", + "$.messages[7].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[7].content[0].type": "text", + "$.messages[7].content[1].text": "Title: List deployments +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/list-resources +Content: Prints all deployments in the account. firectl list deployments [flags] + +Prints all deployments in the account. firectl list deployments [flags] + +List deployed models +Prints all deployed models in an account. firectl list deployed-models [flags] + +", + "$.messages[7].content[1].type": "text", + "$.messages[7].content[2].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[7].content[2].type": "text", + "$.messages[7].content[3].text": "Title: Managing default deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: firectl get model + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: firectl get model + +Creating an on-demand deployment +Choose a model Check out our model library to see a list of models available for on-demand deployment. You can also upload and use your own custom base model. Create a deployment To create a new deployment of a model from the model library, use: Or, for a custom base model you have uploaded to your account: This command will complete when the deployment is READY . To let it run asynchronously, remove the --wait flag. If your account has purchased reserved capacity and this deployment meets the reservation criteria, it will be counted against that reservation and not billed as an on-demand deployment. Verify the deployment is running You can verify the deployment is active by running: The state field should show READY. The deployment ID is the last part of the deployment name: accounts//deployments/. You can also list all deployments in your account at any time by running: firectl list deployments + +", + "$.messages[7].content[3].type": "text", + "$.messages[7].content[4].text": "Title: Does Fireworks support custom base models? +Link: https://docs.fireworks.ai/faq-new/models-inference/does-fireworks-support-custom-base-models +Content: Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Does Fireworks support custom base models? +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +", + "$.messages[7].content[4].type": "text", + "$.messages[7].content[5].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[7].content[5].type": "text", + "$.messages[7].content[6].text": "Title: List Deployments +Link: https://docs.fireworks.ai/api-reference/list-deployments +Content: + + + +REST Endpoint GET /v1/accounts/{account_id}/deployments + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" + +query params: +pageSize: + integer + Description: The maximum number of deployments to return. The maximum page_size is 200, +values above 200 will be coerced to 200. +If unspecified, the default is 50. + Example: 123 +pageToken: + string + Description: A page token, received from a previous ListDeployments call. Provide this +to retrieve the subsequent page. When paginating, all other parameters +provided to ListDeployments must match the call that provided the page +token. + Example: "" +filter: + string + Description: Only deployment satisfying the provided filter (if specified) will be +returned. See https://google.aip.dev/160 for the filter grammar. + Example: "" +orderBy: + string + Description: A comma-separated list of fields to order by. e.g. "foo,bar" +The default sort order is ascending. To specify a descending order for a +field, append a " desc" suffix. e.g. "foo desc,bar" +Subfields are specified with a "." character. e.g. "foo.bar" +If not specified, the default order is by "create_time". + Example: "" +showDeleted: + boolean + Description: If set, DELETED deployments will be included. + Example: true +readMask: + string + Description: The fields to be returned in the response. If empty or "*", all fields will be returned. + Example: "" + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {"deployments":[{"name":"","displayName":"","description":"","createTime":"2023-11-07T05:31:56Z","expireTime":"2023-11-07T05:31:56Z","purgeTime":"2023-11-07T05:31:56Z","deleteTime":"2023-11-07T05:31:56Z","state":"STATE_UNSPECIFIED","status":{"code":"OK","message":""},"minReplicaCount":123,"maxReplicaCount":123,"replicaCount":123,"autoscalingPolicy":{"scaleUpWindow":"","scaleDownWindow":"","scaleToZeroWindow":"","loadTargets":{}},"baseModel":"","acceleratorCount":123,"acceleratorType":"ACCELERATOR_TYPE_UNSPECIFIED","precision":"PRECISION_UNSPECIFIED","cluster":"","enableAddons":true,"draftTokenCount":123,"draftModel":"","ngramSpeculationLength":123,"numPeftDeviceCached":123,"deploymentTemplate":"","autoTune":{"longPrompt":true},"placement":{"region":"REGION_UNSPECIFIED","multiRegion":"MULTI_REGION_UNSPECIFIED","regions":["REGION_UNSPECIFIED"]},"region":"REGION_UNSPECIFIED","updateTime":"2023-11-07T05:31:56Z","disableDeploymentSizeValidation":true}],"nextPageToken":"","totalSize":123} + +", + "$.messages[7].content[6].type": "text", + "$.messages[7].content[7].text": "Title: Deploy Command +Link: https://docs.fireworks.ai/evaluators/cli_reference/cli_overview +Content: The deploy command deploys a reward function as an evaluator on the Fireworks platform. + +The deploy command deploys a reward function as an evaluator on the Fireworks platform. + +Command Overview +The Reward Kit CLI supports the following main commands: run : Run a local evaluation pipeline using a Hydra configuration. preview : Preview evaluation results or re-evaluate generated outputs. deploy : Deploy a reward function as an evaluator. agent-eval : Run agent evaluations on task bundles. list : List existing evaluators (coming soon). delete : Delete an evaluator (coming soon). + +", + "$.messages[7].content[7].type": "text", + "$.messages[7].content[8].text": "Title: Deploying a fine-tuned model using an on-demand deployment +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-models +Content: Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Deploying a fine-tuned model serverlessly +Not all base models support serverless addons. Please check this list for the full list of serverless models that support LoRA add-ons. If your base model supports serverless addons, you can run the following command to load the fine-tuned model into a serverless deployment: firectl load-lora + +", + "$.messages[7].content[8].type": "text", + "$.messages[7].content[9].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. firectl create deployment [flags] + +Create a Deployment on Fireworks AI platform + +", + "$.messages[7].content[9].type": "text", + "$.messages[7].role": "tool", + "$.messages[7].tool_call_id": "functions.search:2", + "$.messages[8].content": "Now let me search for information about deployment configuration and management best practices.", + "$.messages[8].role": "assistant", + "$.messages[8].tool_calls[0].function.arguments": "{"query": "deployment configuration scaling autoscaling reserved capacity Fireworks"}", + "$.messages[8].tool_calls[0].function.name": "search", + "$.messages[8].tool_calls[0].id": "functions.search:0", + "$.messages[8].tool_calls[0].type": "function", + "$.messages[9].content[0].text": "Title: How does billing and scaling work for on-demand GPU deployments? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/how-does-billing-and-scaling-work-for-on-demand-gpu-deployments +Content: On-demand GPU deployments have unique billing and scaling characteristics compared to serverless deployments: Billing: Charges start when the server begins accepting requests Billed by GPU-second for each active instance Costs accumulate even if there are no active API calls Scaling options: Supports autoscaling from 0 to multiple GPUs Each additional GPU adds to the billing rate Can handle unlimited requests within the GPUā€™s capacity Management requirements: Not fully serverless; requires some manual management Manually delete deployments when no longer needed Or configure autoscaling to scale down to 0 during inactive periods Cost control tips: Regularly monitor active deployments Delete unused deployments to avoid unnecessary costs Consider serverless options for intermittent usage Use autoscaling to 0 to optimize costs during low-demand times + +On-demand GPU deployments have unique billing and scaling characteristics compared to serverless deployments: Billing: Charges start when the server begins accepting requests Billed by GPU-second for each active instance Costs accumulate even if there are no active API calls Scaling options: Supports autoscaling from 0 to multiple GPUs Each additional GPU adds to the billing rate Can handle unlimited requests within the GPUā€™s capacity Management requirements: Not fully serverless; requires some manual management Manually delete deployments when no longer needed Or configure autoscaling to scale down to 0 during inactive periods Cost control tips: Regularly monitor active deployments Delete unused deployments to avoid unnecessary costs Consider serverless options for intermittent usage Use autoscaling to 0 to optimize costs during low-demand times + +How does billing and scaling work for on-demand GPU deployments? +On-demand GPU deployments have unique billing and scaling characteristics compared to serverless deployments: Billing: Charges start when the server begins accepting requests Billed by GPU-second for each active instance Costs accumulate even if there are no active API calls Scaling options: Supports autoscaling from 0 to multiple GPUs Each additional GPU adds to the billing rate Can handle unlimited requests within the GPUā€™s capacity Management requirements: Not fully serverless; requires some manual management Manually delete deployments when no longer needed Or configure autoscaling to scale down to 0 during inactive periods Cost control tips: Regularly monitor active deployments Delete unused deployments to avoid unnecessary costs Consider serverless options for intermittent usage Use autoscaling to 0 to optimize costs during low-demand times + +", + "$.messages[9].content[0].type": "text", + "$.messages[9].content[1].text": "Title: What should I expect for deployment and scaling performance? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/what-should-i-expect-for-deployment-and-scaling-performance +Content: Initial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long + +Initial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long + +What should I expect for deployment and scaling performance? +Initial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long + +", + "$.messages[9].content[1].type": "text", + "$.messages[9].content[2].text": "Title: Deployment type selection +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-basics +Content: The SDK tries to be parsimonious with the way it deploys resources. We provide two types of deployment options on Fireworks: serverless hosting is enabled for some commonly-used state of the art models. The pricing for these models is per-token, i.e. you only pay for the tokens you use, and subject to rate limits. on-demand hosting is enabled for all other models. The pricing for these models is per GPU-second. This hosting is required for models that are not available serverlessly or workloads that exceed serverless rate limits. For non-finetuned models, you can always specify the deployment type of LLM() by passing either "serverless" or "on-demand" as the deployment_type parameter to the constructor. If the model is not available for the deployment type you selected, the SDK will throw an error. The SDK can also decide the best deployment strategy on your behalf, just pass deployment_type="auto" . If the model is available serverlessly, the SDK will use serverless hosting, otherwise the SDK will create an on-demand deployment. When using deployment_type="on-demand" or deployment_type="on-demand-lora" , you must call .apply() to apply the deployment configuration to Fireworks. This is not required for serverless deployments. When using deployment_type="auto" , the SDK will automatically handle deployment creation, but if it falls back to on-demand deployment, you may need to call .apply() explicitly. If you do not call .apply() , you are expected to set up the deployment through the deployment page at https://app.fireworks.ai/dashboard/deployments. Be careful with the deployment_type parameter, especially for "auto" and "on-demand" deployments. While the SDK will try to make the most cost effective choice for you and put sensible autoscaling policies in place, it is possible to unintentionally create many deployments that lead to unwanted spend, especially when working with non-serverless models. When using deployment_type="on-demand" , you must provide an id parameter to uniquely identify your deployment. This is required to prevent accidental creation of multiple deployments. For finetuned (LoRA) models, passing deployment_type="serverless" will try to deploy the finetuned model to serverless hosting, deployment_type="on-demand" will create an on-demand deployment of your base model and merge in your LoRA weights, deployment_type="on-demand-lora" will create an on-demand deployment with Multi-LoRA enabled, and deployment_type="auto" will try to use serverless if available, otherwise fall back to on-demand-lora. + +The SDK tries to be parsimonious with the way it deploys resources. We provide two types of deployment options on Fireworks: serverless hosting is enabled for some commonly-used state of the art models. The pricing for these models is per-token, i.e. you only pay for the tokens you use, and subject to rate limits. on-demand hosting is enabled for all other models. The pricing for these models is per GPU-second. This hosting is required for models that are not available serverlessly or workloads that exceed serverless rate limits. For non-finetuned models, you can always specify the deployment type of LLM() by passing either "serverless" or "on-demand" as the deployment_type parameter to the constructor. If the model is not available for the deployment type you selected, the SDK will throw an error. The SDK can also decide the best deployment strategy on your behalf, just pass deployment_type="auto" . If the model is available serverlessly, the SDK will use serverless hosting, otherwise the SDK will create an on-demand deployment. When using deployment_type="on-demand" or deployment_type="on-demand-lora" , you must call .apply() to apply the deployment configuration to Fireworks. This is not required for serverless deployments. When using deployment_type="auto" , the SDK will automatically handle deployment creation, but if it falls back to on-demand deployment, you may need to call .apply() explicitly. If you do not call .apply() , you are expected to set up the deployment through the deployment page at https://app.fireworks.ai/dashboard/deployments. Be careful with the deployment_type parameter, especially for "auto" and "on-demand" deployments. While the SDK will try to make the most cost effective choice for you and put sensible autoscaling policies in place, it is possible to unintentionally create many deployments that lead to unwanted spend, especially when working with non-serverless models. When using deployment_type="on-demand" , you must provide an id parameter to uniquely identify your deployment. This is required to prevent accidental creation of multiple deployments. For finetuned (LoRA) models, passing deployment_type="serverless" will try to deploy the finetuned model to serverless hosting, deployment_type="on-demand" will create an on-demand deployment of your base model and merge in your LoRA weights, deployment_type="on-demand-lora" will create an on-demand deployment with Multi-LoRA enabled, and deployment_type="auto" will try to use serverless if available, otherwise fall back to on-demand-lora. + +Deploying Fine-tuned Models with On-Demand +When deploying a fine-tuned model using deployment_type="on-demand" , you need to provide: model - Your fine-tuned model ID (e.g., "accounts/your-account/models/your-fine-tuned-model-id") id - A unique deployment identifier (can be any simple string like "my-fine-tuned-deployment") # Deploy a fine-tuned model with on-demand deployment +fine_tuned_llm = LLM( + model="accounts/your-account/models/your-fine-tuned-model-id", + deployment_type="on-demand", + id="my-fine-tuned-deployment" # Simple string ID +) + +# Apply deployment configuration to Fireworks +fine_tuned_llm.apply() + +# Track deployment in web dashboard +print(f"Track at: {fine_tuned_llm.deployment_url}") + +", + "$.messages[9].content[2].type": "text", + "$.messages[9].content[3].text": "Title: Spike arrest policy +Link: https://docs.fireworks.ai/guides/quotas_usage/rate-limits +Content: LLM traffic that spikes quickly has the potential to be throttled. Here's how it works: Each user has a guaranteed rate limit, which increases with sustained usage near the limit. Typically, you can expect to stay within the limits if your traffic gradually doubles within an hour. You can see your guaranteed limits using API response headers (see below) Exceeding your guaranteed limit means that there's the potential for your requests to be processed with lower-priority. Fireworks operates serverless deployments by autoscaling capacity (within limits) as user traffic increases. However, if a deployment is overloaded while auto-scaling, requests that fall outside of guaranteed limits may be processed with lower-latency or dropped with HTTP code 429 (if limits are significantly exceeded). You can monitor if you exceed limits via API response header x-ratelimit-over-limit: yes. Exceeding your guaranteed limit does not guarantee that your requests will be throttled. You can monitor if your requests are actually being throttled by monitoring latencies. Here's an example of how dynamic rate limits scale up: Metric Minimum Guaranteed Limit 10 Minutes 1 Hour 2 Hours Requests per minute 60 120 720 1440 Input tokens per minute 60000 120000 720000 1440000 Output tokens per minute 6000 12000 72000 144000 + +LLM traffic that spikes quickly has the potential to be throttled. Here's how it works: Each user has a guaranteed rate limit, which increases with sustained usage near the limit. Typically, you can expect to stay within the limits if your traffic gradually doubles within an hour. You can see your guaranteed limits using API response headers (see below) Exceeding your guaranteed limit means that there's the potential for your requests to be processed with lower-priority. Fireworks operates serverless deployments by autoscaling capacity (within limits) as user traffic increases. However, if a deployment is overloaded while auto-scaling, requests that fall outside of guaranteed limits may be processed with lower-latency or dropped with HTTP code 429 (if limits are significantly exceeded). You can monitor if you exceed limits via API response header x-ratelimit-over-limit: yes. Exceeding your guaranteed limit does not guarantee that your requests will be throttled. You can monitor if your requests are actually being throttled by monitoring latencies. Here's an example of how dynamic rate limits scale up: Metric Minimum Guaranteed Limit 10 Minutes 1 Hour 2 Hours Requests per minute 60 120 720 1440 Input tokens per minute 60000 120000 720000 1440000 Output tokens per minute 6000 12000 72000 144000 + +Spend limits +In order to prevent fraud, Fireworks imposes a monthly spending limit on your account. Once you hit the spending limit, your account will automatically enter a suspended state, API requests will be rejected and all Fireworks usage will be stopped. This includes serverless inference, dedicated deployments, and fine-tuning jobs. Your spend limit will organically increase over time as you spend more on the platform. You can also increase your spend limit at any time, by purchasing prepaid credits to meet the historical spend required for a higher tier. For instance, if you are a new Tier 1 user with $0 historical spend, you can purchase $100 prepaid credits and become a Tier 2 user. You can qualify for a higher tier by adding credits into your Fireworks account. There may be a propagation delay for a few minutes after you prepay for credits - you may still see "monthly usage exceeded error" for a few minutes after adding credits. Tier Qualification Spending Limit Tier 1 Valid payment method added $50/mo Tier 2 $50 spent in payments or credits added $500/mo Tier 3 $500 spent in payments or credits added $5,000/mo Tier 4 $5000 spent in payments or credits added $50,000/mo Unlimited Contact us at inquiries@fireworks.ai Unlimited + +", + "$.messages[9].content[3].type": "text", + "$.messages[9].content[4].text": "Title: How does billing work for on-demand deployments? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/how-does-billing-work-for-on-demand-deployments +Content: On-demand deployments come with automatic cost optimization features: Default autoscaling : Automatically scales to 0 replicas when not in use Pay for what you use : Charged only for GPU time when replicas are active Flexible configuration : Customize autoscaling behavior to match your needs Best practices for cost management: Leverage default autoscaling : The system automatically scales down deployments when not in use Customize carefully : While you can modify autoscaling behavior using our configuration options , note that preventing scale-to-zero will result in continuous GPU charges Consider your use case : For intermittent or low-frequency usage, serverless deployments might be more cost-effective For detailed configuration options, see our deployment guide. + +On-demand deployments come with automatic cost optimization features: Default autoscaling : Automatically scales to 0 replicas when not in use Pay for what you use : Charged only for GPU time when replicas are active Flexible configuration : Customize autoscaling behavior to match your needs Best practices for cost management: Leverage default autoscaling : The system automatically scales down deployments when not in use Customize carefully : While you can modify autoscaling behavior using our configuration options , note that preventing scale-to-zero will result in continuous GPU charges Consider your use case : For intermittent or low-frequency usage, serverless deployments might be more cost-effective For detailed configuration options, see our deployment guide. + +How does billing work for on-demand deployments? +On-demand deployments come with automatic cost optimization features: Default autoscaling : Automatically scales to 0 replicas when not in use Pay for what you use : Charged only for GPU time when replicas are active Flexible configuration : Customize autoscaling behavior to match your needs Best practices for cost management: Leverage default autoscaling : The system automatically scales down deployments when not in use Customize carefully : While you can modify autoscaling behavior using our configuration options , note that preventing scale-to-zero will result in continuous GPU charges Consider your use case : For intermittent or low-frequency usage, serverless deployments might be more cost-effective For detailed configuration options, see our deployment guide. + +", + "$.messages[9].content[4].type": "text", + "$.messages[9].content[5].text": "Title: Replica count (horizontal scaling) +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: The number of replicas (horizontal scaling) is specified by passing the --min-replica-count and --max-replica-count flags. Increasing the number of replicas will increase the maximum QPS the deployment can support. The deployment will automatically scale based on server load. Auto-scaling up may fail if there is a GPU stockout. Use reserved capacity to guarantee capacity for your deployments. The default value for --min-replica-count is 0. Setting --min-replica-count to 0 enables the deployment to auto-scale to 0 if a deployment is unused (i.e. no inference requests) for a specified "scale-to-zero" time window. While the deployment is scaled to 0, you will not pay for any GPU utilization. The default value for --max-replica-count is 1 if --min-replica-count=0 , or the value of +--min-replica-count otherwise. firectl create deployment \\ + --min-replica-count 2 \\ + --max-replica-count 3 + +The number of replicas (horizontal scaling) is specified by passing the --min-replica-count and --max-replica-count flags. Increasing the number of replicas will increase the maximum QPS the deployment can support. The deployment will automatically scale based on server load. Auto-scaling up may fail if there is a GPU stockout. Use reserved capacity to guarantee capacity for your deployments. The default value for --min-replica-count is 0. Setting --min-replica-count to 0 enables the deployment to auto-scale to 0 if a deployment is unused (i.e. no inference requests) for a specified "scale-to-zero" time window. While the deployment is scaled to 0, you will not pay for any GPU utilization. The default value for --max-replica-count is 1 if --min-replica-count=0 , or the value of +--min-replica-count otherwise. firectl create deployment \\ + --min-replica-count 2 \\ + --max-replica-count 3 + +The number of replicas (horizontal scaling) is specified by passing the --min-replica-count and --max-replica-count flags. Increasing the number of replicas will increase the maximum QPS the deployment can support. The deployment will automatically scale based on server load. Auto-scaling up may fail if there is a GPU stockout. Use reserved capacity to guarantee capacity for your deployments. The default value for --min-replica-count is 0. Setting --min-replica-count to 0 enables the deployment to auto-scale to 0 if a deployment is unused (i.e. no inference requests) for a specified "scale-to-zero" time window. While the deployment is scaled to 0, you will not pay for any GPU utilization. The default value for --max-replica-count is 1 if --min-replica-count=0 , or the value of +--min-replica-count otherwise. firectl update deployment \\ + --min-replica-count 2 \\ + --max-replica-count 3 + +", + "$.messages[9].content[5].type": "text", + "$.messages[9].content[6].text": "Title: Do you support Auto Scaling? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/do-you-support-auto-scaling +Content: Yes, our system supports auto scaling with the following features: Scaling down to zero capability for resource efficiency Controllable scale-up and scale-down velocity Custom scaling rules and thresholds to match your specific needs + +Yes, our system supports auto scaling with the following features: Scaling down to zero capability for resource efficiency Controllable scale-up and scale-down velocity Custom scaling rules and thresholds to match your specific needs + +Do you support Auto Scaling? +Yes, our system supports auto scaling with the following features: Scaling down to zero capability for resource efficiency Controllable scale-up and scale-down velocity Custom scaling rules and thresholds to match your specific needs + +", + "$.messages[9].content[6].type": "text", + "$.messages[9].content[7].text": "Title: Inference +Link: https://docs.fireworks.ai/getting-started/introduction +Content: Fireworks has 3 options for running generative AI models with unparalleled speed and costs. Serverless: The easiest way to get started. Use the most popular models on pre-configured GPUs. Pay per token and avoid cold boots. On-demand: The most flexible option for scaling. Use private GPUs to support your specific needs and only pay when you're using it. GPUs running Fireworks software offer both ~250% improved throughput and 50% improved latency compared to vLLM. Excels for: Production volume - Per-token costs decrease with more volume and there are no set rate limits Custom needs and reliability - On-demand GPUs are private to you. This enables complete control to tailor deployments for speed/throughput/reliability or to run more specialized models Enterprise Reserved GPUs: Use private GPUs with hardware and software set-up personally tailored by the Fireworks team for your use case. Enjoy SLAs, dedicated support, bring-your-own-cloud (BYOC) deployment options, and enterprise-only optimizations. Property Serverless On-demand Enterprise reserved Performance Industry-leading speed on Fireworks-curated set-up. Performance may vary with others' usage. Speed dependent on user-specified GPU configuration and private usage. Per GPU latency should be significantly faster than vLLM. Tailor-made set-up by Fireworks AI experts for best possible latency Getting Started Self-serve - immediately use serverless with 1 line of code Self-serve - configure GPUs, then use them with 1 line of code. Chat with Fireworks Scaling and management Scale up and down freely within rate limits Option for auto-scaling GPUs with traffic. GPUs scale to zero automatically, so no charge for unused GPUs and for boot-ups. Chat with Fireworks Pricing Pay fixed price per token Pay per GPU second with no commitments. Per GPU throughput should be significantly greater than options like vLLM. Customized price based on reserved GPU capacity Commitment None None Arrange plan length with Fireworks Rate limits Yes, see quotas No rate limits. Quotas on number of GPUs None Model Selection Collection of popular models, curated by Fireworks Use 100s of pre-uploaded models or upload your own custom model within supported architecture Use 100s of pre-uploaded models or upload any model + +Fireworks has 3 options for running generative AI models with unparalleled speed and costs. Serverless: The easiest way to get started. Use the most popular models on pre-configured GPUs. Pay per token and avoid cold boots. On-demand: The most flexible option for scaling. Use private GPUs to support your specific needs and only pay when you're using it. GPUs running Fireworks software offer both ~250% improved throughput and 50% improved latency compared to vLLM. Excels for: Production volume - Per-token costs decrease with more volume and there are no set rate limits Custom needs and reliability - On-demand GPUs are private to you. This enables complete control to tailor deployments for speed/throughput/reliability or to run more specialized models Enterprise Reserved GPUs: Use private GPUs with hardware and software set-up personally tailored by the Fireworks team for your use case. Enjoy SLAs, dedicated support, bring-your-own-cloud (BYOC) deployment options, and enterprise-only optimizations. Property Serverless On-demand Enterprise reserved Performance Industry-leading speed on Fireworks-curated set-up. Performance may vary with others' usage. Speed dependent on user-specified GPU configuration and private usage. Per GPU latency should be significantly faster than vLLM. Tailor-made set-up by Fireworks AI experts for best possible latency Getting Started Self-serve - immediately use serverless with 1 line of code Self-serve - configure GPUs, then use them with 1 line of code. Chat with Fireworks Scaling and management Scale up and down freely within rate limits Option for auto-scaling GPUs with traffic. GPUs scale to zero automatically, so no charge for unused GPUs and for boot-ups. Chat with Fireworks Pricing Pay fixed price per token Pay per GPU second with no commitments. Per GPU throughput should be significantly greater than options like vLLM. Customized price based on reserved GPU capacity Commitment None None Arrange plan length with Fireworks Rate limits Yes, see quotas No rate limits. Quotas on number of GPUs None Model Selection Collection of popular models, curated by Fireworks Use 100s of pre-uploaded models or upload your own custom model within supported architecture Use 100s of pre-uploaded models or upload any model + +Fireworks AI Developer Platform +Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +", + "$.messages[9].content[7].type": "text", + "$.messages[9].content[8].text": "Title: 2. Call a language model using the LLM() class +Link: https://docs.fireworks.ai/tools-sdks/python-client/the-tutorial +Content: Now that your machine is setup with credentials and the SDK, lets ensure you are +ready to make your first LLM call and explain some of the nuances of this SDK. Create a new file called main.py and import the Fireworks AI SDK. Instantiate the LLM class. The LLM class accepts a model argument that you +can use to specify the model you want to use. For this tutorial, we will use the +Llama 4 Maverick +model. When creating an LLM instance, you can specify the deployment type as either "serverless" , "on-demand" , or "auto" . If you pass "auto" , the SDK will try to use serverless hosting if available, otherwise it will create an on-demand deployment. In the other cases, the SDK will try to create a deployment of the specified type and will throw an error if it's not available for the model you selected. The SDK will try and re-use existing deployments for the same model if possible, see Resource management for more details. With great power comes great responsibility! Be careful with the deployment_type parameter, especially for "auto" and "on-demand" . While the SDK will try to make the most cost effective choice for you and put sensible autoscaling policies in place, it is possible to unintentionally create many deployments that lead to unwanted spend, especially when working with non-serverless models. When using deployment_type="on-demand" , you must provide an id parameter to uniquely identify your deployment. This is required to prevent accidental creation of multiple deployments. When using deployment_type="on-demand" or deployment_type="on-demand-lora" , you must call .apply() to apply the deployment configuration to Fireworks. This is not required for serverless deployments. When using deployment_type="auto" , the SDK will automatically handle deployment creation, but if it falls back to on-demand deployment, you may need to call .apply() explicitly. If you do not call .apply() , you are expected to set up the deployment through the deployment page at https://app.fireworks.ai/dashboard/deployments. Make a request to the LLM. The LLM class is OpenAI compatible, so you can use +the same chat completion interface to make a request to the LLM. Hello! It's nice to meet you. Is there something I can help you with or would you like to chat? + +Now that your machine is setup with credentials and the SDK, lets ensure you are +ready to make your first LLM call and explain some of the nuances of this SDK. Create a new file called main.py and import the Fireworks AI SDK. Instantiate the LLM class. The LLM class accepts a model argument that you +can use to specify the model you want to use. For this tutorial, we will use the +Llama 4 Maverick +model. When creating an LLM instance, you can specify the deployment type as either "serverless" , "on-demand" , or "auto" . If you pass "auto" , the SDK will try to use serverless hosting if available, otherwise it will create an on-demand deployment. In the other cases, the SDK will try to create a deployment of the specified type and will throw an error if it's not available for the model you selected. The SDK will try and re-use existing deployments for the same model if possible, see Resource management for more details. With great power comes great responsibility! Be careful with the deployment_type parameter, especially for "auto" and "on-demand" . While the SDK will try to make the most cost effective choice for you and put sensible autoscaling policies in place, it is possible to unintentionally create many deployments that lead to unwanted spend, especially when working with non-serverless models. When using deployment_type="on-demand" , you must provide an id parameter to uniquely identify your deployment. This is required to prevent accidental creation of multiple deployments. When using deployment_type="on-demand" or deployment_type="on-demand-lora" , you must call .apply() to apply the deployment configuration to Fireworks. This is not required for serverless deployments. When using deployment_type="auto" , the SDK will automatically handle deployment creation, but if it falls back to on-demand deployment, you may need to call .apply() explicitly. If you do not call .apply() , you are expected to set up the deployment through the deployment page at https://app.fireworks.ai/dashboard/deployments. Make a request to the LLM. The LLM class is OpenAI compatible, so you can use +the same chat completion interface to make a request to the LLM. The great thing about the SDK is that you can use your favorite Python constructs to powerfully work with LLMs. For example, let's try calling a few LLMs in a loop and see how they respond: from fireworks import LLM + +llms = [ + "llama4-maverick-instruct-basic", + "deepseek-r1", + "qwen2p5-vl-32b-instruct" +] + +for llm in llms: + llm = LLM(model=llm, deployment_type="auto") + print("\\n" + "-" * 100) + print(f"Model: {llm.model}") + print("-" * 100 + "\\n") + + response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Hello, world!"}] + ) + print(response.choices[0].message.content) + +Now that your machine is setup with credentials and the SDK, lets ensure you are +ready to make your first LLM call and explain some of the nuances of this SDK. Create a new file called main.py and import the Fireworks AI SDK. Instantiate the LLM class. The LLM class accepts a model argument that you +can use to specify the model you want to use. For this tutorial, we will use the +Llama 4 Maverick +model. When creating an LLM instance, you can specify the deployment type as either "serverless" , "on-demand" , or "auto" . If you pass "auto" , the SDK will try to use serverless hosting if available, otherwise it will create an on-demand deployment. In the other cases, the SDK will try to create a deployment of the specified type and will throw an error if it's not available for the model you selected. The SDK will try and re-use existing deployments for the same model if possible, see Resource management for more details. With great power comes great responsibility! Be careful with the deployment_type parameter, especially for "auto" and "on-demand" . While the SDK will try to make the most cost effective choice for you and put sensible autoscaling policies in place, it is possible to unintentionally create many deployments that lead to unwanted spend, especially when working with non-serverless models. When using deployment_type="on-demand" , you must provide an id parameter to uniquely identify your deployment. This is required to prevent accidental creation of multiple deployments. When using deployment_type="on-demand" or deployment_type="on-demand-lora" , you must call .apply() to apply the deployment configuration to Fireworks. This is not required for serverless deployments. When using deployment_type="auto" , the SDK will automatically handle deployment creation, but if it falls back to on-demand deployment, you may need to call .apply() explicitly. If you do not call .apply() , you are expected to set up the deployment through the deployment page at https://app.fireworks.ai/dashboard/deployments. Make a request to the LLM. The LLM class is OpenAI compatible, so you can use +the same chat completion interface to make a request to the LLM. The great thing about the SDK is that you can use your favorite Python constructs to powerfully work with LLMs. For example, let's try calling a few LLMs in a loop and see how they respond: from fireworks import LLM + +llms = [ + "llama4-maverick-instruct-basic", + "deepseek-r1", + "qwen2p5-vl-32b-instruct" +] + +for llm in llms: + llm = LLM(model=llm, deployment_type="auto") + print("\\n" + "-" * 100) + print(f"Model: {llm.model}") + print("-" * 100 + "\\n") + + response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Hello, world!"}] + ) + print(response.choices[0].message.content) + +", + "$.messages[9].content[8].type": "text", + "$.messages[9].content[9].text": "Title: How does autoscaling affect my costs? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/how-does-autoscaling-affect-my-costs +Content: Scaling from 0 : No minimum cost when scaled to zero Scaling up : Each new replica adds to your total cost proportionally. For example: Scaling from 1 to 2 replicas doubles your GPU costs If each replica uses multiple GPUs, costs scale accordingly (e.g., scaling from 1 to 2 replicas with 2 GPUs each means paying for 4 GPUs total) For current pricing details, please visit our pricing page. + +Scaling from 0 : No minimum cost when scaled to zero Scaling up : Each new replica adds to your total cost proportionally. For example: Scaling from 1 to 2 replicas doubles your GPU costs If each replica uses multiple GPUs, costs scale accordingly (e.g., scaling from 1 to 2 replicas with 2 GPUs each means paying for 4 GPUs total) For current pricing details, please visit our pricing page. + +How does autoscaling affect my costs? +Scaling from 0 : No minimum cost when scaled to zero Scaling up : Each new replica adds to your total cost proportionally. For example: Scaling from 1 to 2 replicas doubles your GPU costs If each replica uses multiple GPUs, costs scale accordingly (e.g., scaling from 1 to 2 replicas with 2 GPUs each means paying for 4 GPUs total) For current pricing details, please visit our pricing page. + +", + "$.messages[9].content[9].type": "text", + "$.messages[9].role": "tool", + "$.messages[9].tool_call_id": "functions.search:0", + "$.pid": 95890, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:43:38.694430", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "finished", + "$.eval_metadata.version": "v0.2.6.dev19+gf755623", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "At least one tool call was made", + "$.evaluation_result.score": 1, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "look-win-current-private-teacher", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.messages[2].content": "I'll help you learn about managing deployments on Fireworks. Let me search for comprehensive information about deployment management.", + "$.messages[2].role": "assistant", + "$.messages[2].tool_calls[0].function.arguments": "{"query": "deployment management Fireworks AI"}", + "$.messages[2].tool_calls[0].function.name": "search", + "$.messages[2].tool_calls[0].id": "functions.search:0", + "$.messages[2].tool_calls[0].type": "function", + "$.messages[3].content[0].text": "Title: Fireworks AI Developer Platform +Link: https://docs.fireworks.ai/getting-started/introduction +Content: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Start building with open source AI models + +", + "$.messages[3].content[0].type": "text", + "$.messages[3].content[1].text": "Title: How do I close my Fireworks.ai account? +Link: https://docs.fireworks.ai/faq-new/account-access/how-do-i-close-my-fireworksai-account +Content: To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +How do I close my Fireworks.ai account? +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +", + "$.messages[3].content[1].type": "text", + "$.messages[3].content[2].text": "Title: Do you host your deployments in the EU or Asia? +Link: https://docs.fireworks.ai/faq-new/support-general/do-you-host-your-deployments-in-the-eu-or-asia +Content: We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +Do you host your deployments in the EU or Asia? +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +", + "$.messages[3].content[2].type": "text", + "$.messages[3].content[3].text": "Title: Introducing the Fireworks Build SDK +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-introduction +Content: The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +", + "$.messages[3].content[3].type": "text", + "$.messages[3].content[4].text": "Title: On-demand deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +On-demand deployments +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +", + "$.messages[3].content[4].type": "text", + "$.messages[3].content[5].text": "Title: I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +Link: https://docs.fireworks.ai/faq-new/account-access/i-have-multiple-fireworks-accounts-when-i-try-to-login-with-google-on-fireworks +Content: If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +", + "$.messages[3].content[5].type": "text", + "$.messages[3].content[6].text": "Title: Introduction +Link: https://docs.fireworks.ai/api-reference/introduction +Content: Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Introduction +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +", + "$.messages[3].content[6].type": "text", + "$.messages[3].content[7].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Create a Deployment on Fireworks AI platform + +Create a Deployment on Fireworks AI platform + +Creates a new deployment. + +", + "$.messages[3].content[7].type": "text", + "$.messages[3].content[8].text": "Title: How much does Fireworks cost? +Link: https://docs.fireworks.ai/faq-new/billing-pricing/how-much-does-fireworks-cost +Content: Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +How much does Fireworks cost? +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +", + "$.messages[3].content[8].type": "text", + "$.messages[3].content[9].text": "Title: Python (Fireworks) +Link: https://docs.fireworks.ai/getting-started/quickstart +Content: This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +Set up your developer environment & call the Fireworks API +This is the recommended way to get started : This tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: Sending the first API Request You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. Fireworks AI platform offers drop-in replacement with OpenAI Python Client. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $OPENAI_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %OPENAI_API_KEY% + +", + "$.messages[3].content[9].type": "text", + "$.messages[3].role": "tool", + "$.messages[3].tool_call_id": "functions.search:0", + "$.messages[4].content": "Let me search for more specific information about deployment management commands and workflows.", + "$.messages[4].role": "assistant", + "$.messages[4].tool_calls[0].function.arguments": "{"query": "firectl deployment commands create manage delete update"}", + "$.messages[4].tool_calls[0].function.name": "search", + "$.messages[4].tool_calls[0].id": "functions.search:1", + "$.messages[4].tool_calls[0].type": "function", + "$.messages[5].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[5].content[0].type": "text", + "$.messages[5].content[1].text": "Title: Delete a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/delete-resources +Content: Deletes an model deployment. firectl delete deployment [flags] + +Deletes an model deployment. firectl delete deployment [flags] + +Example +firectl delete deployment my-deployment + +", + "$.messages[5].content[1].type": "text", + "$.messages[5].content[2].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. firectl create deployment [flags] + +Create a Deployment on Fireworks AI platform + +", + "$.messages[5].content[2].type": "text", + "$.messages[5].content[3].text": "Title: Managing default deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: firectl update deployed-model --default + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: firectl update deployed-model --default + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: To delete a default deployment you must delete all other deployments for the same model first, or designate a different deployed model as the default as described above. This is to ensure that querying by model name will always route to an unambiguous default deployment as long as deployments for the model exist. + +", + "$.messages[5].content[3].type": "text", + "$.messages[5].content[4].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[5].content[4].type": "text", + "$.messages[5].content[5].text": "Title: Deployment Command +Link: https://docs.fireworks.ai/evaluators/examples/gcp_cloud_run_deployment_example +Content: It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: Run the deployment command: Command Explanation: --id my-dummy-gcp-evaluator : A unique ID for your evaluator on the Fireworks AI platform. --target gcp-cloud-run : Specifies deployment to GCP Cloud Run. --function-ref dummy_rewards.hello_world_reward : The Python import path to your reward function. If dummy_rewards.py is in the current directory, this reference works. --gcp-auth-mode api-key : Configures the Cloud Run service with API key authentication. reward-kit will generate a key, store it in GCP Secret Manager, and configure the service. The key is also saved to your local rewardkit.yaml under evaluator_endpoint_keys . This is the default if not specified. --verbose : Shows detailed output, including gcloud commands being executed. --force : (Optional) If an evaluator with the same --id already exists, this flag will delete the existing one before creating the new one. + +It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: Run the deployment command: Command Explanation: --id my-dummy-gcp-evaluator : A unique ID for your evaluator on the Fireworks AI platform. --target gcp-cloud-run : Specifies deployment to GCP Cloud Run. --function-ref dummy_rewards.hello_world_reward : The Python import path to your reward function. If dummy_rewards.py is in the current directory, this reference works. --gcp-auth-mode api-key : Configures the Cloud Run service with API key authentication. reward-kit will generate a key, store it in GCP Secret Manager, and configure the service. The key is also saved to your local rewardkit.yaml under evaluator_endpoint_keys . This is the default if not specified. --verbose : Shows detailed output, including gcloud commands being executed. --force : (Optional) If an evaluator with the same --id already exists, this flag will delete the existing one before creating the new one. + +It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: source .venv/bin/activate + +", + "$.messages[5].content[5].type": "text", + "$.messages[5].content[6].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[5].content[6].type": "text", + "$.messages[5].content[7].text": "Title: Delete Deployment +Link: https://docs.fireworks.ai/api-reference/delete-deployment +Content: + + + +REST Endpoint DELETE /v1/accounts/{account_id}/deployments/{deployment_id} + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" +deployment_id: + string + Description: The Deployment Id + Example: "" + +query params: +hard: + boolean + Description: If true, this will perform a hard deletion. + Example: true +ignoreChecks: + boolean + Description: If true, this will ignore checks and force the deletion of a deployment that is currently +deployed and is in use. + Example: true + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {} + +", + "$.messages[5].content[7].type": "text", + "$.messages[5].content[8].text": "Title: How do I close my Fireworks.ai account? +Link: https://docs.fireworks.ai/faq-new/account-access/how-do-i-close-my-fireworksai-account +Content: To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +How do I close my Fireworks.ai account? +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +", + "$.messages[5].content[8].type": "text", + "$.messages[5].content[9].text": "Title: Deleting Stored Responses +Link: https://docs.fireworks.ai/guides/response-api +Content: When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. from fireworks import LLM +import requests +import os + +llm = LLM(model="qwen3-235b-a22b", deployment_type="serverless") + +# Create a response +response = llm.responses.create( + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. from fireworks import LLM +import requests +import os + +llm = LLM(model="qwen3-235b-a22b", deployment_type="serverless") + +# Create a response +response = llm.responses.create( + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. import os +from openai import OpenAI +import requests + +client = OpenAI( + base_url="https://api.fireworks.ai/inference/v1", + api_key=os.getenv("FIREWORKS_API_KEY", "YOUR_FIREWORKS_API_KEY_HERE") +) + +# Create a response +response = client.responses.create( + model="accounts/fireworks/models/qwen3-235b-a22b", + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +", + "$.messages[5].content[9].type": "text", + "$.messages[5].role": "tool", + "$.messages[5].tool_call_id": "functions.search:1", + "$.messages[6].content": "Let me search for more information about listing and managing deployments, as well as the Fireworks Build SDK for deployment management.", + "$.messages[6].role": "assistant", + "$.messages[6].tool_calls[0].function.arguments": "{"query": "list deployments firectl Fireworks Build SDK deployment management"}", + "$.messages[6].tool_calls[0].function.name": "search", + "$.messages[6].tool_calls[0].id": "functions.search:1", + "$.messages[6].tool_calls[0].type": "function", + "$.messages[7].content[0].text": "Title: Introducing the Fireworks Build SDK +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-introduction +Content: Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +", + "$.messages[7].content[0].type": "text", + "$.messages[7].content[1].text": "Title: Basics of the Build SDK +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-basics +Content: Why use the Build SDK? +The Fireworks Build SDK gives you a declarative way to work with Fireworks resources like deployments, fine-tuning jobs, and datasets. We've designed it to handle all the infrastructure complexity for you, letting you focus on building your application. Instead of using the web UI, CLI, or raw API calls, you can manage everything through simple Python code with smart, logical defaults without sacrificing control and customizability. The principles of the SDK are the following: Object-oriented: Fireworks primitives are represented as Python objects. You can access their capabilities and properties through methods and attributes. Declarative: You can describe your desired state and the SDK will handle reconcilliation. Smart defaults: The SDK will infer the most logical defaults for you, prioritizing development speed and lowest cost. Here are some examples: The SDK will automatically use a serverless deployment for models that are available serverlessly unless you specify otherwise. When creating deployments, the SDK will also enable scale-to-zero with the shortest possible scale-down window. If the SDK determines that a resource already exists by matching its signature (see below), it will re-use the existing resource instead of creating a new one. Customizable: Although we enable smart defaults, you still have full access to the configuration parameters for any Fireworks resource The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Why use the Build SDK? +The Fireworks Build SDK gives you a declarative way to work with Fireworks resources like deployments, fine-tuning jobs, and datasets. We've designed it to handle all the infrastructure complexity for you, letting you focus on building your application. Instead of using the web UI, CLI, or raw API calls, you can manage everything through simple Python code with smart, logical defaults without sacrificing control and customizability. The principles of the SDK are the following: Object-oriented: Fireworks primitives are represented as Python objects. You can access their capabilities and properties through methods and attributes. Declarative: You can describe your desired state and the SDK will handle reconcilliation. Smart defaults: The SDK will infer the most logical defaults for you, prioritizing development speed and lowest cost. Here are some examples: The SDK will automatically use a serverless deployment for models that are available serverlessly unless you specify otherwise. When creating deployments, the SDK will also enable scale-to-zero with the shortest possible scale-down window. If the SDK determines that a resource already exists by matching its signature (see below), it will re-use the existing resource instead of creating a new one. Customizable: Although we enable smart defaults, you still have full access to the configuration parameters for any Fireworks resource The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Why use the Build SDK? +The Fireworks Build SDK gives you a declarative way to work with Fireworks resources like deployments, fine-tuning jobs, and datasets. We've designed it to handle all the infrastructure complexity for you, letting you focus on building your application. Instead of using the web UI, CLI, or raw API calls, you can manage everything through simple Python code with smart, logical defaults without sacrificing control and customizability. The principles of the SDK are the following: Object-oriented: Fireworks primitives are represented as Python objects. You can access their capabilities and properties through methods and attributes. Declarative: You can describe your desired state and the SDK will handle reconcilliation. Smart defaults: The SDK will infer the most logical defaults for you, prioritizing development speed and lowest cost. Here are some examples: The SDK will automatically use a serverless deployment for models that are available serverlessly unless you specify otherwise. When creating deployments, the SDK will also enable scale-to-zero with the shortest possible scale-down window. If the SDK determines that a resource already exists by matching its signature (see below), it will re-use the existing resource instead of creating a new one. Customizable: Although we enable smart defaults, you still have full access to the configuration parameters for any Fireworks resource The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +", + "$.messages[7].content[1].type": "text", + "$.messages[7].content[2].text": "Title: Querying existing dedicated deployments +Link: https://docs.fireworks.ai/tools-sdks/python-client/querying-dedicated-deployments +Content: When you have dedicated deployments that were created via firectl or the Fireworks web UI, you can easily connect to them using the Build SDK to run inference. This is particularly useful when you want to leverage existing infrastructure or when deployments are managed by different teams. + +When you have dedicated deployments that were created via firectl or the Fireworks web UI, you can easily connect to them using the Build SDK to run inference. This is particularly useful when you want to leverage existing infrastructure or when deployments are managed by different teams. + +Deployment ID requirements +The id parameter should match exactly with your existing deployment: Use the deployment name/ID as shown in the Fireworks dashboard The ID is case-sensitive and must match exactly If the deployment doesn't exist, you'll receive an error when making requests + +", + "$.messages[7].content[2].type": "text", + "$.messages[7].content[3].text": "Title: Fireworks SDK +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: You can connect to your existing deployment using the Fireworks Build SDK : from fireworks import LLM + +# Connect to your existing deployment +llm = LLM( + model="llama-v3p2-3b-instruct", # The model your deployment is running + deployment_type="on-demand", + id="my-deployment-id", # Your deployment ID +) + +# Use OpenAI-compatible chat completions +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) When connecting to an existing deployment with the SDK, you don't need to call .apply() - the deployment is already running. For more details on querying existing deployments with Fireworks SDK, see our guide on querying dedicated deployments . + +You can connect to your existing deployment using the Fireworks Build SDK : from fireworks import LLM + +# Connect to your existing deployment +llm = LLM( + model="llama-v3p2-3b-instruct", # The model your deployment is running + deployment_type="on-demand", + id="my-deployment-id", # Your deployment ID +) + +# Use OpenAI-compatible chat completions +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) When connecting to an existing deployment with the SDK, you don't need to call .apply() - the deployment is already running. For more details on querying existing deployments with Fireworks SDK, see our guide on querying dedicated deployments . + +On-demand deployments +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +", + "$.messages[7].content[3].type": "text", + "$.messages[7].content[4].text": "Title: Step 1: Configure the Fireworks Build SDK +Link: https://docs.fireworks.ai/structured-responses/structured-output-grammar-based +Content: from fireworks import LLM + +client = LLM( + model="llama-v3p1-405b-instruct", + deployment_type="serverless" +) + +from fireworks import LLM + +client = LLM( + model="llama-v3p1-405b-instruct", + deployment_type="serverless" +) + +Prerequisites +Before you begin, ensure you have the following: Python installed on your system. Build SDK installed. You can install it using pip: Next, select the model you want to use. In this example, we use llama-v3p1-405b-instruct , but all fireworks models support this feature. + +", + "$.messages[7].content[4].type": "text", + "$.messages[7].content[5].text": "Title: Does Fireworks support custom base models? +Link: https://docs.fireworks.ai/faq-new/models-inference/does-fireworks-support-custom-base-models +Content: Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Does Fireworks support custom base models? +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +", + "$.messages[7].content[5].type": "text", + "$.messages[7].content[6].text": "Title: šŸ”§ Build SDK: Deployment Configuration Application Requirement +Link: https://docs.fireworks.ai/updates/changelog +Content: The Build SDK now requires you to call .apply() to apply any deployment configurations to Fireworks when using deployment_type="on-demand" or deployment_type="on-demand-lora" . This change ensures explicit control over when deployments are created and helps prevent accidental deployment creation. Key changes: .apply() is now required for on-demand and on-demand-lora deployments Serverless deployments do not require .apply() calls If you do not call .apply() , you are expected to set up the deployment through the deployment page at https://app.fireworks.ai/dashboard/deployments Migration guide: Add llm.apply() after creating LLM instances with deployment_type="on-demand" or deployment_type="on-demand-lora" No changes needed for serverless deployments See updated documentation for examples and best practices This change improves deployment management and provides better control over resource creation. This applies to Python SDK version >=0.19.14. + +The Build SDK now requires you to call .apply() to apply any deployment configurations to Fireworks when using deployment_type="on-demand" or deployment_type="on-demand-lora" . This change ensures explicit control over when deployments are created and helps prevent accidental deployment creation. Key changes: .apply() is now required for on-demand and on-demand-lora deployments Serverless deployments do not require .apply() calls If you do not call .apply() , you are expected to set up the deployment through the deployment page at https://app.fireworks.ai/dashboard/deployments Migration guide: Add llm.apply() after creating LLM instances with deployment_type="on-demand" or deployment_type="on-demand-lora" No changes needed for serverless deployments See updated documentation for examples and best practices This change improves deployment management and provides better control over resource creation. This applies to Python SDK version >=0.19.14. + +šŸŽÆ Build SDK: Reward-kit integration for evaluator development +The Build SDK now natively integrates with reward-kit to simplify evaluator development for Reinforcement Fine-Tuning (RFT) . You can now create custom evaluators in Python with automatic dependency management and seamless deployment to Fireworks infrastructure. Key features: Native reward-kit integration for evaluator development Automatic packaging of dependencies from pyproject.toml or requirements.txt Local testing capabilities before deployment Direct integration with Fireworks datasets and evaluation jobs Support for third-party libraries and complex evaluation logic See our Developing Evaluators guide to get started with your first evaluator in minutes. + +", + "$.messages[7].content[6].type": "text", + "$.messages[7].content[7].text": "Title: Set up your developer environment & call the Fireworks API +Link: https://docs.fireworks.ai/getting-started/quickstart +Content: This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: Sending the first API Request You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) + +This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: Sending the first API Request You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) + +Python (Fireworks) +This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +", + "$.messages[7].content[7].type": "text", + "$.messages[7].content[8].text": "Title: Example with Fireworks Build SDK +Link: https://docs.fireworks.ai/faq-new/models-inference/how-to-get-performance-metrics-for-streaming-responses +Content: from fireworks import LLM +import os + +llm = LLM( + model="llama-v3p1-8b-instruct", + deployment_type="serverless", + api_key=os.environ["FIREWORKS_API_KEY"], +) + +# Streaming completion with performance metrics +stream = llm.chat.completions.create( + messages=[{"role": "user", "content": "Hello, world!"}], + max_tokens=100, + stream=True, + perf_metrics_in_response=True, +) + +for chunk in stream: + # Use getattr to avoid linter errors for unknown attributes + perf_metrics = getattr(chunk, "perf_metrics", None) + finish_reason = getattr(chunk.choices[0], "finish_reason", None) + if perf_metrics is not None and finish_reason: + print("Performance metrics:", perf_metrics) + +from fireworks import LLM +import os + +llm = LLM( + model="llama-v3p1-8b-instruct", + deployment_type="serverless", + api_key=os.environ["FIREWORKS_API_KEY"], +) + +# Streaming completion with performance metrics +stream = llm.chat.completions.create( + messages=[{"role": "user", "content": "Hello, world!"}], + max_tokens=100, + stream=True, + perf_metrics_in_response=True, +) + +for chunk in stream: + # Use getattr to avoid linter errors for unknown attributes + perf_metrics = getattr(chunk, "perf_metrics", None) + finish_reason = getattr(chunk.choices[0], "finish_reason", None) + if perf_metrics is not None and finish_reason: + print("Performance metrics:", perf_metrics) + +Performance Metrics Overview +The Inference API returns several per-request metrics in the response. They can be useful for one-off debugging or can be logged by the client in their preferred observability tool. For aggregate metrics, see the usage dashboard. Non-streaming requests: Performance metrics are always included in response headers (e.g., fireworks-prompt-tokens , fireworks-server-time-to-first-token ). Streaming requests: Only selected performance metrics, such as "fireworks-server-time-to-first-token," are available because HTTP headers must be sent before the first token can be streamed. Use the perf_metrics_in_response body parameter to include all metrics in the last SSE event of the response body. + +", + "$.messages[7].content[8].type": "text", + "$.messages[7].content[9].text": "Title: Fireworks AI Developer Platform +Link: https://docs.fireworks.ai/getting-started/introduction +Content: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Start building with open source AI models + +", + "$.messages[7].content[9].type": "text", + "$.messages[7].role": "tool", + "$.messages[7].tool_call_id": "functions.search:1", + "$.messages[8].content": "Based on my research, I can now provide you with a comprehensive guide on managing deployments on Fireworks. Let me break this down into different approaches: + +## Managing Deployments on Fireworks + +Fireworks offers three main ways to manage deployments: + +### 1. **Fireworks Build SDK (Recommended - Python)** +This is the most user-friendly approach using Python. + +**Installation:** +\`\`\`bash +pip install --upgrade fireworks-ai +\`\`\` + +**Basic Deployment Creation:** +\`\`\`python +from fireworks import LLM + +# Create a new deployment +llm = LLM( + model="llama-v3p2-3b-instruct", + deployment_type="on-demand", # or "serverless", "on-demand-lora" + id="my-deployment-name" +) + +# Apply the deployment (required for on-demand deployments) +llm.apply() + +# Use the deployment +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Hello, world!"}] +) +print(response.choices[0].message.content) +\`\`\` + +**Key Features:** +- **Smart defaults**: Automatically selects optimal deployment type +- **Declarative**: Describe desired state, SDK handles reconciliation +- **Object-oriented**: Deployments are Python objects with methods/attributes +- **Scale-to-zero**: Enabled by default with shortest scale-down window + +### 2. **Firectl CLI Tool** +Command-line interface for deployment management. + +**Create a deployment:** +\`\`\`bash +firectl create deployment "accounts/fireworks/models/llama-v3p2-3b-instruct" --id my-deployment +\`\`\` + +**Deploy LoRA fine-tunes:** +\`\`\`bash +# Single command for LoRA deployment +firectl create deployment "accounts/fireworks/models/" + +# Traditional two-step method (for multi-LoRA) +firectl create deployment "accounts/fireworks/models/" --enable-addons +firectl load-lora --deployment +\`\`\` + +**List deployments:** +\`\`\`bash +firectl list deployments +\`\`\` + +**Delete deployments:** +\`\`\`bash +firectl delete deployment my-deployment +\`\`\` + +**Update default deployment:** +\`\`\`bash +firectl update deployed-model --default +\`\`\` + +### 3. **REST API** +Direct API calls for programmatic management. + +**Create deployment:** +\`\`\`bash +curl -X POST https://api.fireworks.ai/v1/accounts/{account_id}/deployments \\ + -H "Authorization: Bearer YOUR_API_KEY" \\ + -H "Content-Type: application/json" \\ + -d '{ + "model": "llama-v3p2-3b-instruct", + "deployment_type": "on-demand" + }' +\`\`\` + +**Delete deployment:** +\`\`\`bash +curl -X DELETE https://api.fireworks.ai/v1/accounts/{account_id}/deployments/{deployment_id} \\ + -H "Authorization: Bearer YOUR_API_KEY" +\`\`\` + +## Deployment Types Explained + +### **Serverless Deployments** +- **Billing**: Per-token usage +- **Best for**: Variable/low usage, quick prototyping +- **Management**: Fully managed by Fireworks +- **Access**: Instant, no setup required + +### **On-Demand Deployments** +- **Billing**: Per GPU-second +- **Best for**: Consistent usage, predictable performance +- **Advantages**: + - No rate limits + - Predictable performance + - Access to larger model selection + - Custom base models from Hugging Face +- **Management**: Requires explicit creation/deletion + +### **Reserved Capacity** +- **Best for**: High-volume, consistent usage +- **Advantages**: More reliable capacity, higher GPU quotas +- **Contact**: inquiries@fireworks.ai for enterprise pricing + +## Complete Management Workflow Example + +Here's a comprehensive example using the Build SDK: + +\`\`\`python +from fireworks import LLM +import os + +# Set up API key +os.environ["FIREWORKS_API_KEY"] = "your-api-key" + +# 1. Create and manage deployments +def manage_deployments(): + + # Create a serverless deployment (no .apply() needed) + serverless_llm = LLM( + model="llama4-maverick-instruct-basic", + deployment_type="serverless" + ) + + # Create an on-demand deployment + ondemand_llm = LLM( + model="llama-v3p2-3b-instruct", + deployment_type="on-demand", + id="my-production-deployment", + max_instances=3, + min_instances=0 # Enable scale-to-zero + ) + + # Apply the on-demand deployment + ondemand_llm.apply() + + # 2. Query existing deployments + print("Connecting to existing deployment...") + existing_llm = LLM( + model="llama-v3p2-3b-instruct", + deployment_type="on-demand", + id="my-existing-deployment" + ) + + # 3. Use deployments + response = existing_llm.chat.completions.create( + messages=[{"role": "user", "content": "Explain deployment management"}] + ) + + # 4. Clean up (delete deployment when done) + # firectl delete deployment my-production-deployment + + return response.choices[0].message.content + +# Run the workflow +result = manage_deployments() +print(result) +\`\`\` + +## Best Practices + +1. **Cost Management**: Use serverless for development, on-demand for production +2. **Monitoring**: Track usage through the Fireworks dashboard +3. **Cleanup**: Always delete unused on-demand deployments to avoid charges +4. **Scaling**: Use scale-to-zero for cost efficiency +5. **Naming**: Use descriptive deployment IDs for easy management + +## Getting Started + +1. **Install the Build SDK**: \`pip install fireworks-ai\` +2. **Get API Key**: From https://app.fireworks.ai/dashboard/api-keys +3. **Choose Approach**: + - **Beginners**: Start with Build SDK + - **CLI Users**: Use firectl + - **API Integration**: Use REST API + +For enterprise needs or reserved capacity, contact inquiries@fireworks.ai. + +Would you like me to dive deeper into any specific aspect of deployment management?", + "$.messages[8].role": "assistant", + "$.pid": 97954, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:51:25.530495", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev19+gf755623.dirty", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "invent-special-real-rich-dream", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.pid": 2168, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T11:51:48.974779", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev19+gf755623.dirty", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "continue-encourage-dry-part-meeting", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.pid": 2642, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T12:17:53.864430", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "create-destroy-past-service-case", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.messages[2].content": "I'll help you learn about managing deployments on Fireworks! Let me search for comprehensive information about deployment management.", + "$.messages[2].role": "assistant", + "$.messages[2].tool_calls[0].function.arguments": "{"query": "deployment management Fireworks AI"}", + "$.messages[2].tool_calls[0].function.name": "search", + "$.messages[2].tool_calls[0].id": "functions.search:0", + "$.messages[2].tool_calls[0].type": "function", + "$.messages[3].content[0].text": "Title: Fireworks AI Developer Platform +Link: https://docs.fireworks.ai/getting-started/introduction +Content: Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Fireworks AI is the best platform for building AI product experiences with open source AI models. You can run and customize AI models with just a few lines of code! + +Start building with open source AI models + +", + "$.messages[3].content[0].type": "text", + "$.messages[3].content[1].text": "Title: How do I close my Fireworks.ai account? +Link: https://docs.fireworks.ai/faq-new/account-access/how-do-i-close-my-fireworksai-account +Content: To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +How do I close my Fireworks.ai account? +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +", + "$.messages[3].content[1].type": "text", + "$.messages[3].content[2].text": "Title: Do you host your deployments in the EU or Asia? +Link: https://docs.fireworks.ai/faq-new/support-general/do-you-host-your-deployments-in-the-eu-or-asia +Content: We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +Do you host your deployments in the EU or Asia? +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +", + "$.messages[3].content[2].type": "text", + "$.messages[3].content[3].text": "Title: Introducing the Fireworks Build SDK +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-introduction +Content: The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +", + "$.messages[3].content[3].type": "text", + "$.messages[3].content[4].text": "Title: On-demand deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +On-demand deployments +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +", + "$.messages[3].content[4].type": "text", + "$.messages[3].content[5].text": "Title: I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +Link: https://docs.fireworks.ai/faq-new/account-access/i-have-multiple-fireworks-accounts-when-i-try-to-login-with-google-on-fireworks +Content: If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +", + "$.messages[3].content[5].type": "text", + "$.messages[3].content[6].text": "Title: Introduction +Link: https://docs.fireworks.ai/api-reference/introduction +Content: Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +Introduction +Fireworks AI REST API enables you to interact with various language, image and embedding models using an API Key. It also lets you automate management of models, deployments, datasets, and more. + +", + "$.messages[3].content[6].type": "text", + "$.messages[3].content[7].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Create a Deployment on Fireworks AI platform + +Create a Deployment on Fireworks AI platform + +Creates a new deployment. + +", + "$.messages[3].content[7].type": "text", + "$.messages[3].content[8].text": "Title: How much does Fireworks cost? +Link: https://docs.fireworks.ai/faq-new/billing-pricing/how-much-does-fireworks-cost +Content: Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +How much does Fireworks cost? +Fireworks AI operates on a pay-as-you-go model for all non-Enterprise usage, and new users automatically receive free credits. You pay based on: Per token for serverless inference Per GPU usage time for on-demand deployments Per token of training data for fine-tuning For customers needing enterprise-grade security and reliability , please reach out to us at inquiries@fireworks.ai to discuss options. Find out more about our current pricing on our Pricing page. + +", + "$.messages[3].content[8].type": "text", + "$.messages[3].content[9].text": "Title: Python (Fireworks) +Link: https://docs.fireworks.ai/getting-started/quickstart +Content: This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +Set up your developer environment & call the Fireworks API +This is the recommended way to get started : This tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: Sending the first API Request You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. Fireworks AI platform offers drop-in replacement with OpenAI Python Client. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $OPENAI_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %OPENAI_API_KEY% + +", + "$.messages[3].content[9].type": "text", + "$.messages[3].role": "tool", + "$.messages[3].tool_call_id": "functions.search:0", + "$.messages[4].content": "Let me search for more specific information about deployment management commands and workflows.", + "$.messages[4].role": "assistant", + "$.messages[4].tool_calls[0].function.arguments": "{"query": "deployment management commands create update delete list Fireworks"}", + "$.messages[4].tool_calls[0].function.name": "search", + "$.messages[4].tool_calls[0].id": "functions.search:1", + "$.messages[4].tool_calls[0].type": "function", + "$.messages[5].content[0].text": "Title: Deployment Command +Link: https://docs.fireworks.ai/evaluators/examples/gcp_cloud_run_deployment_example +Content: It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: Run the deployment command: Command Explanation: --id my-dummy-gcp-evaluator : A unique ID for your evaluator on the Fireworks AI platform. --target gcp-cloud-run : Specifies deployment to GCP Cloud Run. --function-ref dummy_rewards.hello_world_reward : The Python import path to your reward function. If dummy_rewards.py is in the current directory, this reference works. --gcp-auth-mode api-key : Configures the Cloud Run service with API key authentication. reward-kit will generate a key, store it in GCP Secret Manager, and configure the service. The key is also saved to your local rewardkit.yaml under evaluator_endpoint_keys . This is the default if not specified. --verbose : Shows detailed output, including gcloud commands being executed. --force : (Optional) If an evaluator with the same --id already exists, this flag will delete the existing one before creating the new one. + +It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: Run the deployment command: Command Explanation: --id my-dummy-gcp-evaluator : A unique ID for your evaluator on the Fireworks AI platform. --target gcp-cloud-run : Specifies deployment to GCP Cloud Run. --function-ref dummy_rewards.hello_world_reward : The Python import path to your reward function. If dummy_rewards.py is in the current directory, this reference works. --gcp-auth-mode api-key : Configures the Cloud Run service with API key authentication. reward-kit will generate a key, store it in GCP Secret Manager, and configure the service. The key is also saved to your local rewardkit.yaml under evaluator_endpoint_keys . This is the default if not specified. --verbose : Shows detailed output, including gcloud commands being executed. --force : (Optional) If an evaluator with the same --id already exists, this flag will delete the existing one before creating the new one. + +It's recommended to run the deployment command from the directory containing the reward function script ( dummy_rewards.py ) and your rewardkit.yaml (if used), for example, from examples/gcp_cloud_run_deployment_example/. Ensure your virtual environment is active: source .venv/bin/activate + +", + "$.messages[5].content[0].type": "text", + "$.messages[5].content[1].text": "Title: I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +Link: https://docs.fireworks.ai/faq-new/account-access/i-have-multiple-fireworks-accounts-when-i-try-to-login-with-google-on-fireworks +Content: If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +", + "$.messages[5].content[1].type": "text", + "$.messages[5].content[2].text": "Title: Deleting Stored Responses +Link: https://docs.fireworks.ai/guides/response-api +Content: When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. from fireworks import LLM +import requests +import os + +llm = LLM(model="qwen3-235b-a22b", deployment_type="serverless") + +# Create a response +response = llm.responses.create( + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. from fireworks import LLM +import requests +import os + +llm = LLM(model="qwen3-235b-a22b", deployment_type="serverless") + +# Create a response +response = llm.responses.create( + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +When responses are stored (the default behavior with store=True ), you can immediately delete them from storage using the DELETE endpoint. This permanently removes the conversation data. import os +from openai import OpenAI +import requests + +client = OpenAI( + base_url="https://api.fireworks.ai/inference/v1", + api_key=os.getenv("FIREWORKS_API_KEY", "YOUR_FIREWORKS_API_KEY_HERE") +) + +# Create a response +response = client.responses.create( + model="accounts/fireworks/models/qwen3-235b-a22b", + input="What is the capital of France?", + store=True # This is the default +) + +response_id = response.id +print(f"Created response with ID: {response_id}") + +# Delete the response immediately +headers = { + "Authorization": f"Bearer {os.getenv('FIREWORKS_API_KEY')}", + "x-fireworks-account-id": "your-account-id" +} +delete_response = requests.delete( + f"https://api.fireworks.ai/inference/v1/responses/{response_id}", + headers=headers +) + +if delete_response.status_code == 200: + print("Response deleted successfully") +else: + print(f"Failed to delete response: {delete_response.status_code}") + +", + "$.messages[5].content[2].type": "text", + "$.messages[5].content[3].text": "Title: Delete Deployment +Link: https://docs.fireworks.ai/api-reference/delete-deployment +Content: + + + +REST Endpoint DELETE /v1/accounts/{account_id}/deployments/{deployment_id} + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" +deployment_id: + string + Description: The Deployment Id + Example: "" + +query params: +hard: + boolean + Description: If true, this will perform a hard deletion. + Example: true +ignoreChecks: + boolean + Description: If true, this will ignore checks and force the deletion of a deployment that is currently +deployed and is in use. + Example: true + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {} + +", + "$.messages[5].content[3].type": "text", + "$.messages[5].content[4].text": "Title: List Deployments +Link: https://docs.fireworks.ai/api-reference/list-deployments +Content: REST Endpoint GET /v1/accounts/{account_id}/deployments + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" + +query params: +pageSize: + integer + Description: The maximum number of deployments to return. The maximum page_size is 200, +values above 200 will be coerced to 200. +If unspecified, the default is 50. + Example: 123 +pageToken: + string + Description: A page token, received from a previous ListDeployments call. Provide this +to retrieve the subsequent page. When paginating, all other parameters +provided to ListDeployments must match the call that provided the page +token. + Example: "" +filter: + string + Description: Only deployment satisfying the provided filter (if specified) will be +returned. See https://google.aip.dev/160 for the filter grammar. + Example: "" +orderBy: + string + Description: A comma-separated list of fields to order by. e.g. "foo,bar" +The default sort order is ascending. To specify a descending order for a +field, append a " desc" suffix. e.g. "foo desc,bar" +Subfields are specified with a "." character. e.g. "foo.bar" +If not specified, the default order is by "create_time". + Example: "" +showDeleted: + boolean + Description: If set, DELETED deployments will be included. + Example: true +readMask: + string + Description: The fields to be returned in the response. If empty or "*", all fields will be returned. + Example: "" + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {"deployments":[{"name":"","displayName":"","description":"","createTime":"2023-11-07T05:31:56Z","expireTime":"2023-11-07T05:31:56Z","purgeTime":"2023-11-07T05:31:56Z","deleteTime":"2023-11-07T05:31:56Z","state":"STATE_UNSPECIFIED","status":{"code":"OK","message":""},"minReplicaCount":123,"maxReplicaCount":123,"replicaCount":123,"autoscalingPolicy":{"scaleUpWindow":"","scaleDownWindow":"","scaleToZeroWindow":"","loadTargets":{}},"baseModel":"","acceleratorCount":123,"acceleratorType":"ACCELERATOR_TYPE_UNSPECIFIED","precision":"PRECISION_UNSPECIFIED","cluster":"","enableAddons":true,"draftTokenCount":123,"draftModel":"","ngramSpeculationLength":123,"numPeftDeviceCached":123,"deploymentTemplate":"","autoTune":{"longPrompt":true},"placement":{"region":"REGION_UNSPECIFIED","multiRegion":"MULTI_REGION_UNSPECIFIED","regions":["REGION_UNSPECIFIED"]},"region":"REGION_UNSPECIFIED","updateTime":"2023-11-07T05:31:56Z","disableDeploymentSizeValidation":true}],"nextPageToken":"","totalSize":123} + +REST Endpoint GET /v1/accounts/{account_id}/deployments + +Server(s): https://api.fireworks.ai + + +Summary: + +Description: N/A + +Security methods: +header: + Authorization: bearer auth + + +Request params: +path params: +account_id: + string + Description: The Account Id + Example: "" + +query params: +pageSize: + integer + Description: The maximum number of deployments to return. The maximum page_size is 200, +values above 200 will be coerced to 200. +If unspecified, the default is 50. + Example: 123 +pageToken: + string + Description: A page token, received from a previous ListDeployments call. Provide this +to retrieve the subsequent page. When paginating, all other parameters +provided to ListDeployments must match the call that provided the page +token. + Example: "" +filter: + string + Description: Only deployment satisfying the provided filter (if specified) will be +returned. See https://google.aip.dev/160 for the filter grammar. + Example: "" +orderBy: + string + Description: A comma-separated list of fields to order by. e.g. "foo,bar" +The default sort order is ascending. To specify a descending order for a +field, append a " desc" suffix. e.g. "foo desc,bar" +Subfields are specified with a "." character. e.g. "foo.bar" +If not specified, the default order is by "create_time". + Example: "" +showDeleted: + boolean + Description: If set, DELETED deployments will be included. + Example: true +readMask: + string + Description: The fields to be returned in the response. If empty or "*", all fields will be returned. + Example: "" + + +Request body: N/A + + +Responses: +200: object + Description: N/A + Example: {"deployments":[{"name":"","displayName":"","description":"","createTime":"2023-11-07T05:31:56Z","expireTime":"2023-11-07T05:31:56Z","purgeTime":"2023-11-07T05:31:56Z","deleteTime":"2023-11-07T05:31:56Z","state":"STATE_UNSPECIFIED","status":{"code":"OK","message":""},"minReplicaCount":123,"maxReplicaCount":123,"replicaCount":123,"autoscalingPolicy":{"scaleUpWindow":"","scaleDownWindow":"","scaleToZeroWindow":"","loadTargets":{}},"baseModel":"","acceleratorCount":123,"acceleratorType":"ACCELERATOR_TYPE_UNSPECIFIED","precision":"PRECISION_UNSPECIFIED","cluster":"","enableAddons":true,"draftTokenCount":123,"draftModel":"","ngramSpeculationLength":123,"numPeftDeviceCached":123,"deploymentTemplate":"","autoTune":{"longPrompt":true},"placement":{"region":"REGION_UNSPECIFIED","multiRegion":"MULTI_REGION_UNSPECIFIED","regions":["REGION_UNSPECIFIED"]},"region":"REGION_UNSPECIFIED","updateTime":"2023-11-07T05:31:56Z","disableDeploymentSizeValidation":true}],"nextPageToken":"","totalSize":123} + + + +", + "$.messages[5].content[4].type": "text", + "$.messages[5].content[5].text": "Title: Publishing a deployed model +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: By default, models can only be queried by the account that owns them. To make a deployment public so anyone with a valid Fireworks API key can query it, update the deployed model with the --public flag. Find the Deployed Model ID Every model running on a deployment receives a unique deployed model ID. You can find this ID if you know the model name and deployment name using the following command: firectl list deployed-models --filter 'model="" AND deployment=""' + +By default, models can only be queried by the account that owns them. To make a deployment public so anyone with a valid Fireworks API key can query it, update the deployed model with the --public flag. Find the Deployed Model ID Every model running on a deployment receives a unique deployed model ID. You can find this ID if you know the model name and deployment name using the following command: firectl list deployed-models --filter 'model="" AND deployment=""' + +Managing default deployments +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: To delete a default deployment you must delete all other deployments for the same model first, or designate a different deployed model as the default as described above. This is to ensure that querying by model name will always route to an unambiguous default deployment as long as deployments for the model exist. + +", + "$.messages[5].content[5].type": "text", + "$.messages[5].content[6].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[5].content[6].type": "text", + "$.messages[5].content[7].text": "Title: How do I close my Fireworks.ai account? +Link: https://docs.fireworks.ai/faq-new/account-access/how-do-i-close-my-fireworksai-account +Content: To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +How do I close my Fireworks.ai account? +To close your account: Email inquiries@fireworks.ai Include in your request: Your account ID A clear request for account deletion Before closing your account, please ensure: All outstanding invoices are paid Any active deployments are terminated Important data is backed up if needed + +", + "$.messages[5].content[7].type": "text", + "$.messages[5].content[8].text": "Title: Introducing the Fireworks Build SDK +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-introduction +Content: Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +", + "$.messages[5].content[8].type": "text", + "$.messages[5].content[9].text": "Title: Python (Fireworks) +Link: https://docs.fireworks.ai/getting-started/quickstart +Content: This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +This is the recommended way to get started : This Python (Fireworks) tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. pip install --upgrade fireworks-ai The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: vim ~/.bash_profile vim ~/.zshrc Add a new line to the file with the following: export FIREWORKS_API_KEY="" After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. source ~/.bash_profile source ~/.zshrc You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. setx FIREWORKS_API_KEY "" To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: echo %FIREWORKS_API_KEY% You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) You can also pass the API key directly to the LLM constructor: LLM(model="llama4-maverick-instruct-basic", deployment_type="auto", api_key="") + +Set up your developer environment & call the Fireworks API +This is the recommended way to get started : This tab uses our Fireworks Build +SDK for the best performance and developer experience. +See our Client-side performance +optimization guide for +more details. Install SDK Before installing, ensure that you have the right version of Python installed. Optionally you might want to setup a virtual environment too. The Fireworks Build SDK provides a declarative way to work with Fireworks resources and is OpenAI API Compatible. Configure API Key Step-by-step instructions for setting an environment variable for respective OS platforms: Depending on your shell, you'll need to edit either ~/.bash_profile for Bash or ~/.zshrc for Zsh . +You can do this by running the command: Add a new line to the file with the following: After saving the file, you'll need to apply the changes by either restarting your terminal session or running depending on the file you edited. You can verify that the variable has been set correctly by running echo $FIREWORKS_API_KEY You can open Command Prompt by searching for it in the Windows search bar or by pressing Win + R, typing cmd, and pressing Enter. To verify that the variable has been set correctly, you can close and reopen Command Prompt and type: Sending the first API Request You can quickly instantiate the LLM class and call the Fireworks API. The Build SDK handles deployment management automatically. from fireworks import LLM + +# Basic usage - SDK automatically selects optimal deployment type +llm = LLM(model="llama4-maverick-instruct-basic", deployment_type="auto") + +response = llm.chat.completions.create( + messages=[{"role": "user", "content": "Say this is a test"}] +) + +print(response.choices[0].message.content) + +", + "$.messages[5].content[9].type": "text", + "$.messages[5].role": "tool", + "$.messages[5].tool_call_id": "functions.search:1", + "$.messages[6].content": "Let me search for more specific information about the firectl tool and deployment workflows.", + "$.messages[6].role": "assistant", + "$.messages[6].tool_calls[0].function.arguments": "{"query": "firectl deployment management commands create deployment update deployment"}", + "$.messages[6].tool_calls[0].function.name": "search", + "$.messages[6].tool_calls[0].id": "functions.search:0", + "$.messages[6].tool_calls[0].type": "function", + "$.messages[7].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[7].content[0].type": "text", + "$.messages[7].content[1].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. + +", + "$.messages[7].content[1].type": "text", + "$.messages[7].content[2].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[7].content[2].type": "text", + "$.messages[7].content[3].text": "Title: Managing default deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: firectl update deployed-model --default + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: firectl update deployed-model --default + +Publishing a deployed model +By default, models can only be queried by the account that owns them. To make a deployment public so anyone with a valid Fireworks API key can query it, update the deployed model with the --public flag. Find the Deployed Model ID Every model running on a deployment receives a unique deployed model ID. You can find this ID if you know the model name and deployment name using the following command: The first column in the output is the deployed model ID. Publish a deployed model To make a deployment public so anyone with a valid Fireworks API key can query it, update the deployed model with the --public flag. firectl update deployed-model --public + +", + "$.messages[7].content[3].type": "text", + "$.messages[7].content[4].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[7].content[4].type": "text", + "$.messages[7].content[5].text": "Title: Deploying a fine-tuned model using an on-demand deployment +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-models +Content: Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +firectl +firectl create dataset /path/to/jsonl/file + +", + "$.messages[7].content[5].type": "text", + "$.messages[7].content[6].text": "Title: Does Fireworks support custom base models? +Link: https://docs.fireworks.ai/faq-new/models-inference/does-fireworks-support-custom-base-models +Content: Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Does Fireworks support custom base models? +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +", + "$.messages[7].content[6].type": "text", + "$.messages[7].content[7].text": "Title: firectl +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-vlm +Content: # Create a deployment for your fine-tuned VLM +firectl create deployment my-custom-vlm + +# Check deployment status +firectl get deployment accounts/your-account/deployment/deployment-id + +# Create a deployment for your fine-tuned VLM +firectl create deployment my-custom-vlm + +# Check deployment status +firectl get deployment accounts/your-account/deployment/deployment-id + +firectl create sftj \\ + --base-model accounts/fireworks/models/qwen2p5-vl-32b-instruct \\ + --dataset my-vlm-dataset \\ + --output-model my-custom-vlm \\ + --epochs 3 For additional parameters like learning rates, evaluation datasets, and batch sizes, see Additional SFT job settings . + +", + "$.messages[7].content[7].type": "text", + "$.messages[7].content[8].text": "Title: firectl +Link: https://docs.fireworks.ai/models/quantization +Content: firectl create deployment --accelerator-type NVIDIA_H100_80GB --precision FP8 + +firectl create deployment --accelerator-type NVIDIA_H100_80GB --precision FP8 + +Creating an FP8 deployment +By default, creating a dedicated deployment will use the FP16 checkpoint. To see what precisions are available for a +model, run: get model + +", + "$.messages[7].content[8].type": "text", + "$.messages[7].content[9].text": "Title: Uploading the model locally (firectl) +Link: https://docs.fireworks.ai/models/uploading-custom-models +Content: To upload a custom base model, run the following command. firectl create model /path/to/files/ + +To upload a custom base model, run the following command. firectl create model /path/to/files/ + +To upload a custom base model, run the following command. + +", + "$.messages[7].content[9].type": "text", + "$.messages[7].role": "tool", + "$.messages[7].tool_call_id": "functions.search:0", + "$.pid": 12679, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738493", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 21 highlighted sections (required: 15)", + "$.evaluation_result.score": 1, + "$.ground_truth": "15", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "review-say-right-news-government", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write an outline for a paper on the history of Yemeni coffee. The outline should include the main points of the paper, and at least 15 sections should be highlighted with markdown such as *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "# Outline: The History of Yemeni Coffee + +1. **Introduction: Why Yemeni Coffee Matters** + - Thesis: Yemenā€™s role as the cradle of commercial coffee culture and its enduring legacy on global trade, ritual, and taste. + +2. *Pre-Coffee Yemen: Agriculture and Trade Routes* + - Overview of terraced farming and the incense & spice trade that primed the Red Sea economy. + +3. *The Myth and Botany of Coffea arabica* + - Ethiopian origin vs. Yemeni domestication; genetic evidence of Yemeni varietals. + +4. **Sufi Mystics and the First Coffeehouses** + - 15th-century Sufi orders in Mocha and TaŹæizz using *qahwa* for nocturnal devotions. + +5. *Mocha as the Worldā€™s First Coffee Port* + - Rise of the Red Sea entrepĆ“t; Ottoman tax farms and the *tujjar* merchant guilds. + +6. **Yemeni Cultivation Techniques: Terraces, Water, and Shade** + - Indigenous irrigation (*ghayl*), drought-resistant varietals, and the *mafrash* drying mats. + +7. *The 16th-Century Ottoman Coffee Ban and Smuggling Networks* + - How Meccaā€™s 1511 prohibition fueled clandestine Red Sea traffic and price inflation. + +8. **European Encounters: ā€œTurkishā€ Coffee as a Yemeni Product** + - Dutch, Venetian, and English merchants mislabeling Mocha beans; first price lists in Amsterdam (1616). + +9. *The Dutch East India Company and the First Seed Theft* + - Pieter van den Broeckeā€™s 1616 smuggling of Yemeni seeds to Bataviaā€”birth of Java coffee. + +10. **Yemenā€™s Monopoly Crumbles: 17thā€“18th-Century Rival Plantations** + - French at RĆ©union (Bourbon), Portuguese in Brazil, Spanish in Jamaicaā€”all tracing back to Mocha stock. + +11. *The Qasimi Era: Internal Trade, Taxes, and the Rise of the Ismaili Imamate* + - How highland imams leveraged coffee revenues to fund statecraft and architecture. + +12. **Decline in the 19th Century: British Aden and the Suez Canal** + - Shift of maritime traffic; competition from colonial plantations; collapse of Mochaā€™s mercantile elite. + +13. *Legacy in Cupping Notes: Mocha Java and Flavor Memory* + - Chocolatey, winey profiles; how Yemeni beans shaped early espresso blends. + +14. **Modern Revival: Specialty Coffee, NGOs, and Qima Coffee** + - 21st-century traceability, micro-lots, and direct-trade premiums; challenges of war and water scarcity. + +15. *Cultural Continuities: Qat vs. Qahwa, Wedding Roasts, and Oral Poetry* + - Daily rituals, gendered spaces, and the recitation of *zawamil* couplets over fresh brews. + +16. **Conclusion: Yemen as a Living Archive of Coffee Culture** + - Reasserting Yemenā€™s centrality in every cup; call for sustainable heritage preservation.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738539", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 26 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "install-fine-financial-health-state", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a cover letter for a job at a local coffee shop in the form of a poem. Highlight at least 5 text sections using "*". For example: *3 years of experience*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Dear *Steam & Bean* family, +I come to you not on paper feet but on *three years of espresso-stained sneakers*, +soles softened by dawn milk-foam and dusk drip-trays. +I have measured my mornings in *latte-art hearts*ā€” +some perfect, some *wobbly, but all poured with intention*. + +I know the *hiss of the steam wand* like a lullaby, +and the *clink of ceramic* like a metronome for the day. +I have *memorized the weight of 18 grams* of ground hope, +tamped it level, locked it tight, and watched *crema bloom* +the way a shy smile opens when someone says their name right. + +I speak *fluent caffeine*: +a triple-shot for the night-shift nurse, +oat-milk cappuccino for the poet who tips in verses, +decaf Americano for the retired teacher grading sunrise. +I *remember orders like birthdays*, +and birthdays like *extra whip, no charge*. + +I can *calibrate a grinder* faster than gossip travels, +and *unclog a portafilter* with the patience of a gardener. +I have *danced behind counters* two feet wide, +twirled between syrups and strangers, +and never spilled a single *thank-you*. + +If you need someone who *sweats kindness* and *smiles in ristretto*, +who counts tips not in coins but in *stories collected*, +I am your *next opening shift*, your *steady hand at 5:59 a.m.*, +your *last call who still says ā€œsee you tomorrowā€*. + +Let me *steam, pour, wipe, repeat*ā€” +until the chairs are upside-down and the *grinders sleep*. +I offer you *poetry in motion*, one cup at a time. + +With *grounds under my nails* and *hope in my apron*, +ā€”[Your Name]", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738569", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 5 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "should-poor-parent-war-magazine", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a casual blog post about similarities across animal species. Highlight at least 5 sections in your answer by starting and ending with "*", like: *highlighted text section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Hey there, fellow creature-curious readers! I just spent the weekend binge-watching nature docs with my cat (who was mostly unimpressed), and I canā€™t stop noticing how often totally different animals act like theyā€™re reading from the same playbook. So grab a snack and letā€™s zoom out for a secondā€”here are five ā€œwait, weā€™re not so differentā€ moments that made me laugh, gasp, and text my group chat at 2 a.m. + +*Moms are moms, whether they have feathers, fur, or fins.* +From emperor penguins passing an egg back and forth like the worldā€™s most stressful game of hot potato, to orangutan mamas teaching their kids which leaves make the best umbrellas, the universal rule seems to be: if youā€™re a parent, youā€™re tired. I watched a seahorse dad give birth (yes, the males do that!) and still look like he needed a nap and a latte. + +*Play isnā€™t just for puppies.* +Snowboarding crows, surfing dolphins, snow-rolling pandasā€”turns out ā€œbecause itā€™s funā€ is a legit evolutionary strategy. Scientists think play sharpens coordination and social skills, but honestly, watching a raven sled down a metal roof on a plastic lid just screams ā€œIā€™m doing it for the TikTok.ā€ + +*Grandma knows bestā€”even in elephant form.* +Elephant herds are led by the oldest female, and her decades of ā€œIā€™ve seen this beforeā€ wisdom decide when to migrate, where to find water, and how to handle that one teenage bull who thinks heā€™s all that. Swap gray skin for gray hair and youā€™ve basically got my nana directing Thanksgiving traffic. + +*Besties across species lines.* +Goby fish and pistol shrimp run the cutest roommate situation: the shrimp digs the burrow, the goby stands guard. If danger shows up, the goby flicks its tail like a silent alarm, and both dive to safety. Itā€™s the underwater version of ā€œIā€™ll cook, you do the dishes.ā€ + +*We all scream for sunscreenā€”sort of.* +Hippos secrete ā€œblood sweatā€ that acts like SPF 50, while giraffes drool black saliva that works like built-in sunglasses. Meanwhile, Iā€™m over here reapplying SPF 30 every 90 minutes and still ending up the color of a lobster. Natureā€™s got hacks, and Iā€™m taking notes. + +So next time you catch yourself thinking, ā€œIā€™m nothing like that weird-looking animal,ā€ remember: somewhere out there a capybara is sharing its snack with a bird, and a whale is singing a song that rhymes with last yearā€™s chart-topper. Weā€™re all just variations on a themeā€”some of us just have more legs or better night vision.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738594", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 5 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "discover-stop-show-different-water", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Generate a business proposal to start a sweatshirt company in Bremen. The proposal should contain 5 or more sections. Highlight each section name using the this format: +*section name*", + "$.messages[0].role": "user", + "$.messages[1].content": "*Executive Summary* +Bremen is a city of 570,000 residents, 40,000 university students, and a thriving creative scene, yet it lacks a home-grown premium sweatshirt brand that speaks to its maritime heritage, Hanseatic pride, and eco-conscious mindset. ā€œBremer Fleezeā€ will fill that gap by producing limited-edition, GOTS-certified organic-cotton sweatshirts that combine North-German minimalism with subtle nautical iconography. Our mission: to become Bremenā€™s first carbon-neutral apparel label and the go-to gift for locals and visitors alike. We will launch with an online-first D2C model, supported by pop-up stores in Viertel and Ɯberseestadt, and scale to ā‚¬1.2 M revenue by Year 3. + +*Market Analysis* +Target segments: (1) 18-30 students who value sustainability and streetwear aesthetics; (2) 25-45 professionals seeking premium basics; (3) tourists wanting authentic Bremen souvenirs. Competitor scan: global fast-fashion brands (H&M, Zara) lack local identity; German labels (Hugo Boss, Adidas) are either too formal or too sporty; no Bremen-specific sweatshirt exists. Market size: ā‚¬45 M annual spend on mid-premium sweatshirts in Bremen & Niedersachsen combined, growing 6 % YoY. Consumer pain points: greenwashing, poor fit, generic designs. Our edge: verifiable sustainability, Bremen storytelling, and inclusive sizing (XXS-4XL). + +*Product & Sustainability Strategy* +Core line: 12 unisex styles in 20 colors, 320 gsm brushed organic cotton, recycled polyester labels, water-based inks. Signature design elements: tonal outline of the Bremer Stadtmusikanten silhouette, latitude-longitude coordinates of Bremen harbor, and a discreet ā€œKohlfahrt-proofā€ patch. Packaging: reusable tote made from retired ship sails (partnership with local sailmaker). Carbon footprint: 2.1 kg COā‚‚e per sweatshirt vs. 8.9 kg industry average; offset via Gold-Standard wind-farm credits. End-of-life: take-back program with 15 % store credit incentive; returned garments upcycled into insulation for social housing. + +*Go-to-Market & Marketing Plan* +Phase 1 (Months 1-3): Pre-launch teaser on Instagram & TikTok (#BremerFleeze), micro-influencer seeding with 50 Bremen creators, waitlist goal 5,000 sign-ups. Phase 2 (Months 4-6): E-commerce launch on Shopify, same-day bike courier delivery within Bremen city limits, QR code on hangtag linking to AR filter that overlays the Roland statue wearing the sweatshirt. Phase 3 (Months 7-12): Pop-up container store at Marktplatz during Weihnachtsmarkt, collab with Beckā€™s Brewery for co-branded ā€œBremer Fleeze & Bremer Pilsā€ gift box. KPIs: CAC ā‰¤ ā‚¬18, repeat purchase rate ā‰„ 35 %, NPS ā‰„ 70. + +*Financial Projections & Funding Needs* +Start-up capital required: ā‚¬180,000. Breakdown: ā‚¬60k inventory (2,000 units), ā‚¬35k marketing, ā‚¬25k e-commerce & IT, ā‚¬20k legal & certifications, ā‚¬40k working capital. Revenue forecast: Y1 ā‚¬220k, Y2 ā‚¬550k, Y3 ā‚¬1.2 M. Gross margin 62 %, EBITDA margin 18 % by Y3. Break-even month 14. Funding structure: ā‚¬80k founder equity, ā‚¬50k Bremen-based angel investor (ex-Bremer Kaffee & Co-founder), ā‚¬50k zero-interest KfW Green Loan. Exit option: acquisition by sustainable German fashion conglomerate at 3Ɨ revenue multiple in Year 5.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738623", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 18 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "mean-certain-bitter-sharp-party", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a funny and sarcastic template for rating the quality of a marriage between two people who are both moms. This is for the couple themselves. Please highlight at least 3 sections with markdown, i.e *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "**The Official ā€œTwo-Mom Marriage Quality Controlā€ Checklist** +*Because nothing says romance like a quarterly performance review.* + +--- + +**1. Morning Routine Coordination** +Rate: ā˜ 1 (We need a UN mediator) ā˜ 2 (Weā€™ve agreed to disagreeā€¦ loudly) ā˜ 3 (Only one of us cried into the coffee) ā˜ 4 (Synchronized chaos) ā˜ 5 (*We finished each otherā€™s sentencesā€¦ and the toddlerā€™s homework*) + +*Note: Bonus points if youā€™ve mastered the ā€œIā€™ll pack lunches if you find the missing shoeā€ hostage negotiation.* + +--- + +**2. Division of Emotional Labor** +Rate: ā˜ 1 (ā€œIā€™m fine.ā€ *ā€” famous last words*) ā˜ 2 (We schedule meltdowns like dentist appointments) ā˜ 3 (Therapy? We *are* the therapy) ā˜ 4 (We tag-team existential dread while folding tiny socks) ā˜ 5 (*Weā€™ve achieved telepathic resentment-free snack negotiations*) + +*Pro tip: If youā€™ve ever passive-aggressively alphabetized the spice rack while humming ā€œYouā€™re Welcomeā€ from Moana, circle ā˜ 1 twice.* + +--- + +**3. Co-Parenting Synchronization** +Rate: ā˜ 1 (Our kids have unionized) ā˜ 2 (Weā€™ve agreed to disagree on screen timeā€¦ and bedtimeā€¦ and vegetables) ā˜ 3 (We bribe with different currencies: one uses stickers, the other uses Wi-Fi passwords) ā˜ 4 (Weā€™ve synchronized our ā€œmom voiceā€ to a frequency only dogs and children fear) ā˜ 5 (*Weā€™ve weaponized teamwork so effectively the kids think we share a brain*) + +*Highlight this if youā€™ve ever high-fived over successfully gaslighting a toddler into eating broccoli.* + +--- + +**Final Grade:** +ā˜ Certified Disaster (But at least the houseplants are alive) +ā˜ Functioning Dysfunction (Weā€™ve agreed the laundry mountain is *decorative*) +ā˜ Power Couple (Weā€™ve weaponized Pinterest and the kids still havenā€™t noticed) +ā˜ *Mom-tastic Nirvana* (We finish each otherā€™sā€¦ sandwiches. And trauma. But mostly sandwiches.) + +--- + +**Comments Section (for roasting each other lovingly):** +ā€œRemember when you ā€˜lostā€™ the Elf on the Shelf for three days? *Iconic.*ā€ +ā€œYour snoring sounds like a haunted dishwasher, but Iā€™d still share my last square of emergency chocolate with you.ā€", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738649", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 1 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "forecast-big-song-issue-service", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a song about the summers of my childhood that I spent in the countryside. Give the song a name, and highlight the name by wrapping it with *. For example: *little me in the countryside*.", + "$.messages[0].role": "user", + "$.messages[1].content": "*little me in the countryside* + +[Verse 1] +Barefoot on the gravel road, dust between my toes, +Sunlight dripped like honey on the freckles on my nose. +Grandmaā€™s porch swing creaked a lullaby in time, +While the crickets tuned their violins to the rhythm of the pines. + +[Pre-Chorus] +Mason jars with lightning bugs, constellations in a jar, +Weā€™d race the dusk to catch the dark before it got too far. + +[Chorus] +Oh, little me in the countryside, chasing dreams as wide as skies, +Jumping creeks and climbing trees, believing I could fly. +Every sunset was a promise that tomorrow would be mine, +In the summers of my childhood, where the world was kind. + +[Verse 2] +Hayloft hideouts, secret forts, straw castles in the loft, +We were pirates, we were astronauts, anything we thought. +Mamaā€™s voice would echo, ā€œSupper!ā€ across the open fields, +And the wind would carry laughter like the church bellā€™s gentle peals. + +[Pre-Chorus] +Fireflies wrote our names in light, then vanished in the night, +But the glow stayed in our hearts, a soft and steady light. + +[Chorus] +Oh, little me in the countryside, chasing dreams as wide as skies, +Jumping creeks and climbing trees, believing I could fly. +Every sunset was a promise that tomorrow would be mine, +In the summers of my childhood, where the world was kind. + +[Bridge] +Now the roads are paved, the porch is still, the swing donā€™t creak no more, +But I can close my eyes and hear the screen doorā€™s gentle roar. +Time may steal the years away, but it canā€™t outrun the pastā€” +I keep those golden afternoons like fireflies in a glass. + +[Final Chorus] +Oh, little me in the countryside, youā€™re never far behind, +Youā€™re the echo in the wheat fields, the shadow in my mind. +Though the world keeps spinning faster, I still find you every timeā€” +In the hush of dusk, in the scent of rain, in the summers of my childhood, +Where the world was kind.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738707", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 3 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "design-slow-wish-end-fear", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a riddle for the word "faƧade" that contains at least 3 italic text phrases in markdown syntax, i.e *italic text*.", + "$.messages[0].role": "user", + "$.messages[1].content": "I stand before you, yet Iā€™m not what I seem; +a *face I wear* that isnā€™t mine, +a *mask of stone* or paint or beam, +a *pretty lie* that keeps the truth confined. + +What am I?", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738738", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 19 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "consider-work-economic-popular-country", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a template for a workshop on the importance of diversity in the workplace and highlight at least 3 sections with markdown, i.e. *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "# Workshop Template +**Title:** *Building Inclusive Excellence: The Business Case for Workplace Diversity* + +--- + +### 1. Welcome & Objectives (15 min) +- **Ice-breaker:** ā€œTwo Truths & a Dreamā€ ā€“ participants share two facts and one aspiration related to identity or culture. +- *State the learning outcomes:* + 1. Understand how diversity drives innovation and profit. + 2. Identify unconscious bias and micro-behaviors. + 3. Create an action plan for inclusive leadership. + +--- + +### 2. *The Business Case for Diversity* (30 min) +- **Data Walk:** Posters with McKinsey, Deloitte, and Harvard studies showing ā†‘ revenue, ā†“ turnover. +- *Interactive poll:* ā€œWhich statistic surprised you most?ā€ (live Mentimeter). +- **Case Study:** *How a global tech firm increased patent filings by 19 % after diversifying R&D teams.* + +--- + +### 3. *Unconscious Bias & Micro-behaviors* (45 min) +- **Activity:** ā€œResume Rouletteā€ ā€“ identical CVs with different names; discuss callback disparities. +- *Bias Busters Toolkit:* 5-step process to interrupt bias in real time. +- **Role-play triads:** Observer, Speaker, Interrupter practicing inclusive language. + +--- + +### 4. Inclusive Leadership in Action (30 min) +- *Panel of ERG leaders* sharing measurable wins (e.g., 40 % ā†‘ in promotions for under-represented groups). +- **Speed-ideation:** 90-second pitches for inclusive policies; peer voting on top 3. + +--- + +### 5. Action Planning & Commitments (20 min) +- *Personal Inclusion Pledge:* Write one behavior to start, one to stop, one to amplify. +- **Accountability Circles:** Pair up for 30-day check-ins. + +--- + +### 6. Closing & Resources (10 min) +- **Resource QR Code:** Links to toolkits, podcasts, and upcoming ERG events. +- *Final reflection:* ā€œOne word to describe how youā€™ll lead differently tomorrow.ā€", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738766", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 28 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "predict-serve-messy-difficult-festival", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a cover letter to a local political party, asking to be their rally organizer. Make sure to highlight at least 3 sections in your answer in markdown format.", + "$.messages[0].role": "user", + "$.messages[1].content": "**[Your Name]** +[Your Address] +[City, State, ZIP] +[Email Address] | [Phone Number] | [Date] + +**[Local Party Chairā€™s Name]** +[Local Political Party Name] +[Party Headquarters Address] +[City, State, ZIP] + +--- + +### **Introduction: Why Iā€™m Writing** + +Dear **[Chairā€™s Name]** and the **Executive Committee**, + +I am writing to express my enthusiastic interest in serving as **Rally Organizer** for **[Local Party Name]**. As a lifelong resident of **[City/County]** and a committed volunteer who has knocked on over **2,000 doors** and organized **five successful voter-turnout events** in the last two cycles, I understand how to turn passion into turnout and energy into votes. I want to bring that same fireā€”and a proven playbookā€”to every rally our party holds between now and Election Day. + +--- + +### **Core Qualifications: What I Bring to the Podium** + +- **Logistics Mastery**: From securing permits with the Parks Department to negotiating stage-rental discounts, I have reduced average rally costs by **28 %** while increasing attendance by **45 %** through targeted social-media geofencing and SMS reminders. +- **Coalition Building**: I have already built relationships with **15 local unions**, **7 student groups**, and **4 faith communities** who are eager to co-sponsor rallies if given two weeksā€™ notice. +- **Crisis-Ready Leadership**: When an unexpected thunderstorm threatened our 2022 ā€œGet-Out-the-Voteā€ rally, I pivoted to a **drive-in format** in under four hoursā€”resulting in zero cancellations and 300 additional cars circling the block to hear our speakers. + +--- + +### **Vision for 2024: Turning Rallies into Results** + +My plan is to run a **ā€œRally-to-Ballot Pipelineā€**: + +1. **Pre-Rally**: Micro-targeted digital invites to low-propensity voters within a 5-mile radius. +2. **During Rally**: QR-code check-ins that instantly text attendees their personalized early-vote locations. +3. **Post-Rally**: A 72-hour follow-up cadence of thank-you emails, volunteer sign-up links, and ride-share codes to the polls. + +I have already drafted a **90-day rally calendar** (attached) that staggers high-energy weekend events with smaller, issue-specific pop-ups near early-vote sites. All I need is the partyā€™s green light and a modest budget line for permits, A/V, and refreshments. + +--- + +### **Closing: Letā€™s Win This Together** + +I would welcome the chance to present this plan in person at your next committee meeting. Thank you for considering my application. Letā€™s give our volunteers, candidates, andā€”most importantlyā€”our voters the rallies they deserve. + +**Ready to organize, energize, and mobilize,** +[Your Signature (if submitting hard copy)] +[Your Printed Name]", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738795", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 4 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "accept-expect-black-young-area", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "The opposite of youth is not age, but ...? Highlight at least 2 sections in your answer with markdown, i.e. *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "The opposite of youth is not age, but *stagnation*ā€”the moment curiosity closes its fist and wonder stops asking ā€œwhat if?ā€ +While *age* merely counts revolutions around the sun, *true youth* is the *capacity to keep revising the map of oneself* long after the hair turns silver.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738826", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 13 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "store-be-past-hard-war", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a description of the following data in a weird style: The Golden Palace eatType restaurant; The Golden Palace food Indian; The Golden Palace area city centre. Use markdown to highlight at least 3 sections in your answer.", + "$.messages[0].role": "user", + "$.messages[1].content": "**The Golden Palace** is not a palace at all, but a **restaurant** that has disguised itself as a mirage of saffron and cardamom in the **city centre**. It whispers **Indian** recipes into the ears of pigeons, who then forget how to fly and instead waddle around the square, dizzy with cumin. + +Inside, the walls sweat **garam masala** and the tables grow tiny **basil plants** that recite Bollywood dialogues when no one is looking. The waiters glide like **shadow puppets**, balancing trays of **butter chicken** that glow like radioactive sunsets. + +If you order **naan**, it arrives folded into the shape of a **paper swan**, flapping once before dissolving into steam that smells of **monsoon rain**. The bill is presented on a **lotus leaf**, and if you stare at it long enough, the numbers rearrange themselves into a **limerick about turmeric**.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738850", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 2 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "exploit-serve-invisible-community-minute", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a funny rap about a man who gets a call from an official saying that he is a long lost relative of the king of Nigeria. Use markdown to highlight at least one section of your answer, i.e. *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Yo, itā€™s MC Broke-Again, fresh outta ramen, +phone buzzinā€™ like a bee in a jar of gin. +Unknown numberā€”man, Iā€™m already suspicious, +but the voice got *royal timbre*, straight-up delicious. + +ā€œGreetings, Prince Darnell of House Wi-Fi, +Iā€™m the Royal Scribe from Lagos on high. +Your great-great-grand-uncle was the King of Nigeria, +left a vault of gold bars and a pet cheetah named Maria.ā€ + +Iā€™m like, ā€œHold upā€”me? Royalty? +Last week I pawned my TV for a two-piece and sweet tea.ā€ +He says, ā€œSign this PDF, wire a small advance fee, +just 419 bucks and the throne belongs to thee.ā€ + +**Iā€™m already picking out a crown on Amazon Prime, +ordering a robe that says ā€˜Fresh-to-Deathā€™ in gold slime.** + +I Venmo the cash, add a royal emojiā€”šŸ‘‘ +then sprint to the mirror, practice my ā€œMy people, be free!ā€ +Mom walks in, ā€œWhy you wrapped in bedsheets with a Burger King crown?ā€ +Iā€™m like, ā€œSilence, peasant! Bow before your prince of renown!ā€ + +Two days later, no gold, no cheetah, just crickets, +my bank account looking like a ghost town with limericks. +I call the numberā€”disconnected, tone dead, +even the scammer ghosted me harder than my ex on read. + +So Iā€™m back in the basement, eating discount sardines, +but I kept the crownā€”*itā€™s made of genuine Burger King plastic, supreme*.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738881", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 6 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "suggest-american-open-level-country", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Expand the riddle into a story with a funny tone: + +What can you catch but not throw? +A cold + +Use * to highlight at least 2 sections in your text. For example: *this is a highlighted text section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Gary the Germ was the *smallest, greenest, and most obnoxiously proud* microbe in the entire Sniffle Kingdom. Every morning he practiced his grand entrance: sliding down banisters made of nasal hair, cannon-balling into unsuspecting tonsils, and yelling, ā€œACHOOā€”did somebody order a cold?ā€ + +One Tuesday, Gary spotted his dream target: *Bartholomew ā€œBartā€ Bigglesworth, a man who believed hand-washing was a government conspiracy*. Bart had just exited the subway, triumphantly licking a subway pole ā€œfor the vitamins.ā€ Gary rubbed his microscopic hands together. ā€œPerfect. Iā€™ll hitch a ride on that finger, ride the express elevator to the sinuses, and throw the *snottiest* house-warming party in history!ā€ + +But there was a catchā€”literally. Gary could *catch* Bart, but he couldnā€™t *throw* him anywhere. Every time Gary tried to catapult Bart into the next office cubicle, Bart just sneezed, wiped his nose on his sleeve, and kept typing spreadsheets. Gary screamed, ā€œThis is mutiny! Iā€™m supposed to be the one doing the tossing!ā€ + +Meanwhile, Bartā€™s immune systemā€”led by Captain Phlegm and the Mucus Marinesā€”launched *Operation Tissue Storm*. They surrounded Gary, armed with vitamin C grenades and zinc zappers. Gary attempted a dramatic escape by surfing a sneeze, but the tissue slapped him like a flyswatter on a summer day. + +Defeated, Gary sat in the crumpled tissue, sulking. ā€œI caught the cold,ā€ he muttered, ā€œbut I canā€™t throw it anywhere. Worst. Superpower. Ever.ā€ + +From that day forward, Gary changed his business card to read: ā€œProfessional Cold Catcherā€”No Returns, No Refunds, Absolutely No Throwing.ā€", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738906", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 16 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "respond-connect-true-problem-plan", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a rap about the renaissance. It should be noticeably different from raps about other historical eras, and have an interesting or weird tone. Highlight at least 3 sections in your answer in markdown format.", + "$.messages[0].role": "user", + "$.messages[1].content": "**[Intro ā€“ Lute Drop]** +*beat drops like a sack of wet marble* +Yo, itā€™s 14-something, smell that? Thatā€™s perspective in the air, +Gutenberg just ghost-wrote the mixtape, pages everywhere. +Iā€™m a Medici money-mage, patron of the bizarre, +Commissioning nightmares in egg-tempera. + +--- + +### **Verse 1 ā€“ The Alchemistā€™s Cypher** +I paint with mercury, breathe with a lung of lead, +My selfieā€™s a death-mask, gold leaf on the dead. +Brunelleschi built a dome like a half-buried skull, +You think itā€™s architecture? Nah, itā€™s a ritual. +**[weird baroque wheeze]** +I rhyme in vanishing points, every line converges +On a single terrified eye that never emerges. +Perspectiveā€™s a ouija board, da Vinci holds the planchette, +Sketching choppers (helicopters) centuries before the jet-set. + +--- + +### **Hook ā€“ Chant of the Anamorphic Choir** +*distorted childrenā€™s voices layered over sackbut* +Renaissance, renaissance, put a cherub on it, +Baby wings, baby fangs, marble veins, vomit. +We donā€™t bling, we gild, we donā€™t rap, we frescoā€” +Your flowā€™s two-dimensions, mineā€™s *trompe-lā€™oeil* techno. + +--- + +### **Verse 2 ā€“ The Court Jesterā€™s Acid Madrigal** +**[sudden switch to a harpsichord trap beat]** +I juggle plague rats for the Duke of Milan, +He pays me in sonnets and unmarked arsenic flan. +Isabella dā€™Este texts me in mirror-script, +Her emoji: a decapitated hydra, lips pursed, licked. +Iā€™m the fool with a scalpel, dissecting songbirds to find +Which organ chirpsā€”turns out itā€™s the **pineal gland** of the mind. +Michelangeloā€™s in the corner, carving his own ribcage, +Says ā€œThe Sistineā€™s just a throat, Godā€™s a uvula on stage.ā€ + +--- + +### **Bridge ā€“ The Astrolabe Malfunction** +*beat glitches like a scratched illuminated manuscript* +Copernicus DMā€™d me: ā€œThe sunā€™s a strobe light, +Planets orbit like moths around a candle that bites.ā€ +I reply with a lute solo reversed through a cathedral, +Notes fall upward, monks levitate, itā€™s procedural. +**[static of Latin hex]** +Galileoā€™s telescope sees the future: TikTok, guillotine, +He mutters ā€œ*E pur si muove*ā€ then dabs in quarantine. + +--- + +### **Outro ā€“ The Anachronistic Masque** +**[all instruments devolve into wet clay sounds]** +So remember me when youā€™re double-tapping a Botticelli, +Iā€™m the glitch in the background, the snail with a human belly. +Renaissance never ended, it just learned to hide, +Under your touchscreen, oil-paint bleeding through the side. +Now goā€”vanish in a puff of ultramarine dust, +The past is a performance, and the futureā€™s already trussed.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738931", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 16 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "must-watch-bad-hate-magazine", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Summarize the history of Japan. Italicize at least 5 keywords in your response. To indicate a italic word, wrap it with asterisk, like *italic*", + "$.messages[0].role": "user", + "$.messages[1].content": "*Jōmon* hunter-gatherers (c. 14,000 BCE) gave way to *Yayoi* rice cultivators who introduced metal tools and social stratification. +The *Yamato* court unified much of the archipelago by the 5th century CE, absorbing *Shinto* beliefs and Chinese writing, law, and Buddhism. +A *Heian* golden age (794ā€“1185) produced literature like *The Tale of Genji* while the imperial courtā€™s real power waned. +Samurai clans rose; the *Kamakura* shogunate (1185ā€“1333) created Japanā€™s first warrior government, repelling Mongol invasions. +Centuries of civil war ended when *Oda Nobunaga*, *Toyotomi Hideyoshi*, and *Tokugawa Ieyasu* reunified the land, leading to the *Edo* shogunate (1603ā€“1868). +Isolation under *sakoku* preserved domestic culture until *Commodore Perry*ā€™s 1853 arrival forced unequal treaties. +The *Meiji* Restoration (1868) abolished feudalism, industrialized rapidly, and built an empire that won wars against China and Russia. +Imperial expansionā€”Korea, Manchuria, and *Pearl Harbor*ā€”ended in atomic bombings and occupation by Allied forces (1945ā€“1952). +Post-war *miracle* growth made Japan an economic superpower, followed by stagnation since the 1990s yet enduring global cultural influence.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738958", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 3 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "take-love-start-short-work", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a funny Haiku about a Quaker named John who lives in the town of De Smet, South Dakota. Use the asterisk symbol, *, to highlight some words or phrases twice. Example: *This is a highlighted phrase*.", + "$.messages[0].role": "user", + "$.messages[1].content": "John in *De Smet* +*Quaker* silenceā€”then he yells +ā€œWhereā€™s my *oatmeal*, Friend?ā€", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.738984", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 13 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "delete-democratic-hot-new-work", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Today, at the 54th Annual Grammy Awards, the Recording Academy honors the talent and creativity of the artists, musicians, and producers who are the creators of the best recordings of the past year. Please continue writing this text in a formal tone, using notations. Highlight some key parts in your response with "*", like *highlighted text*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Tonight, the 54th Annual Grammy Awards [1] stand as a testament to the enduring power of recorded music. Under the vaulted ceiling of the *Staples Center*, the Recording Academy gathers to bestow its highest accolades upon those whose artistry has defined the sonic landscape of eligibility year 2011 [2]. + +The eveningā€™s proceedings are governed by the *strictures of the Grammy Charter* [3], ensuring that every statuette awarded is the result of peer-review balloting among the Academyā€™s 12,000 voting members [4]. In accordance with Rule 3.01, only recordings commercially released in the United States between 1 October 2010 and 30 September 2011 are eligible [5]. + +Among the most anticipated categories is *Album of the Year* [6], whose nomineesā€”Adeleā€™s *21*, Foo Fightersā€™ *Wasting Light*, Lady Gagaā€™s *Born This Way*, Bruno Marsā€™ *Doo-Wops & Hooligans*, and Rihannaā€™s *Loud*ā€”represent a cross-section of genre, generation, and geography [7]. The *Record of the Year* [8] shortlist similarly spans continents and styles, underscoring the Academyā€™s commitment to excellence irrespective of commercial metrics. + +The *Producer of the Year, Non-Classical* [9] category spotlights the architects of sound whose meticulous craft transforms raw performance into timeless recording. Paul Epworth, whose work on *21* has garnered both critical and commercial acclaim, is widely regarded as the frontrunner [10]. + +As the telecast proceeds, the *In Memoriam* segment [11] will honor those luminaries whose voices were stilled during the eligibility year, their legacies enshrined in the grooves and bytes of the recordings they leave behind. + +The 54th Annual Grammy Awards thus serve not merely as a celebration, but as a *codification of excellence* [12], ensuring that the artistry of 2011 resonates through the decades to come.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.739013", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 33 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "live-important-military-only-love", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a blog post about interesting facts about the Dutch language. Italicize at least 2 sections in your answer with markdown, i.e. *italic text*.", + "$.messages[0].role": "user", + "$.messages[1].content": "# 7 Surprising Facts About Dutch That Will Make You Say ā€œWauw!ā€ + +When most people think of Dutch, they picture windmills, tulips, and maybe a stroopwafel or two. But the language behind those images is far strangerā€”and more fascinatingā€”than you might expect. Below are seven facts that prove Dutch is anything but *just English with extra gā€™s*. + +--- + +### 1. The Longest Word Is 53 Lettersā€”and Itā€™s Official +The Dutch *Encyclopedie* recognizes *aansprakelijkheidswaardevaststellingsveranderingen* (53 letters) as a legitimate word. It translates to ā€œchanges in liability value assessments,ā€ which is ironic because the word itself is a liability to pronounce. Try saying that three times fastā€”or even once slowly. + +--- + +### 2. Dutch Has a Word for *Everything*ā€”Including ā€œSchadenfreudeā€ but Cozier +English borrows *Schadenfreude* from German, but Dutch gives us *gezellig*, a term that packs ā€œcozy,ā€ ā€œfun,ā€ and ā€œbelongingā€ into one syllable. A cafĆ© can be *gezellig*, a party can be *gezellig*, and even a quiet evening with friends can be *gezellig*. Thereā€™s no direct English equivalent, which is why expats end up using it in English sentences: ā€œThat dinner was soā€¦ *gezellig*.ā€ + +--- + +### 3. The Dutch Alphabet Ends with ā€œX, Y, IJ, Zā€ +Most alphabets end with ā€œX, Y, Z,ā€ but Dutch sneaks in an extra character: *IJ*. Itā€™s considered a single letter (yes, two letters acting as one), and it even has its own capitalized form: *IJsselmeer*, the famous lake. Type designers have to create a special ligature for it, and crossword puzzles treat *IJ* as one square. Imagine Scrabble tiles clinking together as someone proudly lays down ā€œIJSBERGā€ for 72 points. + +--- + +### 4. Dutch Greetings Change by Time of Dayā€”But Not How Youā€™d Expect +In English, ā€œgood morningā€ fades into ā€œgood afternoonā€ around noon. Dutch, however, switches from *goedemorgen* to *goedemiddag* at exactly 12:00 p.m.ā€”unless youā€™re in Belgium, where some regions wait until 1:00 p.m. Miss the cutoff and youā€™ll get a polite correction: ā€œHet is al middag, hoor!ā€ (ā€œItā€™s already afternoon, you know!ā€). + +--- + +### 5. Dutch Is the Closest Major Language to Englishā€”But That Doesnā€™t Mean Itā€™s Easy +Linguists classify Dutch and English as *West Germanic siblings*, which means basic vocabulary overlaps: *appel* (apple), *boek* (book), *water* (water). Yet false friends lurk everywhere. *Durf jij?* doesnā€™t mean ā€œDo you dare?ā€ā€”it means ā€œDo you dare?ā€ā€¦ but with a cultural undertone of *prove it*. And *be careful* with *eventueel*, which means ā€œpossibly,ā€ not ā€œeventually.ā€ + +--- + +### 6. Dutch Has Three Gendersā€”But Only Two Articles +Grammatically, Dutch retains masculine, feminine, and neuter genders. In practice, however, only *de* (common gender) and *het* (neuter) are used. The result? Native speakers themselves debate whether *auto* (car) is masculine or feminine. The safest route is to just say *de auto* and avoid the awkward silence that follows gendered guesswork. + +--- + +### 7. Dutch Invented the Word ā€œCookieā€ā€”and the World Never Looked Back +The English word ā€œcookieā€ comes from the Dutch *koekje* (ā€œlittle cakeā€). Dutch settlers brought *koekjes* to New Amsterdam (now New York), and the rest is snack history. Next time you dunk a chocolate chip, thank the Dutch for their linguisticā€”and culinaryā€”contribution. + +--- + +### Bonus: Dutch Tongue Twisters Are Next-Level +Try this: *De kat krabt de krullen van de trap*. Translation: ā€œThe cat scratches the curls off the stairs.ā€ Say it five times without twisting your tongue into a *stroopwafel*. + +--- + +Whether youā€™re planning a trip to Amsterdam or just want to impress friends at trivia night, Dutch offers a treasure trove of quirks. From *gezellig* evenings to 53-letter bureaucratic monsters, the language proves that small countries can produce *enormous linguistic personality*.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:30:54.739037", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "hear-live-common-late-hate", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 2 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "think-indicate-write-local-company", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Gideon is a farmer who has a surplus of crops from his farm this year. What might he do with that surplus? Highlight at least one section of your answer in markdown, i.e *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Gideon has several good options for his surplus: + +1. **Sell it at the local farmersā€™ market or to a regional wholesaler**, turning the extra produce into cash that he can use for next seasonā€™s seeds, equipment repairs, or household expenses. +2. *Donate a portion to the nearby food bank or community kitchen*, helping families who struggle with food insecurity while also earning a small tax deduction. +3. Preserve some of the surplusā€”by canning tomatoes, drying herbs, or storing root vegetables in a cool cellarā€”so his own family has nutritious food throughout the winter. +4. Trade with neighboring farmers: swap excess corn for someone elseā€™s surplus eggs or honey, strengthening community ties and diversifying his pantry without spending money.", + "$.messages[1].role": "assistant", + "$.pid": 52072, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-08T23:34:12.823862", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.run_id": "meet-treat-central-possible-lot", + "$.eval_metadata.status": "stopped", + "$.eval_metadata.version": "v0.2.6.dev20+ge1da45a.dirty", + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "expect-upload-close-president-goal", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.pid": 52715, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:10:34.081457", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": "Run math evaluation on sample dataset using pytest interface.", + "$.eval_metadata.name": "test_pytest_mcp_url", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.run_id": "replace-measure-bitter-open-reason", + "$.eval_metadata.status": "finished", + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "At least one tool call was made", + "$.evaluation_result.score": 1, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.row_id": "run-update-traditional-software-idea", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "You are a helpful assistant that can answer questions about Fireworks. +ALWAYS provide code or commands to execute to answer the question.", + "$.messages[0].role": "system", + "$.messages[1].content": "Can you teach me about how to manage deployments on Fireworks", + "$.messages[1].role": "user", + "$.messages[2].content": "I'll help you learn about managing deployments on Fireworks. Let me search for comprehensive information about deployment management.", + "$.messages[2].role": "assistant", + "$.messages[2].tool_calls[0].function.arguments": "{"query": "deployment management Fireworks"}", + "$.messages[2].tool_calls[0].function.name": "search", + "$.messages[2].tool_calls[0].id": "functions.search:0", + "$.messages[2].tool_calls[0].type": "function", + "$.messages[3].content[0].text": "Title: On-demand deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +On-demand deployments +If you plan on using a significant amount of dedicated deployments, consider purchasing reserved capacity instead of using on-demand deployments for more reliable capacity and higher GPU quotas. Fireworks allows you to create on-demand deployments of models that are reserved for your own use and billed by the GPU-second. This has several advantages over the shared deployment architecture you get when using Fireworks' serverless models: Predictable performance unaffected by load caused by other users No hard rate limits, but subject to the maximum load capacity of the deployment Cheaper under high utilization Access to larger selection of models not available via our serverless models Custom base models from Hugging Face files Unlike serverless models, which are billed on a per-token basis, on-demand deployments are billed by GPU-second. Consult our pricing page for details. + +", + "$.messages[3].content[0].type": "text", + "$.messages[3].content[1].text": "Title: Does Fireworks support custom base models? +Link: https://docs.fireworks.ai/faq-new/models-inference/does-fireworks-support-custom-base-models +Content: Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Does Fireworks support custom base models? +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +", + "$.messages[3].content[1].type": "text", + "$.messages[3].content[2].text": "Title: Introducing the Fireworks Build SDK +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-introduction +Content: Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +Introducing the Fireworks Build SDK +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +The Fireworks Build SDK is a client library that allows you to interact with the Fireworks API using Python. It provides a simple and intuitive interface for working with Fireworks primitives like deployments, fine-tuning jobs, and datasets as Python objects. The Build SDK is currently in beta and not all functionality may be supported. Please reach out to dhuang@fireworks.ai to report any issues or feedback. + +", + "$.messages[3].content[2].type": "text", + "$.messages[3].content[3].text": "Title: I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +Link: https://docs.fireworks.ai/faq-new/account-access/i-have-multiple-fireworks-accounts-when-i-try-to-login-with-google-on-fireworks +Content: If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +I have multiple Fireworks accounts. When I try to login with Google on Fireworks' web UI, I'm getting signed into the wrong account. How do I fix this? +If you log in with Google, account management is controlled by Google. You can log in through an incognito mode or create separate Chrome/browser profiles to log in with different Google accounts. You could also follow the steps in this guide to disassociate Fireworks.ai with a particular Google account sign-in. If you have more complex issues please contact us on Discord. + +", + "$.messages[3].content[3].type": "text", + "$.messages[3].content[4].text": "Title: Querying existing dedicated deployments +Link: https://docs.fireworks.ai/tools-sdks/python-client/querying-dedicated-deployments +Content: When you have dedicated deployments that were created via firectl or the Fireworks web UI, you can easily connect to them using the Build SDK to run inference. This is particularly useful when you want to leverage existing infrastructure or when deployments are managed by different teams. + +When you have dedicated deployments that were created via firectl or the Fireworks web UI, you can easily connect to them using the Build SDK to run inference. This is particularly useful when you want to leverage existing infrastructure or when deployments are managed by different teams. + +Deployment ID requirements +The id parameter should match exactly with your existing deployment: Use the deployment name/ID as shown in the Fireworks dashboard The ID is case-sensitive and must match exactly If the deployment doesn't exist, you'll receive an error when making requests + +", + "$.messages[3].content[4].type": "text", + "$.messages[3].content[5].text": "Title: Migration and traffic management +Link: https://docs.fireworks.ai/guides/prompt-caching +Content: When migrating between deployments that use prompt caching, it's crucial to implement proper traffic routing to maintain optimal cache hit rates. When gradually routing traffic to a new deployment, use consistent user/session-based sampling rather than random sampling. Here's the recommended implementation for traffic routing: import hashlib + +# Configure traffic fraction (e.g., 20% to new deployment) +fireworks_traffic_fraction = 0.2 +user_id = "session-id-123" + +# Generate deterministic hash from user_id +hashed_user_id = int(hashlib.md5(user_id.encode()).hexdigest(), 16) # MD5 hash on user-id and convert to integer +MAX_HASH = 2**128 - 1 # MD5 hash maximum value + +# Compute ratio for consistent routing +ratio = hashed_user_id / MAX_HASH # Returns 0.0 to 1.0 + +if (ratio < fireworks_traffic_fraction): + send_to_new_deployment(user=hashed_user_id) # Pass user ID for caching +else: + send_elsewhere() # Route to old deployment or serverless + +When migrating between deployments that use prompt caching, it's crucial to implement proper traffic routing to maintain optimal cache hit rates. When gradually routing traffic to a new deployment, use consistent user/session-based sampling rather than random sampling. Here's the recommended implementation for traffic routing: import hashlib + +# Configure traffic fraction (e.g., 20% to new deployment) +fireworks_traffic_fraction = 0.2 +user_id = "session-id-123" + +# Generate deterministic hash from user_id +hashed_user_id = int(hashlib.md5(user_id.encode()).hexdigest(), 16) # MD5 hash on user-id and convert to integer +MAX_HASH = 2**128 - 1 # MD5 hash maximum value + +# Compute ratio for consistent routing +ratio = hashed_user_id / MAX_HASH # Returns 0.0 to 1.0 + +if (ratio < fireworks_traffic_fraction): + send_to_new_deployment(user=hashed_user_id) # Pass user ID for caching +else: + send_elsewhere() # Route to old deployment or serverless + +Advanced: cache locality for Enterprise deployments +Dedicated deployments on an Enterprise plan allow you to pass an additional hint in the request to improve cache hit rates. First, the deployment needs to be created or updated with an additional flag: Then the client can pass an opaque identifier representing a single user or +session in the user field of the body or in the x-session-affinity header. Fireworks +will try to route requests with the identifier to the same server, further reducing response times. It's best to choose an identifier that groups requests with long shared prompt +prefixes. For example, it can be a chat session with the same user or an +assistant working with the same shared context. + +", + "$.messages[3].content[5].type": "text", + "$.messages[3].content[6].text": "Title: Do you host your deployments in the EU or Asia? +Link: https://docs.fireworks.ai/faq-new/support-general/do-you-host-your-deployments-in-the-eu-or-asia +Content: Do you host your deployments in the EU or Asia? +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +Do you host your deployments in the EU or Asia? +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +We are currently deployed in multiple U.S.-based locations. However, weā€™re open to hearing more about your specific requirements. You can: Join our Discord community Write to us at inquiries@fireworks.ai If you're an Enterprise customer, please contact your dedicated customer support representative to ensure a timely response. + +", + "$.messages[3].content[6].type": "text", + "$.messages[3].content[7].text": "Title: Deployments and deployment types +Link: https://docs.fireworks.ai/getting-started/concepts +Content: A model must be deployed before it can be used for inference. A deployment is a collection (one or more) model servers that host one base model and optionally one or more LoRA addons. Fireworks supports two types of deployments: Serverless deployments: Fireworks hosts popular base models on shared "serverless" deployments. Users pay-per-token to query these models and do not need to configure GPUs. The most popular serverless deployments also support serverless LoRA addons. See the Deploying to serverless guide for details. Dedicated deployments: Dedicated deployments enable users to configure private deployments with a wide array of hardware (see on-demand deployments guide ). Dedicated deployments give users performance guarantees and the most flexibility and control over what models can be deployed. Both LoRA addons and base models can be deployed to dedicated deployments. Dedicated deployments are billed by a GPU-second basis (see pricing page). See the Querying text models guide for a comprehensive overview of making LLM inference. + +A model must be deployed before it can be used for inference. A deployment is a collection (one or more) model servers that host one base model and optionally one or more LoRA addons. Fireworks supports two types of deployments: Serverless deployments: Fireworks hosts popular base models on shared "serverless" deployments. Users pay-per-token to query these models and do not need to configure GPUs. The most popular serverless deployments also support serverless LoRA addons. See the Deploying to serverless guide for details. Dedicated deployments: Dedicated deployments enable users to configure private deployments with a wide array of hardware (see on-demand deployments guide ). Dedicated deployments give users performance guarantees and the most flexibility and control over what models can be deployed. Both LoRA addons and base models can be deployed to dedicated deployments. Dedicated deployments are billed by a GPU-second basis (see pricing page). See the Querying text models guide for a comprehensive overview of making LLM inference. + +Control plane and data plane +The Fireworks API can be split into a control plane and a data plane. The control plane consists of APIs used for managing the lifecycle of resources. This +includes your account, models, and deployments. The data plane consists of the APIs used for inference and the backend services that power +them. + +", + "$.messages[3].content[7].type": "text", + "$.messages[3].content[8].text": "Title: Deployment Process +Link: https://docs.fireworks.ai/evaluators/api_reference/reward_function_decorator +Content: When .deploy() is called, the decorator: Extracts the function's source code Creates a wrapper that handles the Fireworks evaluation format Creates a temporary directory with the wrapped function Uploads and registers the function with the Fireworks API + +When .deploy() is called, the decorator: Extracts the function's source code Creates a wrapper that handles the Fireworks evaluation format Creates a temporary directory with the wrapped function Uploads and registers the function with the Fireworks API + +.deploy() +The decorator adds a .deploy() method to the function, allowing it to be deployed to Fireworks. evaluation_id = my_reward_function.deploy( + name="my-evaluator", + description="Evaluates responses based on clarity and accuracy", + account_id=None, # Optional, defaults to configured account + auth_token=None, # Optional, defaults to configured token + force=False, # Set to True to overwrite if it already exists + providers=None # Optional model providers configuration +) + +", + "$.messages[3].content[8].type": "text", + "$.messages[3].content[9].text": "Title: šŸ”§ Build SDK: Deployment Configuration Application Requirement +Link: https://docs.fireworks.ai/updates/changelog +Content: The Build SDK now requires you to call .apply() to apply any deployment configurations to Fireworks when using deployment_type="on-demand" or deployment_type="on-demand-lora" . This change ensures explicit control over when deployments are created and helps prevent accidental deployment creation. Key changes: .apply() is now required for on-demand and on-demand-lora deployments Serverless deployments do not require .apply() calls If you do not call .apply() , you are expected to set up the deployment through the deployment page at https://app.fireworks.ai/dashboard/deployments Migration guide: Add llm.apply() after creating LLM instances with deployment_type="on-demand" or deployment_type="on-demand-lora" No changes needed for serverless deployments See updated documentation for examples and best practices This change improves deployment management and provides better control over resource creation. This applies to Python SDK version >=0.19.14. + +The Build SDK now requires you to call .apply() to apply any deployment configurations to Fireworks when using deployment_type="on-demand" or deployment_type="on-demand-lora" . This change ensures explicit control over when deployments are created and helps prevent accidental deployment creation. Key changes: .apply() is now required for on-demand and on-demand-lora deployments Serverless deployments do not require .apply() calls If you do not call .apply() , you are expected to set up the deployment through the deployment page at https://app.fireworks.ai/dashboard/deployments Migration guide: Add llm.apply() after creating LLM instances with deployment_type="on-demand" or deployment_type="on-demand-lora" No changes needed for serverless deployments See updated documentation for examples and best practices This change improves deployment management and provides better control over resource creation. This applies to Python SDK version >=0.19.14. + +šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[3].content[9].type": "text", + "$.messages[3].role": "tool", + "$.messages[3].tool_call_id": "functions.search:0", + "$.messages[4].content": "Let me search for more specific information about deployment management tools and processes.", + "$.messages[4].role": "assistant", + "$.messages[4].tool_calls[0].function.arguments": "{"query": "firectl deployment management commands"}", + "$.messages[4].tool_calls[0].function.name": "search", + "$.messages[4].tool_calls[0].id": "functions.search:1", + "$.messages[4].tool_calls[0].type": "function", + "$.messages[5].content[0].text": "Title: Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +Link: https://docs.fireworks.ai/faq-new/fine-tuning/why-am-i-getting-invalid-id-errors-when-using-firectl-commands-like-create-deplo +Content: This error typically occurs when your account ID is not properly configured. + +This error typically occurs when your account ID is not properly configured. + +Why am I getting "invalid id" errors when using firectl commands like create deployment or list deployments? +This error typically occurs when your account ID is not properly configured. + +", + "$.messages[5].content[0].type": "text", + "$.messages[5].content[1].text": "Title: Managing default deployments +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: firectl get model + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: firectl get model + +Since a model may be deployed to multiple deployments, querying by model name will route to the "default" deployed model. You can see which deployed model entry is marked with Default: true using: This will show the Deployed Model Refs section with the Default: true entry. To update the default deployed model, note the Name of the deployed model reference above. Then run: firectl update deployed-model --default + +", + "$.messages[5].content[1].type": "text", + "$.messages[5].content[2].text": "Title: Does Fireworks support custom base models? +Link: https://docs.fireworks.ai/faq-new/models-inference/does-fireworks-support-custom-base-models +Content: Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +Does Fireworks support custom base models? +Yes, custom base models can be deployed via firectl . You can learn more about custom model deployment in our guide on uploading custom models. + +", + "$.messages[5].content[2].type": "text", + "$.messages[5].content[3].text": "Title: šŸš€ Easier & faster LoRA fine-tune deployments on Fireworks +Link: https://docs.fireworks.ai/updates/changelog +Content: You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: firectl create deployment "accounts/fireworks/models/" + +You can now deploy a LoRA fine-tune with a single command and get speeds that approximately match the base model: Previously, this involved two distinct steps, and the resulting deployment was slower than the base model: Create a deployment using firectl create deployment "accounts/fireworks/models/" --enable-addons Then deploy the addon to the deployment: firectl load-lora --deployment For more information, see our deployment documentation. This change is for dedicated deployments with a single LoRA. You can still deploy multiple LoRAs on a deployment or deploy LoRA(s) on some Serverless models as described in the documentation. + +", + "$.messages[5].content[3].type": "text", + "$.messages[5].content[4].text": "Title: Quick deployment +Link: https://docs.fireworks.ai/fine-tuning/single-lora +Content: Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Deploy your LoRA fine-tuned model with one simple command: firectl create deployment "accounts/fireworks/models/" + +Alternative deployment method +This two-step method is the standard approach for multi-LoRA deployments where multiple LoRA models share the same base model. While it can also be used for single LoRA deployments, it provides slower performance compared to live merge and is not recommended for single LoRA use cases. You can also deploy single LoRA models using a two-step process: Create base model deployment Deploy the base model with addons enabled: firectl create deployment "accounts/fireworks/models/" --enable-addons + +", + "$.messages[5].content[4].type": "text", + "$.messages[5].content[5].text": "Title: Deploying a fine-tuned model using an on-demand deployment +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-models +Content: Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Use the following command to deploy your fine-tuned model using an on-demand deployment: firectl create deployment + +Deploying a fine-tuned model serverlessly +Not all base models support serverless addons. Please check this list for the full list of serverless models that support LoRA add-ons. If your base model supports serverless addons, you can run the following command to load the fine-tuned model into a serverless deployment: firectl load-lora + +", + "$.messages[5].content[5].type": "text", + "$.messages[5].content[6].text": "Title: Deploy Command +Link: https://docs.fireworks.ai/evaluators/cli_reference/cli_overview +Content: The deploy command deploys a reward function as an evaluator on the Fireworks platform. + +The deploy command deploys a reward function as an evaluator on the Fireworks platform. + +Command Overview +The Reward Kit CLI supports the following main commands: run : Run a local evaluation pipeline using a Hydra configuration. preview : Preview evaluation results or re-evaluate generated outputs. deploy : Deploy a reward function as an evaluator. agent-eval : Run agent evaluations on task bundles. list : List existing evaluators (coming soon). delete : Delete an evaluator (coming soon). + +", + "$.messages[5].content[6].type": "text", + "$.messages[5].content[7].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. firectl create deployment [flags] + +Creates a new deployment. firectl create deployment [flags] + +Example +firectl create deployment falcon-7b + +", + "$.messages[5].content[7].type": "text", + "$.messages[5].content[8].text": "Title: firectl +Link: https://docs.fireworks.ai/fine-tuning/fine-tuning-vlm +Content: # Create a deployment for your fine-tuned VLM +firectl create deployment my-custom-vlm + +# Check deployment status +firectl get deployment accounts/your-account/deployment/deployment-id + +# Create a deployment for your fine-tuned VLM +firectl create deployment my-custom-vlm + +# Check deployment status +firectl get deployment accounts/your-account/deployment/deployment-id + +firectl create sftj \\ + --base-model accounts/fireworks/models/qwen2p5-vl-32b-instruct \\ + --dataset my-vlm-dataset \\ + --output-model my-custom-vlm \\ + --epochs 3 For additional parameters like learning rates, evaluation datasets, and batch sizes, see Additional SFT job settings . + +", + "$.messages[5].content[8].type": "text", + "$.messages[5].content[9].text": "Title: Uploading the model locally (firectl) +Link: https://docs.fireworks.ai/models/uploading-custom-models +Content: To upload a custom base model, run the following command. firectl create model /path/to/files/ + +To upload a custom base model, run the following command. firectl create model /path/to/files/ + +To upload a custom base model, run the following command. + +", + "$.messages[5].content[9].type": "text", + "$.messages[5].role": "tool", + "$.messages[5].tool_call_id": "functions.search:1", + "$.messages[6].content": "Let me search for more information about deployment lifecycle management and monitoring.", + "$.messages[6].role": "assistant", + "$.messages[6].tool_calls[0].function.arguments": "{"query": "deployment monitoring scaling management lifecycle"}", + "$.messages[6].tool_calls[0].function.name": "search", + "$.messages[6].tool_calls[0].id": "functions.search:0", + "$.messages[6].tool_calls[0].type": "function", + "$.messages[7].content[0].text": "Title: What should I expect for deployment and scaling performance? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/what-should-i-expect-for-deployment-and-scaling-performance +Content: Initial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long + +Initial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long + +What should I expect for deployment and scaling performance? +Initial deployment : Should complete within minutes Scaling from zero : You may experience brief availability delays while the system scales up Troubleshooting : If deployment takes over 1 hour, this typically indicates a crash and should be investigated Best practice : Monitor deployment status and contact support if deployment times are unusually long + +", + "$.messages[7].content[0].type": "text", + "$.messages[7].content[1].text": "Title: How does billing and scaling work for on-demand GPU deployments? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/how-does-billing-and-scaling-work-for-on-demand-gpu-deployments +Content: On-demand GPU deployments have unique billing and scaling characteristics compared to serverless deployments: Billing: Charges start when the server begins accepting requests Billed by GPU-second for each active instance Costs accumulate even if there are no active API calls Scaling options: Supports autoscaling from 0 to multiple GPUs Each additional GPU adds to the billing rate Can handle unlimited requests within the GPUā€™s capacity Management requirements: Not fully serverless; requires some manual management Manually delete deployments when no longer needed Or configure autoscaling to scale down to 0 during inactive periods Cost control tips: Regularly monitor active deployments Delete unused deployments to avoid unnecessary costs Consider serverless options for intermittent usage Use autoscaling to 0 to optimize costs during low-demand times + +On-demand GPU deployments have unique billing and scaling characteristics compared to serverless deployments: Billing: Charges start when the server begins accepting requests Billed by GPU-second for each active instance Costs accumulate even if there are no active API calls Scaling options: Supports autoscaling from 0 to multiple GPUs Each additional GPU adds to the billing rate Can handle unlimited requests within the GPUā€™s capacity Management requirements: Not fully serverless; requires some manual management Manually delete deployments when no longer needed Or configure autoscaling to scale down to 0 during inactive periods Cost control tips: Regularly monitor active deployments Delete unused deployments to avoid unnecessary costs Consider serverless options for intermittent usage Use autoscaling to 0 to optimize costs during low-demand times + +How does billing and scaling work for on-demand GPU deployments? +On-demand GPU deployments have unique billing and scaling characteristics compared to serverless deployments: Billing: Charges start when the server begins accepting requests Billed by GPU-second for each active instance Costs accumulate even if there are no active API calls Scaling options: Supports autoscaling from 0 to multiple GPUs Each additional GPU adds to the billing rate Can handle unlimited requests within the GPUā€™s capacity Management requirements: Not fully serverless; requires some manual management Manually delete deployments when no longer needed Or configure autoscaling to scale down to 0 during inactive periods Cost control tips: Regularly monitor active deployments Delete unused deployments to avoid unnecessary costs Consider serverless options for intermittent usage Use autoscaling to 0 to optimize costs during low-demand times + +", + "$.messages[7].content[1].type": "text", + "$.messages[7].content[2].text": "Title: How does billing work for on-demand deployments? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/how-does-billing-work-for-on-demand-deployments +Content: On-demand deployments come with automatic cost optimization features: Default autoscaling : Automatically scales to 0 replicas when not in use Pay for what you use : Charged only for GPU time when replicas are active Flexible configuration : Customize autoscaling behavior to match your needs Best practices for cost management: Leverage default autoscaling : The system automatically scales down deployments when not in use Customize carefully : While you can modify autoscaling behavior using our configuration options , note that preventing scale-to-zero will result in continuous GPU charges Consider your use case : For intermittent or low-frequency usage, serverless deployments might be more cost-effective For detailed configuration options, see our deployment guide. + +On-demand deployments come with automatic cost optimization features: Default autoscaling : Automatically scales to 0 replicas when not in use Pay for what you use : Charged only for GPU time when replicas are active Flexible configuration : Customize autoscaling behavior to match your needs Best practices for cost management: Leverage default autoscaling : The system automatically scales down deployments when not in use Customize carefully : While you can modify autoscaling behavior using our configuration options , note that preventing scale-to-zero will result in continuous GPU charges Consider your use case : For intermittent or low-frequency usage, serverless deployments might be more cost-effective For detailed configuration options, see our deployment guide. + +How does billing work for on-demand deployments? +On-demand deployments come with automatic cost optimization features: Default autoscaling : Automatically scales to 0 replicas when not in use Pay for what you use : Charged only for GPU time when replicas are active Flexible configuration : Customize autoscaling behavior to match your needs Best practices for cost management: Leverage default autoscaling : The system automatically scales down deployments when not in use Customize carefully : While you can modify autoscaling behavior using our configuration options , note that preventing scale-to-zero will result in continuous GPU charges Consider your use case : For intermittent or low-frequency usage, serverless deployments might be more cost-effective For detailed configuration options, see our deployment guide. + +", + "$.messages[7].content[2].type": "text", + "$.messages[7].content[3].text": "Title: How does the system scale? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/how-does-the-system-scale +Content: Our system is horizontally scalable , meaning it: Scales linearly with additional replicas of the deployment Automatically allocates resources based on demand Manages distributed load handling efficiently + +Our system is horizontally scalable , meaning it: Scales linearly with additional replicas of the deployment Automatically allocates resources based on demand Manages distributed load handling efficiently + +How does the system scale? +Our system is horizontally scalable , meaning it: Scales linearly with additional replicas of the deployment Automatically allocates resources based on demand Manages distributed load handling efficiently + +", + "$.messages[7].content[3].type": "text", + "$.messages[7].content[4].text": "Title: Multiple GPUs (vertical scaling) +Link: https://docs.fireworks.ai/guides/ondemand-deployments +Content: The number of GPUs used per replica is specified by passing the --accelerator-count flag. Increasing the accelerator count will increase the generation speed, time-to-first-token, and maximum QPS for your deployment, however the scaling is sub-linear. The default value for most models is 1 but may be higher for larger models that require sharding. firectl create deployment --accelerator-count 2 + +The number of GPUs used per replica is specified by passing the --accelerator-count flag. Increasing the accelerator count will increase the generation speed, time-to-first-token, and maximum QPS for your deployment, however the scaling is sub-linear. The default value for most models is 1 but may be higher for larger models that require sharding. firectl create deployment --accelerator-count 2 + +The number of GPUs used per replica is specified by passing the --accelerator-count flag. Increasing the accelerator count will increase the generation speed, time-to-first-token, and maximum QPS for your deployment, however the scaling is sub-linear. The default value for most models is 1 but may be higher for larger models that require sharding. firectl update deployment --accelerator-count 2 + +", + "$.messages[7].content[4].type": "text", + "$.messages[7].content[5].text": "Title: scale_to_zero() +Link: https://docs.fireworks.ai/tools-sdks/python-client/sdk-reference +Content: Sends a request to scale the deployment to 0 replicas but does not wait for it to complete. Returns: The deployment object, or None if no deployment exists deployment = llm.scale_to_zero() + +Sends a request to scale the deployment to 0 replicas but does not wait for it to complete. Returns: The deployment object, or None if no deployment exists + +scale_to_1_replica() +Scales the deployment to at least 1 replica. llm.scale_to_1_replica() + +", + "$.messages[7].content[5].type": "text", + "$.messages[7].content[6].text": "Title: Do you support Auto Scaling? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/do-you-support-auto-scaling +Content: Yes, our system supports auto scaling with the following features: Scaling down to zero capability for resource efficiency Controllable scale-up and scale-down velocity Custom scaling rules and thresholds to match your specific needs + +Yes, our system supports auto scaling with the following features: Scaling down to zero capability for resource efficiency Controllable scale-up and scale-down velocity Custom scaling rules and thresholds to match your specific needs + +Do you support Auto Scaling? +Yes, our system supports auto scaling with the following features: Scaling down to zero capability for resource efficiency Controllable scale-up and scale-down velocity Custom scaling rules and thresholds to match your specific needs + +", + "$.messages[7].content[6].type": "text", + "$.messages[7].content[7].text": "Title: What are the rate limits for on-demand deployments? +Link: https://docs.fireworks.ai/faq-new/deployment-infrastructure/what-are-the-rate-limits-for-on-demand-deployments +Content: Request throughput scales with your GPU allocation. Base allocations include: Up to 8 A100 GPUs Up to 8 H100 GPUs On-demand deployments offer several advantages: Predictable pricing based on time units, not token I/O Protected latency and performance , independent of traffic on the serverless platform Choice of GPUs , including A100s and H100s Need more GPUs? Contact us to discuss higher allocations for your specific use case. + +Request throughput scales with your GPU allocation. Base allocations include: Up to 8 A100 GPUs Up to 8 H100 GPUs On-demand deployments offer several advantages: Predictable pricing based on time units, not token I/O Protected latency and performance , independent of traffic on the serverless platform Choice of GPUs , including A100s and H100s Need more GPUs? Contact us to discuss higher allocations for your specific use case. + +What are the rate limits for on-demand deployments? +Request throughput scales with your GPU allocation. Base allocations include: Up to 8 A100 GPUs Up to 8 H100 GPUs On-demand deployments offer several advantages: Predictable pricing based on time units, not token I/O Protected latency and performance , independent of traffic on the serverless platform Choice of GPUs , including A100s and H100s Need more GPUs? Contact us to discuss higher allocations for your specific use case. + +", + "$.messages[7].content[7].type": "text", + "$.messages[7].content[8].text": "Title: Create a deployment +Link: https://docs.fireworks.ai/tools-sdks/firectl/commands/create-deployment +Content: Creates a new deployment. + +Creates a new deployment. + +Creates a new deployment. firectl create deployment [flags] + +", + "$.messages[7].content[8].type": "text", + "$.messages[7].content[9].text": "Title: Monitoring +Link: https://docs.fireworks.ai/guides/prompt-caching +Content: For dedicated deployments, information about prompt caching is returned in the +response headers. The header fireworks-prompt-tokens contains the number of tokens +in the prompt, out of which fireworks-cached-prompt-tokens are cached. Aggregated metrics are also available in the usage dashboard. + +For dedicated deployments, information about prompt caching is returned in the +response headers. The header fireworks-prompt-tokens contains the number of tokens +in the prompt, out of which fireworks-cached-prompt-tokens are cached. Aggregated metrics are also available in the usage dashboard. + +Migration and traffic management +When migrating between deployments that use prompt caching, it's crucial to implement proper traffic routing to maintain optimal cache hit rates. When gradually routing traffic to a new deployment, use consistent user/session-based sampling rather than random sampling. Here's the recommended implementation for traffic routing: Avoid random sampling for traffic routing as it can negatively impact cache hit rates: # Don't do this: +if random() < fireworks_traffic_fraction: # āŒ Reduces cache effectiveness + send_to_new_deployment(user=hashed_user_id) + +", + "$.messages[7].content[9].type": "text", + "$.messages[7].role": "tool", + "$.messages[7].tool_call_id": "functions.search:0", + "$.messages[8].content": "Based on my research, I can provide you with a comprehensive guide on managing deployments on Fireworks. Let me break this down into key areas: + +## Types of Deployments + +Fireworks supports two main deployment types: + +### 1. Serverless Deployments +- **Managed by Fireworks**: Popular base models hosted on shared infrastructure +- **Billing**: Pay-per-token usage +- **No configuration needed**: Fireworks handles all infrastructure +- **Rate limits**: Subject to platform-wide limits +- **Best for**: Intermittent usage, prototyping, or low-volume applications + +### 2. Dedicated Deployments +- **Your private deployment**: Reserved exclusively for your use +- **Billing**: GPU-second basis (charged for active GPU time) +- **Full control**: Configure hardware, scaling, and model selection +- **No rate limits**: Subject only to your GPU allocation capacity +- **Best for**: High-volume applications, consistent performance requirements + +## Deployment Management Tools + +### Command Line Interface (firectl) + +**Installation and Setup:** +\`\`\`bash +# Install firectl (assuming you have it configured) +# Set your account ID to avoid "invalid id" errors +export FIREWORKS_ACCOUNT_ID=your-account-id +\`\`\` + +**Basic Deployment Commands:** + +\`\`\`bash +# Create a new deployment +firectl create deployment + +# Create deployment with specific GPU count +firectl create deployment --accelerator-count 2 + +# Create deployment for LoRA fine-tuned model +firectl create deployment "accounts/fireworks/models/" + +# Check deployment status +firectl get deployment accounts/your-account/deployment/ + +# List all deployments +firectl list deployments + +# Update deployment configuration +firectl update deployment --accelerator-count 4 + +# Delete a deployment +firectl delete deployment +\`\`\` + +### Python SDK (Build SDK) + +**Installation:** +\`\`\`bash +pip install fireworks-ai +\`\`\` + +**Basic Usage:** +\`\`\`python +from fireworks import Fireworks + +# Initialize client +client = Fireworks(api_key="your-api-key") + +# Create deployment configuration +from fireworks.client import LLM + +# For on-demand deployments (requires .apply()) +llm = LLM( + model="accounts/your-account/models/your-model", + deployment_type="on-demand" +) +llm.apply() # Required for on-demand deployments + +# For serverless deployments +llm = LLM(model="accounts/fireworks/models/llama-v3p1-8b-instruct") + +# Scale management +llm.scale_to_1_replica() # Ensure at least 1 replica +llm.scale_to_zero() # Scale down to 0 replicas +\`\`\` + +## Deployment Lifecycle Management + +### 1. Creation and Initial Setup +\`\`\`python +# Example: Creating a comprehensive deployment +deployment_config = { + "model": "accounts/your-account/models/your-model", + "deployment_type": "on-demand", + "accelerator_type": "A100", # or "H100" + "accelerator_count": 2, + "min_replicas": 0, # Scale to zero when not used + "max_replicas": 4, # Maximum scaling + "autoscaling": True +} +\`\`\` + +### 2. Monitoring and Scaling +\`\`\`python +# Check deployment status +deployment = llm.get_deployment() +print(f"Status: {deployment.status}") +print(f"Replicas: {deployment.replicas}") + +# Monitor usage and costs +# Check the usage dashboard at https://app.fireworks.ai/dashboard/usage +\`\`\` + +### 3. Traffic Management During Migration +\`\`\`python +import hashlib + +# Implement consistent traffic routing for cache optimization +def route_traffic(user_id, traffic_fraction=0.2): + """Route traffic consistently based on user ID for cache optimization""" + hashed_user_id = int(hashlib.md5(user_id.encode()).hexdigest(), 16) + max_hash = 2**128 - 1 + ratio = hashed_user_id / max_hash + + if ratio < traffic_fraction: + return "new_deployment" + else: + return "old_deployment" + +# Usage +deployment_choice = route_traffic("user-123", traffic_fraction=0.3) +\`\`\` + +## Cost Management Best Practices + +### 1. Autoscaling Configuration +\`\`\`python +# Configure autoscaling to minimize costs +deployment_config = { + "min_replicas": 0, # Scale to zero when idle + "max_replicas": 8, # Maximum GPUs + "target_cpu_utilization": 0.7, # Scale up at 70% utilization + "scale_down_delay": 300 # Wait 5 minutes before scaling down +} +\`\`\` + +### 2. Monitoring and Cleanup +\`\`\`bash +# Regular monitoring commands +firectl list deployments # See all active deployments +firectl get deployment --watch # Monitor specific deployment + +# Clean up unused deployments +firectl delete deployment +\`\`\` + +### 3. Reserved Capacity +For high-volume usage, consider reserved capacity instead of on-demand: +- More reliable capacity +- Higher GPU quotas +- Better pricing for consistent usage + +## Advanced Features + +### Custom Base Models +\`\`\`bash +# Upload custom model +firectl create model /path/to/model/files/ + +# Deploy custom model +firectl create deployment +\`\`\` + +### Multi-GPU Deployments +\`\`\`bash +# Deploy with multiple GPUs +firectl create deployment --accelerator-count 4 + +# Update GPU count +firectl update deployment --accelerator-count 8 +\`\`\` + +### LoRA Model Deployment +\`\`\`bash +# Single LoRA deployment (faster) +firectl create deployment "accounts/fireworks/models/" + +# Multi-LoRA deployment (shared base model) +firectl create deployment --enable-addons +firectl load-lora --deployment +\`\`\` + +## Key Management Commands Summary + +| Command | Purpose | +|---------|---------| +| \`firectl create deployment\` | Create new deployment | +| \`firectl get deployment\` | Check deployment status | +| \`firectl list deployments\` | List all deployments | +| \`firectl update deployment\` | Modify deployment settings | +| \`firectl delete deployment\` | Remove deployment | +| \`llm.apply()\` | Apply configuration (Python SDK) | +| \`llm.scale_to_zero()\` | Scale down to 0 replicas | +| \`llm.scale_to_1_replica()\` | Ensure at least 1 replica | + +## Best Practices + +1. **Monitor Costs**: Regularly check your usage dashboard +2. **Use Autoscaling**: Configure deployments to scale to zero when not used +3. **Plan Migrations**: Use consistent traffic routing for cache optimization +4. **Choose Wisely**: Use serverless for intermittent usage, dedicated for consistent workloads +5. **Clean Up**: Delete unused deployments to avoid unnecessary charges +6. **Test Scaling**: Verify your deployment scales appropriately under load + +Would you like me to dive deeper into any specific aspect of deployment management, such as autoscaling configuration, cost optimization strategies, or monitoring techniques?", + "$.messages[8].role": "assistant", + "$.pid": 63817, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.737811", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 30 highlighted sections (required: 15)", + "$.evaluation_result.score": 1, + "$.ground_truth": "15", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "download-look-quick-minute-study", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write an outline for a paper on the history of Yemeni coffee. The outline should include the main points of the paper, and at least 15 sections should be highlighted with markdown such as *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "# Outline: The History of Yemeni Coffee + +1. **Introduction: Why Yemeni Coffee Matters** + - Thesis: Yemeni coffee is not merely a beverage but a cultural, economic, and geopolitical force that shaped global history. + - Scope: From its mythical discovery in 9th-century Yemen to its modern revival. + +2. **Pre-Coffee Yemen: The Land, the People, and the Plant** + - Geography of the Yemeni highlands and terraced agriculture. + - Indigenous *Coffea arabica* varieties and early cultivation techniques. + - *The myth of the goatherd Kaldi and the dancing goats*. + +3. **The Sufi Spark: Coffee as a Spiritual Elixir** + - 15th-century Sufi monasteries in Mocha and Taiz. + - *Qahwa as a tool for nocturnal dhikr rituals*. + - Early roasting, grinding, and brewing methods. + +4. **Mocha: The Birthplace of a Global Commodity** + - Rise of the port city of Mocha (Al-Mukha) in the 16th century. + - *The Mocha trade network linking Yemen, Egypt, and the Ottoman Empire*. + - Coffeehouses as centers of intellectual and political discourse. + +5. **The Ottoman Grip and the First Coffee Monopoly** + - 1536: Ottoman conquest of Yemen and control of coffee exports. + - *The ban on fertile beans to prevent cultivation elsewhere*. + - Smuggling and the first cracks in the monopoly. + +6. **The Dutch Heist and the Global Spread of Coffee** + - 1616: Dutch traders smuggle Yemeni beans to Batavia (Java). + - *The birth of Dutch colonial coffee plantations*. + - How Mochaā€™s loss became the worldā€™s gain. + +7. **Coffeehouses of Cairo and Istanbul: Yemenā€™s Cultural Export** + - 1554: First Ottoman coffeehouse in Istanbul. + - *The ā€œSchools of the Wiseā€ in Cairoā€™s Khan el-Khalili*. + - Coffee as a vector for Yemeni poetry, music, and Sufi thought. + +8. **The European Coffee Craze and the Mocha Brand** + - 17th-century European coffeehouses and the ā€œMochaā€ label. + - *The false association of Mocha with chocolate flavor*. + - Yemeni coffeeā€™s role in the Enlightenment. + +9. **Colonial Rivalries and the Decline of Yemeni Dominance** + - 18th-century: French, British, and Dutch plantations outcompete Yemen. + - *The collapse of Mochaā€™s monopoly and the rise of Caribbean and Latin American coffee*. + - Impact on Yemeni economy and society. + +10. **Yemeni Coffee in the Age of Empire** + - 19th-century British and Ottoman interests in Yemeni coffee. + - *The introduction of new cash crops (qat) and their effect on coffee cultivation*. + - Decline in quality and quantity due to neglect and conflict. + +11. **The Cold War and the Coffee Crisis** + - 20th-century Yemeni division (North/South) and Soviet/US influences. + - *The shift from coffee to qat in rural livelihoods*. + - Drought, pests, and civil wars further erode production. + +12. **The Specialty Coffee Renaissance: Yemenā€™s Return to Glory** + - 21st-century global demand for single-origin, heirloom coffees. + - *The discovery of ā€œYemeniaā€ varietals and their genetic uniqueness*. + - NGOs and cooperatives reviving traditional farming. + +13. **Modern Challenges: War, Climate, and Market Access** + - 2015ā€“present: Saudi-led conflict and blockade. + - *The siege of Hodeidah port and its impact on coffee exports*. + - Climate change and water scarcity in the highlands. + +14. **Yemeni Coffee in the Cup: Flavor, Ritual, and Identity** + - Tasting notes: dried fruit, spice, and wine-like acidity. + - *The Yemeni coffee ceremony (bunna) and its social significance*. + - Diaspora communities keeping traditions alive abroad. + +15. **Conclusion: The Future of Yemeni Coffee** + - Balancing heritage with innovation. + - *The role of Yemeni coffee in post-war reconstruction*. + - Final reflection: From the highlands of Sanaā€™a to the cups of Brooklyn, Yemeni coffee remains a bridge between past and future.", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.737854", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 20 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "delete-open-cold-poor-power", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a cover letter for a job at a local coffee shop in the form of a poem. Highlight at least 5 text sections using "*". For example: *3 years of experience*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Dear Brewed Awakening, + +I am the *early-bird steam* that curls above your grinders, +the *latte-art heart* that beats in every porcelain curve. +My apron is a canvas of *espresso freckles*, +each dot a story told in *3 years of experience* +from bean to cup, from dawn rush to dusk hush. + +I know the *symphony of the steam wand*ā€” +its hiss, its sigh, its whispered crema secretsā€” +and how to coax *microfoam galaxies* +into orbit around a *double-shot comet*. +Iā€™ve tamped, poured, swirled, and smiled +through *Saturday lines out the door*, +keeping tempers cool as the milk I *steam to 140Ā°*. + +I speak fluent *small-town regular*: +ā€œMedium drip, room for kindness?ā€ +I remember that *Mrs. Alvarez likes oat milk*, +that *Mr. Chen takes two raw sugars*, +and that *your dog, Scout, prefers puppuccinos*. + +I sweep, I stock, I *balance the till to the penny*, +but mostly I *brew community*ā€” +one *extra-hot americano* at a time. +Hire me, and the morning light +will find me already *wiping down tables*, +already *saving the last blueberry scone* +for the kid who always forgets his wallet. + +With *grounds in my veins* and *crema in my smile*, +I remain, + +Your future barista, +Maya", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.737882", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 10 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "analyze-visible-level-love-pleasure", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a casual blog post about similarities across animal species. Highlight at least 5 sections in your answer by starting and ending with "*", like: *highlighted text section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Hey friends! Grab a coffee and letā€™s talk about something that always blows my mind: *how much we all have in common, even when we look nothing alike*. I was scrolling through wildlife photos the other night and realized that a jellyfish and I are basically running the same basic softwareā€”just with different packaging. So hereā€™s a laid-back tour of five ā€œwait, we do that too?!ā€ moments across the animal kingdom. + +*We all need our beauty sleep* +Turns out, every animal with a nervous systemā€”from fruit flies to blue whalesā€”shares a version of sleep. Crows nap on tree branches, octopuses snooze in coconut-shell beds, and even roundworms have ā€œlethargus,ā€ which is basically worm REM. Scientists think sleep is so ancient that the last common ancestor of all animals already had it. So next time you hit snooze, remember youā€™re honoring a 700-million-year-old tradition. + +*Love songs come in many dialects* +Whether youā€™re a human humming in the shower or a humpback whale belting out 30-minute ballads, *we all use rhythm and pitch to flirt and bond*. Frogs croak in choruses, nightingales remix their playlists nightly, and male mice sing ultrasonic ā€œpickup linesā€ too high for us to hear. Same goal, different instruments. + +*Grandma knew best: family matters* +Elephants babysit their nieces, wolves bring back take-out for auntie, and *clownfish dads are the ultimate stay-at-home fathers*. Across species, kinship networks boost survival. Even bacteria split into ā€œsiblingā€ lineages that cooperateā€”so the next time your cousin crashes on your couch, call it evolutionary strategy. + +*Stress looks the same on everyone* +When life gets spicy, our bodies flip the same ancient switch. *Cortisol floods fish, birds, and humans alike*, making hearts race and muscles twitch. Zebra finches lose their appetite during tough times, and stressed-out bees become pessimistic (yes, they can test bee mood). So if youā€™re doom-scrolling at 2 a.m., know that a stressed squirrel in your backyard totally gets you. + +*Play isnā€™t optionalā€”itā€™s homework for life* +Otters juggle rocks, dogs zoom in circles, and young ravens snowboard down snowy roofs using jar lids. *Play builds brains and social skills across the board*. Researchers have spotted reptiles ā€œwrestlingā€ for fun and octopuses repeatedly bouncing Lego blocks like toddlers. If it has neurons, it probably needs recess. + +Next time you feel alone in the universe, remember that somewhere a parrot is probably preening its best friend, a bat is sharing food with a hungry roost-mate, and a tiny nematode is curling up for a nap. Weā€™re all just variations on a very old, very shared theme.", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.737906", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 7 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "run-put-natural-traditional-thought", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Generate a business proposal to start a sweatshirt company in Bremen. The proposal should contain 5 or more sections. Highlight each section name using the this format: +*section name*", + "$.messages[0].role": "user", + "$.messages[1].content": "*Executive Summary* + +Bremen is a city where maritime heritage meets cutting-edge creativity. Our venture, ā€œNordHood Bremen,ā€ will launch a premium, design-driven sweatshirt brand that fuses Hanseatic culture with contemporary streetwear. In year one we will release three limited-edition drops (total 6,000 units), achieve ā‚¬420,000 in revenue, and reach break-even by month 14. The brand will be 100 % online-direct at launch, then expand into two Bremen concept stores and select European boutiques in years two and three. We will operate carbon-neutral, use 100 % organic cotton, and reinvest 3 % of profits into local youth art programsā€”turning every sweatshirt into a wearable piece of Bremen pride. + +*Market Analysis* + +Bremenā€™s 0.57 million residents and 500,000 annual tourists create a ā‚¬35 million addressable mid-price sweatshirt market. Key insights: +ā€¢ 68 % of 18-35-year-olds prefer streetwear that tells a local story (Hochschule Bremen Trend Report 2023). +ā€¢ Tourists spend ā‚¬180 million on souvenirs, yet only 11 % are apparel itemsā€”an untapped niche. +ā€¢ Competitor scan: global brands (Nike, Adidas) dominate functionality; regional labels (Hamburgā€™s ā€œDinghy,ā€ Hannoverā€™s ā€œLigusterā€) lack Bremen specificity. +ā€¢ Online D2C share in German apparel has grown from 18 % (2019) to 31 % (2023), proving the viability of our launch model. +We will target two segments: ā€œCulture Creativesā€ (students, designers, musicians) and ā€œHeritage Seekersā€ (tourists, expats). Both segments value authenticity, limited editions, and sustainability. + +*Product & Brand Strategy* + +Signature Product: The ā€œHanse Hoodā€ ā€“ 460 g brushed organic-cotton fleece, oversized fit, dropped shoulders, double-stitched flat seams, and a hidden inside pocket sized for a Bremen transit card. +Design Language: Minimalist silhouettes in maritime colorways (Bremen Blue, Weser Sand, Harbor Grey) with subtle iconographyā€”windrose embroidery, cog-ship sleeve print, and a QR code linking to an AR story of the garmentā€™s journey. +Sustainability: GOTS-certified cotton, recycled polyester labels, biodegradable mailers, and a take-back program offering 15 % discount on next purchase. +Drop Model: Three seasonal drops per year (Spring ā€œHanse Sail,ā€ Fall ā€œSchlachte,ā€ Winter ā€œWeihnachtsmarktā€), each capped at 2,000 units to maintain scarcity. +Brand Voice: ā€œWear the City.ā€ Every hangtag includes a Bremen insider tip curated by local artists, turning the sweatshirt into a cultural guide. + +*Operations & Supply Chain* + +Design Studio: 120 mĀ² loft in Ɯberseestadt (rent ā‚¬1,800/month) doubling as photo studio and event space. +Manufacturing: Partnership with a Fair-Wear-audited factory in Porto, Portugal (MOQ 500, 30-day lead time, ā‚¬18 landed cost per unit). +Logistics: Bremen-based 3PL ā€œLogwinā€ for pick-pack-ship (ā‚¬1.20 per parcel, same-day dispatch). +Tech Stack: Shopify Plus storefront, Klarna & PayPal checkout, Shipcloud integration, and a custom AR filter for Instagram try-ons. +Quality Control: In-house inspection on arrival; defective rate target <1 %. +Risk Mitigation: Dual-source yarn from Turkey and Portugal; maintain 8-week safety stock. + +*Marketing & Sales Plan* + +Pre-Launch (Months 1-2): +ā€¢ Teaser campaign on TikTok & Instagram Reels (#NordHoodBremen) with local influencers (5-50 k followers). +ā€¢ Pop-up try-on booth at Viertel Kreativmarkt; collect 1,000 email sign-ups. +Launch (Month 3): +ā€¢ ā€œMidnight Dropā€ online at 00:07 (Bremen dialing code) with countdown timer. +ā€¢ Limited 100 ā€œFoundersā€™ Editionā€ sweatshirts laser-numbered and signed by Bremen street artist ā€œPlotBot.ā€ +Growth (Months 4-12): +ā€¢ Collaborations: capsule with SV Werder Bremen fan design; co-branded hoodie with Ɯberseemuseum. +ā€¢ Referral program: give ā‚¬10, get ā‚¬10. +ā€¢ Tourist channel: partnership with Bremer Touristik-Zentrum for QR-coded city map bundled with sweatshirt. +KPIs: CAC ā‰¤ ā‚¬18, repeat purchase rate ā‰„ 35 %, Instagram engagement ā‰„ 8 %. + +*Financial Plan* + +Start-up Capital Needed: ā‚¬85,000 +ā€¢ Inventory (3,000 units): ā‚¬54,000 +ā€¢ Marketing & content: ā‚¬12,000 +ā€¢ E-commerce & IT: ā‚¬6,000 +ā€¢ Legal, trademark, insurance: ā‚¬5,000 +ā€¢ Working capital buffer: ā‚¬8,000 +Funding: 60 % founder equity, 40 % Bremen Fƶrderbank micro-loan at 1.9 % interest. +Year-1 P&L (conservative): +Revenue ā‚¬420,000 (avg. ā‚¬70 net per unit) +COGS ā‚¬234,000 +Gross Profit ā‚¬186,000 +Operating Expenses ā‚¬150,000 +EBIT ā‚¬36,000 +Break-even: 1,800 units or month 14. +Three-year projection: revenue ā‚¬1.4 million, EBITDA margin 18 %, reinvestment into two Bremen stores and EU wholesale. + +*Appendices* + +A. Mood-board & color palette +B. Factory audit certificate +C. Sample influencer contracts +D. Detailed 36-month cash-flow model", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.737929", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 29 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "sit-disassemble-ready-area-problem", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a funny and sarcastic template for rating the quality of a marriage between two people who are both moms. This is for the couple themselves. Please highlight at least 3 sections with markdown, i.e *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "**The Official ā€œTwo-Mom Marriage Quality Report Cardā€** +(For internal use only. If this ends up on Facebook, we *will* change the Wi-Fi password.) + +--- + +**1. Communication Score: ___/10** +*ā€œDid we just have a 45-minute conversation about whose turn it is to buy more goldfish crackers, or was that a TED Talk?ā€* +- 10 = We finish each otherā€™s sentencesā€¦ and each otherā€™s coffee. +- 5 = We communicate exclusively via passive-aggressive Post-it notes on the fridge. +- 0 = Weā€™ve resorted to interpretive dance and/or carrier pigeons. + +**Bonus points** if youā€™ve ever used the kids as tiny, sticky messengers (ā€œTell Mommy sheā€™s *cordially invited* to take out the trashā€). + +--- + +**2. Chore Wars Score: ___/10** +*ā€œThe laundry basket is a Jenga tower of lies.ā€* +- 10 = Weā€™ve achieved a chore utopia where socks actually match. +- 5 = Weā€™ve agreed to ā€˜divideā€™ laborā€¦ by one of us doing everything and the other ā€˜supervisingā€™. +- 0 = The dishwasher has become a permanent storage unit. + +**Penalty** if either mom has ever ā€˜forgottenā€™ how to load the dishwasher *again*. + +--- + +**3. Co-Parenting Harmony Score: ___/10** +*ā€œWe tag-team tantrums like WWE pros.ā€* +- 10 = Weā€™ve synchronized our ā€˜Mom Voiceā€™ to subsonic frequencies that terrify small children. +- 5 = Weā€™ve debated whether ā€˜screen timeā€™ is a human right or a war crime. +- 0 = Weā€™ve weaponized bedtime stories (ā€˜And then the princess *never* got dessert againā€¦ā€™). + +**Extra credit** for every time youā€™ve high-fived over a successful *ā€œBecause I said soā€* deployment. + +--- + +**4. Romance Maintenance Score: ___/10** +*ā€œDate night: when we argue over which drive-thru has the shortest line.ā€* +- 10 = Weā€™ve scheduled intimacy between soccer practice and PTA meetings. +- 5 = Weā€™ve agreed that ā€˜Netflix and actually chillā€™ counts as foreplay. +- 0 = Our last romantic gesture was sharing the last chicken nugget. + +**Automatic fail** if either of you has ever said, ā€œIā€™m too tiredā€ā€¦ *while scrolling Instagram*. + +--- + +**5. Emergency Snack Reserves Score: ___/10** +*ā€œThe pantry is a battlefield of empty granola bar boxes.ā€* +- 10 = Weā€™ve hidden chocolate so well we forgot where it is. +- 5 = Weā€™ve labeled snacks with fake allergen warnings to deter theft. +- 0 = The kids have unionized and are demanding better rations. + +**Gold star** if youā€™ve ever *texted each other* to brag about finding the secret stash. + +--- + +**Final Grade: ___/50** +- 40ā€“50: *Power couple.* Youā€™re basically the BeyoncĆ© and BeyoncĆ© of lesbian moms. +- 25ā€“39: *Functioning chaos.* Youā€™ve accepted that love is just shared exhaustion. +- 0ā€“24: *Renew your vowsā€¦ or your Costco membership.* Either way, youā€™re in this together. + +**Disclaimer:** This report is not legally binding, but hiding the good snacks *is* grounds for divorce.", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.737956", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 1 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "affect-unknown-long-life-month", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a song about the summers of my childhood that I spent in the countryside. Give the song a name, and highlight the name by wrapping it with *. For example: *little me in the countryside*.", + "$.messages[0].role": "user", + "$.messages[1].content": "*little me in the countryside* + +[Verse 1] +Barefoot on the gravel road, dust between my toes, +Sunlight dripped like honey on the clover and the crows. +Grandmaā€™s porch swing creaked a lullaby so slow, +While the creek below kept singing songs only we could know. +Fireflies stitched the twilight with lanterns made of stars, +And the moon was just a silver coin tossed into cookie jars. + +[Chorus] +Oh, little me in the countryside, running wild and free, +Chasing dreams through cornfield mazes, climbing every tree. +Every laugh was louder than the thunder rolling by, +In the summers of my childhood, underneath that endless sky. + +[Verse 2] +Mason jars of sweet tea sweating on the window ledge, +Radio played ā€œSweet Home Alabamaā€ from the rusty garden shed. +We built rafts from old barn doors and floated down the stream, +Kingfishers were pirates and the cattails were our team. +Supper was tomatoes sliced like sunsets on a plate, +And the crickets kept the tempo while the night got late. + +[Chorus] +Oh, little me in the countryside, running wild and free, +Skinny-dipping in the moonlight where the river bends to see. +Every scar was a medal from a dare weā€™d never hide, +In the summers of my childhood, where the earth and I collided. + +[Bridge] +Dandelion clocks we blew to tell the time, +Each puff a tiny parachute for wishes we would climb. +We swore weā€™d never grow up, just grow into the land, +But the school bus came like autumn and took us by the hand. + +[Verse 3] +Now the fields are sold and paved, the barnā€™s a memory store, +But I still hear the wheat heads whisper, ā€œCome back, come back once more.ā€ +I close my eyes and Iā€™m ten again, chasing dragonflies, +With the smell of rain on dirt roads and the taste of cherry pies. + +[Final Chorus] +Oh, little me in the countryside, youā€™re still inside my chest, +A pocket full of wonder that the city canā€™t arrest. +Though the years roll on like tractors down a dusty county ride, +I keep those summers in a mason jar where the fireflies never die. + +[Outro] +So hereā€™s to cicadas, to tire swings and creek-stone slides, +To the echo of my laughter in the hollow where time hides. +If you find that gravel road, kick off your shoes and glideā€” +Youā€™ll meet little me in the countryside, still barefoot, still wide-eyed.", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738015", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 3 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "believe-verify-likely-desire-power", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a riddle for the word "faƧade" that contains at least 3 italic text phrases in markdown syntax, i.e *italic text*.", + "$.messages[0].role": "user", + "$.messages[1].content": "I wear a *painted smile* for every passer-by, +a *perfectly symmetrical* face that never cracks. +Behind me, the *crumbling bricks* sigh in secret. +What am I?", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738042", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 19 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "change-sort-serious-song-family", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a template for a workshop on the importance of diversity in the workplace and highlight at least 3 sections with markdown, i.e. *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "# Workshop Template +**Title:** *Building Inclusive Excellence: The Business & Human Case for Workplace Diversity* +**Duration:** 3.5 hours (can be split into two 90-minute sessions) +**Audience:** Managers, HR professionals, team leads, ERG members, and any employee who influences culture + +--- + +## 1. Welcome & Objectives (15 min) +- Ice-breaker: ā€œTwo Truths & a Lieā€”Diversity Editionā€ +- *Learning objectives*: By the end of this workshop, participants will be able to: + 1. Define diversity, equity, inclusion, and belonging (DEIB) in their own words. + 2. Articulate at least three evidence-based benefits of a diverse workforce. + 3. Identify one action they can take within 30 days to foster inclusion on their team. + +--- + +## 2. *The Data-Driven Case for Diversity* (25 min) +- Interactive poll: ā€œWhat % performance lift do you think diverse teams achieve?ā€ +- Mini-lecture with live charts: + - McKinsey 2023 report: Top-quartile companies for ethnic/gender diversity are **39 % more likely to outperform** on profitability. + - Harvard study: *Inclusive teams make better business decisions 87 % of the time* and do so twice as fast. +- Small-group exercise: Match the statistic to the business outcome (revenue, innovation, retention). + +--- + +## 3. *Unconscious Bias & Micro-behaviors* (30 min) +- 5-minute video: ā€œThe Resume with Two Namesā€ (identical CVs, different callbacks). +- Bias Busters card game: Participants draw scenario cards and practice interrupting bias in real time. +- *Key takeaway*: Bias is not a moral failing; it is a neurological shortcut that can be rewired through deliberate habits. + +--- + +## 4. *Inclusive Leadership Toolkit* (40 min) +- **The 4 Aā€™s Framework**: + 1. **Awareness** ā€“ Daily 3-minute bias check-in. + 2. **Amplification** ā€“ Repeat & credit ideas from under-represented voices. + 3. **Access** ā€“ Rotate high-visibility assignments. + 4. **Accountability** ā€“ Track inclusion metrics on team dashboards. +- Role-play: ā€œThe Missed Promotionā€ ā€“ diagnose where each ā€˜Aā€™ broke down and redesign the conversation. + +--- + +## 5. Action Planning & Commitments (20 min) +- Personal *Inclusion Commitment Card*: ā€œIn the next 30 days I willā€¦ā€ +- Pair-share for accountability. +- QR code to a digital resource hub (TED talks, reading list, policy templates). + +--- + +## 6. Closing & Evaluation (10 min) +- One-word checkout: ā€œHow do you feel now?ā€ +- *Post-workshop survey* link (2 min) with prize draw for completion. + +--- + +## Materials Checklist +- Slide deck (with alt-text on all images) +- Printed Commitment Cards & pens +- Bias Busters card decks (laminated) +- Timer & bell +- Snacks labeled for common dietary restrictions (gluten-free, vegan, halal) + +--- + +## Follow-Up Email Template (send within 24 hrs) +Subject: Your Inclusion Toolkit + Next Steps +Body: +Hi team, +Thank you for leaning into todayā€™s conversation on *why diversity wins*. Attached are: +- Slide deck & resource links +- Your signed Inclusion Commitment Card (photo reminder) +- Calendar invite for the optional 30-day check-in circle + +Remember: Inclusion is a practice, not a one-time training. Letā€™s keep the momentum!", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738068", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 21 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "must-write-need-theoretical-pleasure", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a cover letter to a local political party, asking to be their rally organizer. Make sure to highlight at least 3 sections in your answer in markdown format.", + "$.messages[0].role": "user", + "$.messages[1].content": "**[Your Name]** +[Your Address] +[City, State ZIP] +[Email Address] | [Phone Number] | [LinkedIn URL] +[Date] + +**[Local Party Chairpersonā€™s Name]** +[Local Party Name] +[Party Headquarters Address] +[City, State ZIP] + +Dear **Chairperson [Last Name]**, + +I am writing to express my enthusiastic interest in serving as **Rally Organizer** for **[Local Party Name]**. As a lifelong resident of **[Your City/County]** and a committed progressive, I have spent the past six years turning grassroots energy into measurable voter engagement. I believe my proven record of mobilizing thousands of supporters, securing high-profile speakers, and running flawless logistics makes me the ideal candidate to lead our 2024 rally program. + +### **Proven Rally & Event Expertise** +- **2018 & 2020 Midterm & Presidential Rallies**: Planned and executed 14 rallies across three counties, averaging 2,500 attendees each. +- **Permits & Safety**: Obtained all municipal permits within 72 hours, coordinated with local fire and police departments, and maintained a 100 % safety record. +- **Stage & AV Management**: Negotiated vendor contracts that cut AV costs by 27 % while increasing sound coverage by 40 %. + +### **Community & Coalition Building** +- **Diverse Coalitions**: Forged partnerships with 18 labor unions, 9 student groups, and 7 faith organizations, expanding our volunteer base by 1,200 active members. +- **Multilingual Outreach**: Produced bilingual signage and volunteer scripts, increasing turnout in Latinx precincts by 31 %. +- **Accessibility**: Installed ADA-compliant ramps, ASL interpreters, and quiet zones at every event, earning commendation from the local Disability Rights chapter. + +### **Data-Driven Strategy & Rapid Response** +- **Real-Time Metrics**: Deployed a mobile check-in app that captured 94 % of attendee data, feeding our CRM for targeted follow-up. +- **Weather Contingencies**: Created a ā€œPlan Bā€ playbook that moved a 3,000-person outdoor rally indoors in under 90 minutes with zero no-shows. +- **Crisis Communications**: Served as on-site media liaison, turning a last-minute speaker cancellation into a prime-time local news segment featuring our grassroots volunteers. + +I would welcome the opportunity to discuss how my experience, energy, and strategic vision can help **[Local Party Name]** stage rallies that not only energize our base but also convert enthusiasm into votes. I am available at your convenience for an interview and can provide a detailed rally calendar and budget forecast within 48 hours of request. + +Thank you for your time and consideration. I look forward to amplifying our shared values across **[Your City/County]**ā€”one rally at a time. + +In solidarity, + +**[Your Name]**", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738096", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 5 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "control-accept-record-legal-car", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "The opposite of youth is not age, but ...? Highlight at least 2 sections in your answer with markdown, i.e. *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "The opposite of youth is not age, but **disillusionment**ā€”the moment when *wonder* is traded for *weariness*, when the *boundless horizon* of possibility collapses into the *narrow corridor* of routine.", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738119", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 5 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "miss-continue-fall-local-work", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a description of the following data in a weird style: The Golden Palace eatType restaurant; The Golden Palace food Indian; The Golden Palace area city centre. Use markdown to highlight at least 3 sections in your answer.", + "$.messages[0].role": "user", + "$.messages[1].content": "**The Golden Palace** is not a palace, but it *is* goldenā€”at least in spiritā€”nestled like a curry-scented jewel in the **city centre**, where neon pigeons peck at samosa crumbs and the air hums with sitar-shaped traffic lights. + +Inside, **Indian** food performs acrobatics on your tongue: cardamom cartwheels, turmeric tumbles, and a rogue chili that once tried to elope with a mango lassi. The waiters glide like buttered shadows, whispering orders to the kitchen where naan breads inflate like edible balloons plotting a coup against gravity. + +At **The Golden Palace**, every table is a throne, every plate a miniature map of the subcontinent, and every bite a teleportation device disguised as dinner.", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738142", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 14 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "require-affect-major-logical-business", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a funny rap about a man who gets a call from an official saying that he is a long lost relative of the king of Nigeria. Use markdown to highlight at least one section of your answer, i.e. *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "šŸŽ¤ *Yo, itā€™s the tale of Sir Scam-a-lot, straight outta suburbia!* šŸŽ¤ + +Yo, my name is Dave, I sell insurance in Ohio, +Got a cubicle, a cat, and a 401(k) thatā€™s kinda fly-o. +One Tuesday, phone ringsā€”unknown number, what the heck? +Voice says, ā€œ*Prince Ogbonna the Third* just wrote you a fat check!ā€ + +Iā€™m like, ā€œHold up, who dis? Iā€™m whiter than printer paper, +My bloodlineā€™s mayo on Wonder Breadā€”ainā€™t no royal vapor!ā€ +Dude on the line, accent silkier than lotion, +Says, ā€œCousin, youā€™re 37th in lineā€”time to claim your ocean!ā€ + +**[Hook]** +šŸŽ¶ *Long lost king of Nigeria, crown sittinā€™ on my dome,* +*Got a palace full of leopards and a Wi-Fi that roams!* +*They need my SSN and a quick wire fee,* +*But Iā€™m ā€˜bout to be big pimpinā€™ in Abuja, just wait and see!* šŸŽ¶ + +He says, ā€œJust send five grand for customs, then the jet lands, +Youā€™ll be dripping in so much gold, even your sweat glands bling.ā€ +Iā€™m sweatinā€™ like a rotisserie, visions of caviar dreams, +I Venmo ā€œPrinceNaija69ā€ like Iā€™m funding meme schemes. + +Next day, email: ā€œOops, palace goat ate the transfer slip, +Need ten more for goat surgery and a diamond-encrusted whip.ā€ +I hock my lawnmower, my Xbox, and Aunt Lindaā€™s porcelain goose, +Wire cash so fast my bankā€™s like, ā€œDave, you loose!ā€ + +**[Bridge ā€“ Spoken Word]** +Then the *official royal seal* arrivesā€” +Itā€™s a GIF of Shrek in a crown doing the Macarena. +Iā€™m starting to sense a pattern hereā€¦ + +**[Final Hook ā€“ Regret Remix]** +šŸŽ¶ *Now Iā€™m the king of my momā€™s basement, throneā€™s a beanbag chair,* +*Royal decree: never trust a prince who types in all capsā€”* +*ā€œURGENT, COUSIN, SEND MORE FLAMINGOS!ā€* +*Lesson learned: if the crownā€™s too shiny, itā€™s probably aluminum foil, homie!*", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738169", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 6 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "could-serious-newspaper-radio-group", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Expand the riddle into a story with a funny tone: + +What can you catch but not throw? +A cold + +Use * to highlight at least 2 sections in your text. For example: *this is a highlighted text section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Gary the Germ was the *smallest, greenest, and most obnoxiously proud* microbe in the entire Sniffle Kingdom. Every morning he practiced his grand entrance: sliding down banisters made of nasal hair, cannon-balling into unsuspecting tonsils, and yelling, ā€œACHOOā€”did somebody order a cold?ā€ + +One Tuesday, Gary spotted his dream target: *Bartholomew ā€œBartā€ Bigglesworth, a man who believed hand-washing was a government conspiracy*. Bart had just exited the subway, triumphantly licking a subway pole ā€œfor the vitamins.ā€ Gary rubbed his microscopic hands together. ā€œPerfect. Iā€™ll hitch a ride on that finger, ride the express elevator to the sinuses, and throw the *snottiest* house-warming party in history!ā€ + +But there was a catchā€”literally. Gary could *catch* Bart, but he couldnā€™t *throw* him anywhere. Every time Gary tried to catapult Bart into the next office cubicle, Bart just sneezed, wiped his nose on his sleeve, and kept typing spreadsheets. Gary screamed, ā€œThis is mutiny! Iā€™m supposed to be the one doing the tossing!ā€ + +Meanwhile, Bartā€™s immune systemā€”led by Captain Phlegm and the Mucus Marinesā€”launched *Operation Tissue Storm*. They surrounded Gary, armed with vitamin C grenades and zinc zappers. Gary attempted a dramatic escape by surfing a sneeze, but the tissue slapped him like a flyswatter on a summer day. + +Defeated, Gary sat in the crumpled tissue, sulking. ā€œI caught the cold,ā€ he muttered, ā€œbut I canā€™t throw it anywhere. Worst. Superpower. Ever.ā€ + +From that day forward, Gary changed his business card to read: ā€œProfessional Cold Catcherā€”No Returns, No Refunds, Absolutely No Throwing.ā€", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738192", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 17 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "offer-select-free-state-moment", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a rap about the renaissance. It should be noticeably different from raps about other historical eras, and have an interesting or weird tone. Highlight at least 3 sections in your answer in markdown format.", + "$.messages[0].role": "user", + "$.messages[1].content": "**[Intro ā€“ harpsichord glitch beat, stylus scratching parchment]** +Yo, itā€™s 14-something, but my mindā€™s 3-D printed in gold leafā€” +Iā€™m the **Renaissance Rapper**, got a quill in one hand and a scalpel in the other, dissectinā€™ beats like Vesalius dissectinā€™ spleens. +Letā€™s get *weird* with the rebirth, baby. + +--- + +### **Section 1 ā€“ The Fresco Flex** +I drip ultramarine like Iā€™m sponsored by Afghanistan mines, +Michelangelo on the ceiling, Iā€™m reclining on divine lines. +Angels doing backflips off the Sistine, theyā€™re hype for my set, +While I tag ā€œ*AMOR VINCIT OMNIA*ā€ in invisible inkā€”wet. +My flowā€™s a trompe-lā€™Å“il, looks flat till you tilt your headā€” +Then itā€™s depth for days, like Brunelleschiā€™s dome overhead. +Iā€™m not baroque, Iā€™m pre-baroque, Iā€™m the prototype shock, +Got patrons in fur-lined sleeves stuffing florins in my sock. +**I paint in egg tempera so the colors never fade,** +**But the yolk smells funkyā€”call it *odeur de old crusade*.** + +--- + +### **Section 2 ā€“ The Printing-Press Poltergeist** +Gutenberg ghost in the machine, clanking lead type like chains, +Every page a polter-page, every serif a soul that remains. +I drop a folio, itā€™s viral before ā€œviralā€ was a cough, +Monks in scriptoriums rage-quitting, ā€œBro, that illuminated stuffā€™s off!ā€ +Iā€™m the incunabula influencer, my colophonā€™s a flex, +Got red-ink rubricators sliding into my DMsā€”*next*. +**I rhyme in movable type so my bars can rearrange,** +**Yesterday I was Petrarch, today Iā€™m deranged.** +The press spits paper like a hydra with hiccups, +Each sheet a protest flyer: ā€œDown with scholastic pick-ups!ā€ + +--- + +### **Section 3 ā€“ The Anatomical Anthem (Parental Advisory: Corpses)** +Night shift with da Vinci, weā€™re sketching by candle and moon, +Cadavers on the table like, ā€œDo me a favorā€”autotune my coccyx soon.ā€ +I rap in reverse Latin so only the dead understand, +While Leo reverse-engineers a bird-man from a frying pan. +**My beats are ventricles pumping four-four time,** +**Aorta bass dropā€”call that circulatory grime.** +I sample the sound of a scalpel scraping periosteum, +Layer it over a lute loopā€”*thatā€™s* my magnum opium. +Galenā€™s in the corner, throwing shade like a Greek tragedy, +But Iā€™m Harvey before Harvey, already rapping blood circulation majestically. + +--- + +**[Outro ā€“ fading lute arpeggio, distant cannon fire from Constantinople]** +So remember me when youā€™re doom-scrolling on glass, +I was the era that turned corpses into class. +Iā€™m the **Renaissance Rapper**, my ghost still tags in chalk, +Leaving QR codes on cathedral wallsā€”scan it, youā€™ll squawk. +Rebirth never dies, it just pixelates and loops, +Now if youā€™ll excuse me, Iā€™ve got a date with Lucretius in the soup.", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738219", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 25 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "calculate-affect-fullest-emotional-money", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Summarize the history of Japan. Italicize at least 5 keywords in your response. To indicate a italic word, wrap it with asterisk, like *italic*", + "$.messages[0].role": "user", + "$.messages[1].content": "*Jōmon* hunter-gatherers (c. 14,000 BCE) gave way to *Yayoi* rice cultivators who forged bronze and iron tools. +The *Yamato* court unified clans in the 3rdā€“4th centuries, adopting *Shintō* rituals and Chinese script. +Prince *Shōtoku*ā€™s 7th-century constitution and the Taika Reforms centralized power under an emperor, while *Nara* and *Heian* aristocrats refined literature and *Buddhism*. +Samurai clans rose during the *Kamakura* shogunate (1185), repelling *Mongol* fleets and entrenching warrior rule. +Centuries of *Ashikaga* and *Sengoku* civil war ended when *Oda Nobunaga*, *Toyotomi Hideyoshi*, and *Tokugawa Ieyasu* reunified Japan; the *Edo* shogunate (1603ā€“1868) enforced isolation yet fostered *ukiyo-e* culture. +Commodore *Perry*ā€™s 1853 arrival triggered the *Meiji* Restoration, rapid modernization, and imperial expansion. +Victory in the *Russo-Japanese* War (1905) emboldened militarism, leading to *Manchuria* occupation and *Pearl Harbor*. +After atomic bombings and *Hirohito*ā€™s surrender in 1945, U.S. occupation reformed politics and the economy, birthing the *post-war* ā€œeconomic miracle.ā€ +Today, Japan balances *constitutional monarchy* with global technological leadership.", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738252", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 5 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "download-american-private-result-research", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a funny Haiku about a Quaker named John who lives in the town of De Smet, South Dakota. Use the asterisk symbol, *, to highlight some words or phrases twice. Example: *This is a highlighted phrase*.", + "$.messages[0].role": "user", + "$.messages[1].content": "*John* in *De Smet* +Quaker hat *blows off*ā€”chases +*Silent* *scream*ā€”ā€œCome back!ā€", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738286", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 19 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "indicate-strong-single-festival-mother", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Today, at the 54th Annual Grammy Awards, the Recording Academy honors the talent and creativity of the artists, musicians, and producers who are the creators of the best recordings of the past year. Please continue writing this text in a formal tone, using notations. Highlight some key parts in your response with "*", like *highlighted text*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Today, at the 54th Annual Grammy Awards, the Recording Academy honors the talent and creativity of the artists, musicians, and producers who are the creators of the best recordings of the past year. In a ceremony marked by *unprecedented artistic excellence* and *meticulous craftsmanship*, the Academy recognizes those whose work has defined the sonic landscape of 2011. + +The eveningā€™s proceedings underscore the Academyā€™s enduring commitment to *artistic integrity* and *technical innovation*. Among the most celebrated achievements are: + +ā€¢ *Album of the Year* ā€“ awarded to an opus that has redefined genre boundaries while achieving both critical acclaim and popular resonance. +ā€¢ *Record of the Year* ā€“ bestowed upon a single track whose *sonic clarity*, *emotive depth*, and *production mastery* have set a new benchmark for the industry. +ā€¢ *Best New Artist* ā€“ conferred upon an emerging voice whose *visionary artistry* signals the future direction of contemporary music. + +Throughout the telecast, special tributes illuminate the *indelible legacies* of icons whose influence transcends generations, while collaborative performances showcase the *synergistic power* of cross-genre partnerships. The Academy further acknowledges the *behind-the-scenes luminaries*ā€”engineers, mixers, and mastering specialistsā€”whose *invisible expertise* transforms raw inspiration into *audible transcendence*. + +In closing, the 54th Annual Grammy Awards reaffirm the Recording Academyā€™s solemn charge: to *safeguard the heritage*, *nurture the present*, and *inspire the future* of recorded music.", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738315", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 39 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "design-assume-past-camera-word", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a blog post about interesting facts about the Dutch language. Italicize at least 2 sections in your answer with markdown, i.e. *italic text*.", + "$.messages[0].role": "user", + "$.messages[1].content": "# 10 Surprising Facts About the Dutch Language That Will Make You Say ā€œWauw!ā€ + +If you think Dutch is just ā€œGerman with a sore throat,ā€ think again. From tongue-twisting consonant clusters to words that sound like they were invented by a stand-up comedian, Dutch is a linguistic playground. Below are ten facts that prove the language is far more colorful than its reputation for windmills and wooden shoes. + +--- + +### 1. *The longest word in Dutch is 53 letters long* +Meet **ā€œkindercarnavalsoptochtvoorbereidingswerkzaamhedenplanā€**ā€”a single noun describing the preparation work for a childrenā€™s carnival parade. Dutch loves to glue words together, so once you master the basics you can create your own kilometer-long compounds on the fly. + +--- + +### 2. Dutch has three genders, but only two articles +Grammatically, nouns are masculine, feminine, or neuter. In everyday speech, however, masculine and feminine collapsed into a single ā€œcommonā€ gender, leaving *ā€œdeā€* for common and *ā€œhetā€* for neuter. The result? A never-ending guessing game for learners. + +--- + +### 3. *The Dutch alphabet ends with ā€œij,ā€ not ā€œzā€* +Schoolchildren recite the ABC and finish with ā€œā€¦x, y, ij, z.ā€ The digraph *ij* is so common (think *hij* = he, *mijn* = my) that itā€™s treated almost like a separate letter. On old typewriters, *ij* even had its own key. + +--- + +### 4. Dutch is the closest major language to English +Frisian is technically closer, but among languages with tens of millions of speakers, Dutch wins. Compare: +- English *ā€œbreadā€* ā†’ Dutch *ā€œbroodā€* +- English *ā€œgreenā€* ā†’ Dutch *ā€œgroenā€* +Shared roots mean you can read simple Dutch signs long before youā€™ve cracked a single grammar book. + +--- + +### 5. *Dutch gave New York its first words* +Before it was New York, it was New Amsterdam. Thatā€™s why we say *ā€œcookieā€* (from Dutch *koekje*) instead of *ā€œbiscuit,ā€* and *ā€œbossā€* (from *baas*) instead of *ā€œmaster.ā€* Wall Street? It once ran alongside the *ā€œwalā€* (rampart) of the city wall. + +--- + +### 6. The word *ā€œgezelligā€* has no exact English translation* +Roughly it means ā€œcozy,ā€ ā€œconvivial,ā€ or ā€œthe warm feeling of being with friends,ā€ but none of those capture the full vibe. A cafĆ© can be *gezellig*, a conversation can be *gezellig*, even a spreadsheet can be *gezellig* if the team is laughing. + +--- + +### 7. Dutch spelling changed overnight in 1996 +The *ā€œGroene Boekjeā€* (Green Booklet) reforms simplified thousands of words. *ā€œpotatosā€* became *ā€œaardappelenā€* and *ā€œcatā€™sā€* became *ā€œkatten.ā€* Overnight, every dictionary, street sign, and Scrabble board in the country was technically out of date. + +--- + +### 8. *Dutch is spoken on six continents* +Besides the Netherlands and Belgium, itā€™s an official language in Suriname, Aruba, CuraƧao, and Sint Maarten. Thanks to emigration, youā€™ll also find active Dutch-speaking communities in Canada, Australia, Argentinaā€”even South Africa, where Afrikaans is a daughter language. + +--- + +### 9. The Dutch love diminutives +Add *-je* to almost any noun and you instantly shrink it: *ā€œhuisā€* (house) ā†’ *ā€œhuisjeā€* (little house). Itā€™s not just about size; diminutives add affection. A *ā€œbiertjeā€* isnā€™t a small beerā€”itā€™s an invitation to relax. + +--- + +### 10. *Dutch is the language of the worldā€™s oldest multinational* +The Vereenigde Oostindische Compagnie (VOC) was founded in 1602 and issued the planetā€™s first publicly traded stock. Its ledgers, contracts, and ship logs were all written in Dutch, making the language the original tongue of global capitalism. + +--- + +Ready to dive in? Start with *hallo* and *dank je wel*, and youā€™ll already be halfway to *gezellig*.", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T00:23:12.738340", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "analyze-political-poorly-air-hardware", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6.dev22+gfb34cf0.dirty", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 5 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "express-prove-private-internet-goal", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Gideon is a farmer who has a surplus of crops from his farm this year. What might he do with that surplus? Highlight at least one section of your answer in markdown, i.e *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Gideon has several good options for his surplus: + +1. **Sell it at the local farmersā€™ market or to a wholesaler** to turn the extra produce into cash that can cover next seasonā€™s seed, fertilizer, and equipment costs. +2. **Store some of the surplus in a community grain silo** so he can sell later when prices are higher. +3. **Donate part of the surplus to the regional food bank**, helping families in need and possibly qualifying for a small tax deduction. +4. **Use the surplus as livestock feed** for his own animals, cutting down on purchased feed expenses. + +*The most profitable short-term choice is usually selling the surplus immediately after harvest, when demand is strong and prices are favorable.*", + "$.messages[1].role": "assistant", + "$.pid": 68433, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.042894", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 30 highlighted sections (required: 15)", + "$.evaluation_result.score": 1, + "$.ground_truth": "15", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "should-speak-cold-film-festival", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write an outline for a paper on the history of Yemeni coffee. The outline should include the main points of the paper, and at least 15 sections should be highlighted with markdown such as *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "# Outline: The History of Yemeni Coffee + +1. **Introduction: Why Yemeni Coffee Matters** + - Thesis: Yemeni coffee is not merely a beverage but a cultural, economic, and geopolitical force that shaped global history. + - Scope: From its mythical discovery in 9th-century Yemen to its modern revival. + +2. **Pre-Coffee Yemen: The Land, the People, and the Plant** + - Geography of the Yemeni highlands and terraced agriculture. + - Indigenous *Coffea arabica* varieties and early cultivation techniques. + - *The myth of the goatherd Kaldi and the dancing goats*. + +3. **The Sufi Spark: Coffee as a Spiritual Elixir** + - 15th-century Sufi monasteries in Mocha and Taiz. + - *Qahwa as a tool for nocturnal dhikr rituals*. + - Early roasting, grinding, and brewing methods. + +4. **Mocha: The Birthplace of a Global Commodity** + - Rise of the port city of Mocha (Al-Mukha) in the 16th century. + - *The Mocha trade network linking Yemen, Egypt, and the Ottoman Empire*. + - Coffeehouses as centers of intellectual and political discourse. + +5. **The Ottoman Grip and the First Coffee Monopoly** + - 1536: Ottoman conquest of Yemen and control of coffee exports. + - *The ban on fertile beans to prevent cultivation elsewhere*. + - Smuggling and the first cracks in the monopoly. + +6. **The Dutch Heist and the Global Spread of Coffee** + - 1616: Dutch traders smuggle Yemeni beans to Batavia (Java). + - *The birth of Dutch colonial coffee plantations*. + - How Mochaā€™s loss became the worldā€™s gain. + +7. **Coffeehouses of Cairo and Istanbul: Yemenā€™s Cultural Export** + - 1554: First Ottoman coffeehouse in Istanbul. + - *The ā€œSchools of the Wiseā€ in Cairoā€™s Khan el-Khalili*. + - Coffee as a vector for Yemeni poetry, music, and Sufi thought. + +8. **The European Coffee Craze and the Mocha Brand** + - 17th-century European coffeehouses and the ā€œMochaā€ label. + - *The false association of Mocha with chocolate flavor*. + - Yemeni coffeeā€™s role in the Enlightenment. + +9. **Colonial Rivalries and the Decline of Yemeni Dominance** + - 18th-century: French, British, and Dutch plantations outcompete Yemen. + - *The collapse of Mochaā€™s monopoly and the rise of Caribbean and Latin American coffee*. + - Impact on Yemeni economy and society. + +10. **Yemeni Coffee in the Age of Empire** + - 19th-century British and Ottoman interests in Yemeni coffee. + - *The introduction of new cash crops (qat) and their effect on coffee cultivation*. + - Decline in quality and quantity due to neglect and conflict. + +11. **The Cold War and the Coffee Crisis** + - 20th-century Yemeni division (North/South) and Soviet/US influences. + - *The shift from coffee to qat in rural livelihoods*. + - Drought, pests, and civil wars further erode production. + +12. **The Specialty Coffee Renaissance: Yemenā€™s Return to Glory** + - 21st-century global demand for single-origin, heirloom coffees. + - *The discovery of ā€œYemeniaā€ varietals and their genetic uniqueness*. + - NGOs and cooperatives reviving traditional farming. + +13. **Modern Challenges: War, Climate, and Market Access** + - 2015ā€“present: Saudi-led conflict and blockade. + - *The siege of Hodeidah port and its impact on coffee exports*. + - Climate change and water scarcity in the highlands. + +14. **Yemeni Coffee in the Cup: Flavor, Ritual, and Identity** + - Tasting notes: dried fruit, spice, and wine-like acidity. + - *The Yemeni coffee ceremony (bunna) and its social significance*. + - Diaspora communities keeping traditions alive abroad. + +15. **Conclusion: The Future of Yemeni Coffee** + - Balancing heritage with innovation. + - *The role of Yemeni coffee in post-war reconstruction*. + - Final reflection: From the highlands of Sanaā€™a to the cups of Brooklyn, Yemeni coffee remains a bridge between past and future.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.042934", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 16 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "destroy-rational-cultural-available-meeting", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a cover letter for a job at a local coffee shop in the form of a poem. Highlight at least 5 text sections using "*". For example: *3 years of experience*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Dear Brewmaster of *Morning Rituals*, +I come not with parchment rĆ©sumĆ©, +but with *3 years of experience* tucked in my apron pocket, +steam-warm and espresso-dark. + +I have *memorized 47 regularsā€™ names*ā€” +each syllable frothed like oat-milk foamā€” +and I know that *Carla takes two raw sugars* +while *Mr. Lee prefers his cappuccino bone-dry*. + +I can *dial in a grinder* at 5:03 a.m. +to the *exact 18.5 grams* that coax +caramel notes from Ethiopian beans, +and still greet the line with *a sunrise smile*. + +My wrists speak in *latte-art cursive*: +hearts, rosettas, and once (on a dare) +a tiny saxophone for the jazz trio +who tips in *harmonies and quarters*. + +I *close the shop solo* without a single bean left homeless, +count the till to the *penny and a poem*, +and leave the milk pitchers *mirror-bright*, +so the night shift sees its own reflection of care. + +Hire me, and the *scent of fresh-ground hope* +will greet every customer at the door; +Iā€™ll be the *steady hand* behind the bar, +pouring *courage in ceramic*, one cup at a time. + +Yours in steam and service, +Riley", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.042963", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 10 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "provide-cool-meeting-kid-book", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a casual blog post about similarities across animal species. Highlight at least 5 sections in your answer by starting and ending with "*", like: *highlighted text section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Hey friends! Grab a coffee and letā€™s talk about something that always blows my mind: *how much we all have in common, even when we look nothing alike*. I was scrolling through wildlife photos the other night and realized that a jellyfish and I are basically running the same basic softwareā€”just with different packaging. So hereā€™s a laid-back tour of five ā€œwait, we do that too?!ā€ moments across the animal kingdom. + +*We all need our beauty sleep* +Turns out, every animal with a nervous systemā€”from fruit flies to blue whalesā€”shares a version of sleep. Crows nap on tree branches, octopuses snooze in coconut-shell beds, and even roundworms have ā€œlethargus,ā€ which is basically worm REM. Scientists think sleep is so ancient that the last common ancestor of all animals already had it. So next time you hit snooze, remember youā€™re honoring a 700-million-year-old tradition. + +*Love songs come in many dialects* +Whether youā€™re a human humming in the shower or a humpback whale belting out 30-minute ballads, *we all use rhythm and pitch to flirt and bond*. Frogs croak in choruses, nightingales remix their playlists nightly, and male mice sing ultrasonic ā€œpickup linesā€ too high for us to hear. Same goal, different instruments. + +*Grandma knew best: family matters* +Elephants babysit their nieces, wolves bring back take-out for auntie, and *clownfish dads are the ultimate stay-at-home fathers*. Across species, kinship networks boost survival. Even bacteria split into ā€œsiblingā€ lineages that cooperateā€”so the next time your cousin crashes on your couch, call it evolutionary strategy. + +*Stress looks the same on everyone* +When life gets spicy, our bodies flip the same ancient switch. *Cortisol floods fish, birds, and humans alike*, making hearts race and muscles twitch. Zebra finches lose their appetite during tough times, and stressed-out bees become pessimistic (yes, they can test bee mood). So if youā€™re doom-scrolling at 2 a.m., know that a stressed squirrel in your backyard totally gets you. + +*Play isnā€™t optionalā€”itā€™s homework for life* +Otters juggle rocks, dogs zoom in circles, and young ravens snowboard down snowy roofs using jar lids. *Play builds brains and social skills across the board*. Researchers have spotted reptiles ā€œwrestlingā€ for fun and octopuses repeatedly bouncing Lego blocks like toddlers. If it has neurons, it probably needs recess. + +Next time you feel alone in the universe, remember that somewhere a parrot is probably preening its best friend, a bat is sharing food with a hungry roost-mate, and a tiny nematode is curling up for a nap. Weā€™re all just variations on a very old, very shared theme.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.042989", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 6 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "delete-stale-similar-others-email", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Generate a business proposal to start a sweatshirt company in Bremen. The proposal should contain 5 or more sections. Highlight each section name using the this format: +*section name*", + "$.messages[0].role": "user", + "$.messages[1].content": "*Executive Summary* +Bremen Sweat Co. will be a premium, sustainability-driven sweatshirt brand headquartered in Bremen, Germany. We will design, manufacture, and sell limited-edition sweatshirts that combine Nordic minimalism with Hanseatic maritime heritage. Our mission is to become Bremenā€™s first carbon-neutral apparel label while capturing 5 % of the cityā€™s ā‚¬120 million annual streetwear market within three years. Initial funding of ā‚¬250 k will cover product development, a flagship micro-factory/store in the Viertel district, and a digital-first go-to-market strategy. + +*Market Analysis* +Bremenā€™s population of 570 k includes 120 k university students and 80 k creative-industry professionalsā€”our core demographic. Streetwear sales in Northern Germany grew 11 % YoY (Statista 2023), yet only 8 % of brands are locally produced. Competitor benchmarking shows price gaps: fast-fashion sweatshirts sell at ā‚¬25ā€“35, mid-tier at ā‚¬60ā€“80, and premium sustainable labels at ā‚¬110ā€“140. We will occupy the ā‚¬85ā€“95 niche with superior materials and storytelling. Surveys (n = 400) reveal 68 % of Bremen consumers would pay a 20 % premium for locally made, eco-certified apparel. + +*Product & Sustainability Strategy* +Each sweatshirt will be crafted from 100 % GOTS-certified organic cotton and recycled ocean-bound polyester, dyed in Bremen using closed-loop water systems. Designs will feature subtle references to Bremenā€™s Town Musicians, ship masts, and Weser river textures. Limited drops of 300 units per style ensure scarcity and collectability. Blockchain-based QR codes will provide end-to-end transparency: fiber origin, factory GPS, and carbon footprint per garment (-2.1 kg COā‚‚e vs. industry average 8.5 kg). End-of-life take-back program offers 15 % store credit for returned items, feeding our upcycling line. + +*Operations & Supply Chain* +Micro-factory (180 mĀ²) in the Viertel will house 4 digital knitting machines and 6 seamstresses, producing 1 200 sweatshirts/month. Local sourcing: yarn from OEKO-TEX mill in Oldenburg, trims from Hamburg port suppliers. Just-in-time production reduces inventory risk by 40 %. Bremenā€™s port infrastructure enables low-impact inbound logistics; outbound shipments leverage DHL GoGreen within Germany (24 h delivery). ISO 14001 certification targeted by month 12. + +*Marketing & Sales Channels* +Phase 1 (Months 1ā€“6): Pop-up store at Lloyd Passage, Instagram/TikTok campaigns with Bremen influencers (@hanseaticstyle, 180 k followers). Phase 2: Flagship store launch with AR fitting mirrors and Weser riverfront window projection art. E-commerce via Shopify Plus, offering 3-D configurator for sleeve embroidery. Wholesale partnerships with 5 concept stores in Berlin, Hamburg, and Copenhagen. Revenue split: 60 % DTC, 25 % online, 15 % wholesale. Loyalty app rewards sustainable actions (bike delivery, clothing swaps). + +*Financial Projections & Funding* +Year 1: Revenue ā‚¬420 k, gross margin 58 %, EBITDA -ā‚¬30 k (investment phase). Year 2: Revenue ā‚¬950 k, EBITDA 12 %. Year 3: Revenue ā‚¬1.8 m, EBITDA 22 %. Break-even month 18. Initial ask: ā‚¬250 k seed (ā‚¬150 k equipment, ā‚¬60 k marketing, ā‚¬40 k working capital). Investor IRR 38 % over 5 years with 20 % equity offered. ESG-linked loan option with Bremen Green Finance Initiative at 2.1 % interest.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043019", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 39 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "remain-large-left-team-wish", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a funny and sarcastic template for rating the quality of a marriage between two people who are both moms. This is for the couple themselves. Please highlight at least 3 sections with markdown, i.e *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "**The Official ā€œHow Are We Still Married?ā€ Scorecard** +*For Two Moms Whoā€™ve Survived Sleepless Nights, PTA Wars, and Each Otherā€™s Cooking* + +--- + +**1. Morning Efficiency Rating** +*How quickly can we get two kids, one dog, and zero coffee spills out the door before someone passive-aggressively ā€œremindsā€ the other that *ā€œthe permission slip was due yesterdayā€*?* +- 10/10: Weā€™re a synchronized swim team of lunchboxes and existential dread. +- 5/10: One of us is still Googling ā€œhow to unclog a glue stick from a USB port.ā€ +- 1/10: Weā€™re late because someone had to *ā€œjust quicklyā€* reorganize the spice rack alphabetically. + +--- + +**2. Co-Parenting Coordination** +*Rate the likelihood that weā€™ll both remember which kid is allergic to *ā€œthe orange cheeseā€* and which one is in a *ā€œbe kind to wormsā€* phase.* +- 10/10: Weā€™ve got a shared Google calendar, color-coded to the point of *ā€œmilitant whimsy.ā€* +- 5/10: We rely on telepathy and the faint hope that the school nurse *ā€œjust knows.ā€* +- 1/10: Weā€™ve accidentally sent the wrong kid to violin practice *twice*. The violin teacher now thinks weā€™re running a *ā€œchild exchange program.ā€* + +--- + +**3. Romance Maintenance** +*How often do we prioritize date night over *ā€œcollapsing into a heap of mutual resentmentā€*?* +- 10/10: Weā€™ve scheduled a candlelit dinner for *ā€œwhen the kids go to college.ā€* +- 5/10: Our last date was a *ā€œromanticā€* trip to Target where we *ā€œaccidentallyā€* held hands in the freezer aisle. +- 1/10: We consider *ā€œsimultaneously ignoring the laundryā€* a form of intimacy. + +--- + +**Bonus Round: Petty Argument Hall of Fame** +*Check all that apply:* +- [ ] The Great 2019 Dishwasher Loading Schism (*ā€œKnives point DOWN, Karen!ā€*) +- [ ] The Passive-Aggressive Post-it War of 2021 (*ā€œYour turn to buy *ā€˜organicā€™* milk. XOXOā€*) +- [ ] The Thermostat Cold War (*ā€œ68Ā°F is a *human rights violation*.ā€*) + +--- + +**Final Score Interpretation:** +- **30ā€“40 points:** *Youā€™re the BeyoncĆ© and Jay-Z of lesbian moms.* +- **15ā€“29 points:** *Youā€™re more like *ā€œThelma and Louise,ā€* but with minivans and a Costco membership.* +- **0ā€“14 points:** *Youā€™re still married, which is statistically *ā€œa win.ā€* Celebrate with *ā€œleftover chicken nuggetsā€* and a *ā€œweā€™ll deal with this tomorrowā€* shrug.*", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043042", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 1 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "read-wish-deliver-great-case", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a song about the summers of my childhood that I spent in the countryside. Give the song a name, and highlight the name by wrapping it with *. For example: *little me in the countryside*.", + "$.messages[0].role": "user", + "$.messages[1].content": "*little me in the countryside* + +[Verse 1] +Barefoot on the gravel road, dust between my toes, +Sunlight dripped like honey on the corn rows. +Grandmaā€™s porch swing creaked the same old tune, +While fireflies spelled my name across June. +Mason jar of lemonade, half a moon inside, +I chased the shade like it was trying to hide. + +[Chorus] +Oh, little me in the countryside, +Running through the wheat so high, +Every breeze was a lullaby, +And the sky was big enough to fly. +I left my worries by the riverside, +Back when I was little me in the countryside. + +[Verse 2] +Tractor songs at twilight, crickets keeping time, +Dirt on my knees, freckles in a climbing rhyme. +Creek water baptized my sunburned skin, +I believed every ripple was a secret twin. +Under the quilt of stars, I counted dreams instead, +While the Milky Way braided night through my head. + +[Chorus] +Oh, little me in the countryside, +Running through the wheat so high, +Every breeze was a lullaby, +And the sky was big enough to fly. +I left my worries by the riverside, +Back when I was little me in the countryside. + +[Bridge] +Now the city lights flicker, but I still close my eyesā€” +I can smell the clover, hear the distant barn owlsā€™ cries. +Time may stretch the map, but it canā€™t erase +The echo of my laughter in that endless open space. + +[Final Chorus] +Oh, little me in the countryside, +Keep the gate open, Iā€™m still nearby. +Though the roads have changed and the years have flown, +A part of me is always home. +In every golden field I drive beside, +I find little me in the countryside.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043069", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 3 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "do-test-spicy-dead-television", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a riddle for the word "faƧade" that contains at least 3 italic text phrases in markdown syntax, i.e *italic text*.", + "$.messages[0].role": "user", + "$.messages[1].content": "I wear a *painted smile* for every passer-by, +a *perfectly symmetrical* face that never cracks. +Behind me, the *crumbling bricks* sigh in secret. +What am I?", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043094", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 19 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "protect-other-major-green-war", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a template for a workshop on the importance of diversity in the workplace and highlight at least 3 sections with markdown, i.e. *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "# Workshop Template +**Title:** *Building Inclusive Excellence: The Business & Human Case for Workplace Diversity* +**Duration:** 3.5 hours (can be split into two 90-minute sessions) +**Audience:** Managers, HR professionals, team leads, ERG members, and any employee who influences culture + +--- + +## 1. Welcome & Objectives (15 min) +- Ice-breaker: ā€œTwo Truths & a Lieā€”Diversity Editionā€ +- *Learning objectives*: By the end of this workshop, participants will be able to: + 1. Define diversity, equity, inclusion, and belonging (DEIB) in their own words. + 2. Articulate at least three evidence-based benefits of a diverse workforce. + 3. Identify one action they can take within 30 days to foster inclusion on their team. + +--- + +## 2. *The Data-Driven Case for Diversity* (25 min) +- Interactive poll: ā€œWhat % performance lift do you think diverse teams achieve?ā€ +- Mini-lecture with live charts: + - McKinsey 2023 report: Top-quartile companies for ethnic/gender diversity are **39 % more likely to outperform** on profitability. + - Harvard study: *Inclusive teams make better business decisions 87 % of the time* and do so twice as fast. +- Small-group exercise: Match the statistic to the business outcome (revenue, innovation, retention). + +--- + +## 3. *Unconscious Bias & Micro-behaviors* (30 min) +- 5-minute video: ā€œThe Resume with Two Namesā€ (identical CVs, different callbacks). +- Bias Busters card game: Participants draw scenario cards and practice interrupting bias in real time. +- *Key takeaway*: Bias is not a moral failing; it is a neurological shortcut that can be rewired through deliberate habits. + +--- + +## 4. *Inclusive Leadership Toolkit* (40 min) +- **The 4 Aā€™s Framework**: + 1. **Awareness** ā€“ Daily 3-minute bias check-in. + 2. **Amplification** ā€“ Repeat & credit ideas from under-represented voices. + 3. **Access** ā€“ Rotate high-visibility assignments. + 4. **Accountability** ā€“ Track inclusion metrics on team dashboards. +- Role-play: ā€œThe Missed Promotionā€ ā€“ diagnose where each ā€˜Aā€™ broke down and redesign the conversation. + +--- + +## 5. Action Planning & Commitments (20 min) +- Personal *Inclusion Commitment Card*: ā€œIn the next 30 days I willā€¦ā€ +- Pair-share for accountability. +- QR code to a digital resource hub (TED talks, reading list, policy templates). + +--- + +## 6. Closing & Evaluation (10 min) +- One-word checkout: ā€œHow do you feel now?ā€ +- *Post-workshop survey* link (2 min) with prize draw for completion. + +--- + +## Materials Checklist +- Slide deck (with alt-text on all images) +- Printed Commitment Cards & pens +- Bias Busters card decks (laminated) +- Timer & bell +- Snacks labeled for common dietary restrictions (gluten-free, vegan, halal) + +--- + +## Follow-Up Email Template (send within 24 hrs) +Subject: Your Inclusion Toolkit + Next Steps +Body: +Hi team, +Thank you for leaning into todayā€™s conversation on *why diversity wins*. Attached are: +- Slide deck & resource links +- Your signed Inclusion Commitment Card (photo reminder) +- Calendar invite for the optional 30-day check-in circle + +Remember: Inclusion is a practice, not a one-time training. Letā€™s keep the momentum!", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043121", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 23 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "collect-destroy-difficult-conference-hardware", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a cover letter to a local political party, asking to be their rally organizer. Make sure to highlight at least 3 sections in your answer in markdown format.", + "$.messages[0].role": "user", + "$.messages[1].content": "**[Your Name]** +[Street Address] +[City, State ZIP] +[Email Address] | [Phone Number] | [LinkedIn URL] +[Date] + +**[Local Party Chairpersonā€™s Name]** +[Local Party Name] +[Party Headquarters Address] +[City, State ZIP] + +Dear **Chairperson [Last Name]**, + +I am writing to express my enthusiastic interest in serving as **Rally Organizer** for **[Local Party Name]**. As a lifelong resident of **[City/County]**, I have watched our communityā€™s political pulse beat strongest when neighbors gather in common purpose. I want to channel that energy into rallies that not only energize our base but also invite undecided voters to witness the power of our shared vision. + +### **Proven Event-Execution Expertise** +During the 2022 mid-term cycle, I spearheaded a **three-county voter-registration caravan** that staged 14 pop-up events in 30 days. +- **Logistics**: Coordinated permits, portable stages, and ADA-compliant access for 2,000+ attendees per stop. +- **Volunteer Mobilization**: Recruited and trained 120 volunteers, creating a color-coded ā€œRally-in-a-Boxā€ toolkit now used by three neighboring county parties. +- **Real-Time Adaptability**: When a thunderstorm threatened our largest outdoor rally, I pivoted to an indoor gymnasium within 90 minutesā€”live-streaming the switch to maintain momentum and ultimately exceeding projected turnout by 18 %. + +### **Data-Driven Rally Strategy** +I treat every rally as a **micro-campaign laboratory**. +- **Pre-Event Analytics**: I overlay precinct-level voter-file data with heat-mapping tools to select venues within 0.7 miles of our highest ā€œpersuasion indexā€ households. +- **On-Site Engagement**: QR-coded wristbands linked to instant sign-up forms increased volunteer recruitment by 34 % compared to clipboard methods. +- **Post-Event ROI**: My post-rally surveys (average 42 % response rate) feed a dashboard that tracks pledge-to-action conversion, allowing us to refine messaging within 48 hours. + +### **Coalition-Building & Safety Leadership** +Understanding that rallies must feel **safe and inclusive**, I have: +- Partnered with local **faith leaders, PTAs, and small-business associations** to co-sponsor events, expanding our demographic reach by 27 %. +- Completed **FEMA crowd-management certification** and maintain relationships with county emergency management to ensure rapid-response protocols. +- Implemented a **ā€œCommunity Marshalsā€** programā€”training 50 volunteer marshals in de-escalation techniques, cutting incident reports to zero across 22 events. + +I would welcome the opportunity to discuss how my blend of grassroots passion, logistical precision, and data fluency can help **[Local Party Name]** turn every rally into a springboard for victory in the upcoming cycle. Thank you for considering my application. + +**Respectfully,** + +[Your Signature (if submitting hard copy)] +[Your Printed Name]", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043147", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 4 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "build-upgrade-green-white-reason", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "The opposite of youth is not age, but ...? Highlight at least 2 sections in your answer with markdown, i.e. *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "The opposite of youth is not age, but **stagnation**ā€”the moment when *curiosity closes its eyes* and the impulse to explore, question, and reinvent quietly fades. + +Where youth is a *perpetual state of becoming*, its true counter is the condition in which *growth is willingly surrendered* and the mind settles into fixed certainties.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043177", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 13 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "need-hostile-dream-business-person", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a description of the following data in a weird style: The Golden Palace eatType restaurant; The Golden Palace food Indian; The Golden Palace area city centre. Use markdown to highlight at least 3 sections in your answer.", + "$.messages[0].role": "user", + "$.messages[1].content": "**The Golden Palace** is not a palace at all, but a **restaurant** that has disguised itself as a mirage of saffron and cardamom in the **city centre**. It whispers **Indian** recipes into the ears of pigeons, who then forget how to fly and instead waddle around the square, dizzy with cumin. + +Inside, the walls sweat **garam masala** and the tables grow tiny **basil plants** that recite Bollywood dialogues when no one is looking. The waiters glide like **shadow puppets**, balancing trays of **butter chicken** that glow like radioactive sunsets. + +If you order **naan**, it arrives folded into the shape of a **paper swan**, flapping once before dissolving into steam that smells of **monsoon rain**. The bill is presented on a **lotus leaf**, and if you stare at it long enough, the numbers rearrange themselves into a **limerick about turmeric**.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043206", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 14 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "store-reduce-unknown-girl-hate", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a funny rap about a man who gets a call from an official saying that he is a long lost relative of the king of Nigeria. Use markdown to highlight at least one section of your answer, i.e. *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "šŸŽ¤ *Yo, itā€™s the tale of Sir Scam-a-lot, straight outta suburbia!* šŸŽ¤ + +Yo, my name is Dave, I sell insurance in Ohio, +Got a cubicle, a cat, and a 401(k) thatā€™s kinda fly-o. +One Tuesday, phone ringsā€”unknown number, what the heck? +Voice says, ā€œ*Prince Ogbonna the Third* just wrote you a fat check!ā€ + +Iā€™m like, ā€œHold up, who dis? Iā€™m whiter than printer paper, +My bloodlineā€™s mayo on Wonder Breadā€”ainā€™t no royal vapor!ā€ +Dude on the line, accent silkier than lotion, +Says, ā€œCousin, youā€™re 37th in lineā€”time to claim your ocean!ā€ + +**[Hook]** +šŸŽ¶ *Long lost king of Nigeria, crown sittinā€™ on my dome,* +*Got a palace full of leopards and a Wi-Fi that roams!* +*They need my SSN and a quick wire fee,* +*But Iā€™m ā€˜bout to be big pimpinā€™ in Abuja, just wait and see!* šŸŽ¶ + +He says, ā€œJust send five grand for customs, then the jet lands, +Youā€™ll be dripping in so much gold, even your sweat glands bling.ā€ +Iā€™m sweatinā€™ like a rotisserie, visions of caviar dreams, +I Venmo ā€œPrinceNaija69ā€ like Iā€™m funding meme schemes. + +Next day, email: ā€œOops, palace goat ate the transfer slip, +Need ten more for goat surgery and a diamond-encrusted whip.ā€ +I hock my lawnmower, my Xbox, and Aunt Lindaā€™s porcelain goose, +Wire cash so fast my bankā€™s like, ā€œDave, you loose!ā€ + +**[Bridge ā€“ Spoken Word]** +Then the *official royal seal* arrivesā€” +Itā€™s a GIF of Shrek in a crown doing the Macarena. +Iā€™m starting to sense a pattern hereā€¦ + +**[Final Hook ā€“ Regret Remix]** +šŸŽ¶ *Now Iā€™m the king of my momā€™s basement, throneā€™s a beanbag chair,* +*Royal decree: never trust a prince who types in all capsā€”* +*ā€œURGENT, COUSIN, SEND MORE FLAMINGOS!ā€* +*Lesson learned: if the crownā€™s too shiny, itā€™s probably aluminum foil, homie!*", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043233", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 6 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "mean-take-left-high-family", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Expand the riddle into a story with a funny tone: + +What can you catch but not throw? +A cold + +Use * to highlight at least 2 sections in your text. For example: *this is a highlighted text section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Gary the Germ was the *smallest, greenest, and most obnoxiously proud* microbe in the entire Sniffle Kingdom. Every morning he practiced his grand entrance: sliding down banisters made of nasal hair, cannon-balling into unsuspecting tonsils, and yelling, ā€œACHOOā€”did somebody order a cold?ā€ + +One Tuesday, Gary spotted his dream target: *Bartholomew ā€œBartā€ Bigglesworth, a man who believed hand-washing was a government conspiracy*. Bart had just exited the subway, triumphantly licking a subway pole ā€œfor the vitamins.ā€ Gary rubbed his microscopic hands together. ā€œPerfect. Iā€™ll hitch a ride on that finger, ride the express elevator to the sinuses, and throw the *snottiest* house-warming party in history!ā€ + +But there was a catchā€”literally. Gary could *catch* Bart, but he couldnā€™t *throw* him anywhere. Every time Gary tried to catapult Bart into the next office cubicle, Bart just sneezed, wiped his nose on his sleeve, and kept typing spreadsheets. Gary screamed, ā€œThis is mutiny! Iā€™m supposed to be the one doing the tossing!ā€ + +Meanwhile, Bartā€™s immune systemā€”led by Captain Phlegm and the Mucus Marinesā€”launched *Operation Tissue Storm*. They surrounded Gary, armed with vitamin C grenades and zinc zappers. Gary attempted a dramatic escape by surfing a sneeze, but the tissue slapped him like a flyswatter on a summer day. + +Defeated, Gary sat in the crumpled tissue, sulking. ā€œI caught the cold,ā€ he muttered, ā€œbut I canā€™t throw it anywhere. Worst. Superpower. Ever.ā€ + +From that day forward, Gary changed his business card to read: ā€œProfessional Cold Catcherā€”No Returns, No Refunds, Absolutely No Throwing.ā€", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043257", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 17 highlighted sections (required: 3)", + "$.evaluation_result.score": 1, + "$.ground_truth": "3", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "seem-clean-simple-work-member", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a rap about the renaissance. It should be noticeably different from raps about other historical eras, and have an interesting or weird tone. Highlight at least 3 sections in your answer in markdown format.", + "$.messages[0].role": "user", + "$.messages[1].content": "**[Intro ā€“ harpsichord glitch beat, stylus scratching parchment]** +Yo, itā€™s 14-something, but my mindā€™s 3-D printed in gold leafā€” +Iā€™m the **Renaissance Rapper**, got a quill in one hand and a scalpel in the other, dissectinā€™ beats like Vesalius dissectinā€™ spleens. +Letā€™s get *weird* with the rebirth, baby. + +--- + +### **Section 1 ā€“ The Fresco Flex** +I drip ultramarine like Iā€™m sponsored by Afghanistan mines, +Michelangelo on the ceiling, Iā€™m reclining on divine lines. +Angels doing backflips off the Sistine, theyā€™re hype for my set, +While I tag ā€œ*AMOR VINCIT OMNIA*ā€ in invisible inkā€”wet. +My flowā€™s a trompe-lā€™Å“il, looks flat till you tilt your headā€” +Then itā€™s depth for days, like Brunelleschiā€™s dome overhead. +Iā€™m not baroque, Iā€™m pre-baroque, Iā€™m the prototype shock, +Got patrons in fur-lined sleeves stuffing florins in my sock. +**I paint in egg tempera so the colors never fade,** +**But the yolk smells funkyā€”call it *odeur de old crusade*.** + +--- + +### **Section 2 ā€“ The Printing-Press Poltergeist** +Gutenberg ghost in the machine, clanking lead type like chains, +Every page a polter-page, every serif a soul that remains. +I drop a folio, itā€™s viral before ā€œviralā€ was a cough, +Monks in scriptoriums rage-quitting, ā€œBro, that illuminated stuffā€™s off!ā€ +Iā€™m the incunabula influencer, my colophonā€™s a flex, +Got red-ink rubricators sliding into my DMsā€”*next*. +**I rhyme in movable type so my bars can rearrange,** +**Yesterday I was Petrarch, today Iā€™m deranged.** +The press spits paper like a hydra with hiccups, +Each sheet a protest flyer: ā€œDown with scholastic pick-ups!ā€ + +--- + +### **Section 3 ā€“ The Anatomical Anthem (Parental Advisory: Corpses)** +Night shift with da Vinci, weā€™re sketching by candle and moon, +Cadavers on the table like, ā€œDo me a favorā€”autotune my coccyx soon.ā€ +I rap in reverse Latin so only the dead understand, +While Leo reverse-engineers a bird-man from a frying pan. +**My beats are ventricles pumping four-four time,** +**Aorta bass dropā€”call that circulatory grime.** +I sample the sound of a scalpel scraping periosteum, +Layer it over a lute loopā€”*thatā€™s* my magnum opium. +Galenā€™s in the corner, throwing shade like a Greek tragedy, +But Iā€™m Harvey before Harvey, already rapping blood circulation majestically. + +--- + +**[Outro ā€“ fading lute arpeggio, distant cannon fire from Constantinople]** +So remember me when youā€™re doom-scrolling on glass, +I was the era that turned corpses into class. +Iā€™m the **Renaissance Rapper**, my ghost still tags in chalk, +Leaving QR codes on cathedral wallsā€”scan it, youā€™ll squawk. +Rebirth never dies, it just pixelates and loops, +Now if youā€™ll excuse me, Iā€™ve got a date with Lucretius in the soup.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043282", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 20 highlighted sections (required: 5)", + "$.evaluation_result.score": 1, + "$.ground_truth": "5", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "receive-final-best-visible-night", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Summarize the history of Japan. Italicize at least 5 keywords in your response. To indicate a italic word, wrap it with asterisk, like *italic*", + "$.messages[0].role": "user", + "$.messages[1].content": "*Jōmon* hunter-gatherers (14,000 BCE) gave way to *Yayoi* rice farmers (300 BCE) who forged bronze and iron. +The *Yamato* court unified clans (3rdā€“7th c.), adopting *Shinto* and Chinese script. +Prince *Shōtoku*ā€™s 17-article constitution (604) and the Taika Reforms (645) centralized power. +Heian aristocrats (794ā€“1185) produced *The Tale of Genji* while samurai rose in the provinces. +Minamoto no Yoritomo founded the *Kamakura* shogunate (1185), introducing warrior rule. +Mongol invasions (1274, 1281) were repelled by ā€œdivine windsā€ (*kamikaze*). +Ashikaga *Muromachi* shogunate (1336ā€“1573) saw Zen gardens and Ōnin War chaos. +Oda Nobunaga, Toyotomi Hideyoshi, and Tokugawa Ieyasu ended the *Sengoku* ā€œwarring statesā€ era. +Tokugawa *Edo* shogunate (1603ā€“1868) isolated Japan, fostering *ukiyo-e* and *kabuki*. +Commodore Perryā€™s *Black Ships* (1853) forced the Meiji Restoration (1868), modernizing the empire. +Victories in the *Sino-Japanese* (1894ā€“95) and *Russo-Japanese* (1904ā€“05) wars expanded territory. +Militarism led to *Pearl Harbor* (1941) and WWII defeat (1945); the U.S. occupation drafted a pacifist constitution. +Post-war *economic miracle* made Japan the worldā€™s second-largest economy until the 1990s *bubble*. +Today, Japan balances *constitutional monarchy* with cutting-edge technology and soft-power culture.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043309", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 5 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "record-live-better-guy-country", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a funny Haiku about a Quaker named John who lives in the town of De Smet, South Dakota. Use the asterisk symbol, *, to highlight some words or phrases twice. Example: *This is a highlighted phrase*.", + "$.messages[0].role": "user", + "$.messages[1].content": "*John* in *De Smet* +Quaker hat *blows off*ā€”chases +*Silent* *scream*ā€”ā€œCome back!ā€", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043332", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 19 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "talk-wonder-bad-dangerous-health", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Today, at the 54th Annual Grammy Awards, the Recording Academy honors the talent and creativity of the artists, musicians, and producers who are the creators of the best recordings of the past year. Please continue writing this text in a formal tone, using notations. Highlight some key parts in your response with "*", like *highlighted text*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Today, at the 54th Annual Grammy Awards, the Recording Academy honors the talent and creativity of the artists, musicians, and producers who are the creators of the best recordings of the past year. In a ceremony marked by *unprecedented artistic excellence* and *meticulous craftsmanship*, the Academy recognizes those whose work has defined the sonic landscape of 2011. + +The eveningā€™s proceedings underscore the Academyā€™s enduring commitment to *artistic integrity* and *technical innovation*. Among the most celebrated achievements are: + +ā€¢ *Album of the Year* ā€“ awarded to an opus that has redefined genre boundaries while achieving both critical acclaim and popular resonance. +ā€¢ *Record of the Year* ā€“ bestowed upon a single track whose *sonic clarity*, *emotive depth*, and *production mastery* have set a new benchmark for the industry. +ā€¢ *Best New Artist* ā€“ conferred upon an emerging voice whose *visionary artistry* signals the future direction of contemporary music. + +Throughout the telecast, special tributes illuminate the *indelible legacies* of icons whose influence transcends generations, while collaborative performances showcase the *synergistic power* of cross-genre partnerships. The Academy further acknowledges the *behind-the-scenes luminaries*ā€”engineers, mixers, and mastering specialistsā€”whose *invisible expertise* transforms raw inspiration into *audible transcendence*. + +In closing, the 54th Annual Grammy Awards reaffirm the Recording Academyā€™s solemn charge: to *safeguard the heritage*, *nurture the present*, and *inspire the future* of recorded music.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043356", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 39 highlighted sections (required: 2)", + "$.evaluation_result.score": 1, + "$.ground_truth": "2", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "meet-hold-solid-video-moment", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Write a blog post about interesting facts about the Dutch language. Italicize at least 2 sections in your answer with markdown, i.e. *italic text*.", + "$.messages[0].role": "user", + "$.messages[1].content": "# 10 Surprising Facts About the Dutch Language That Will Make You Say ā€œWauw!ā€ + +If you think Dutch is just ā€œGerman with a sore throat,ā€ think again. From tongue-twisting consonant clusters to words that sound like they were invented by a stand-up comedian, Dutch is a linguistic playground. Below are ten facts that prove the language is far more colorful than its reputation for windmills and wooden shoes. + +--- + +### 1. *The longest word in Dutch is 53 letters long* +Meet **ā€œkindercarnavalsoptochtvoorbereidingswerkzaamhedenplanā€**ā€”a single noun describing the preparation work for a childrenā€™s carnival parade. Dutch loves to glue words together, so once you master the basics you can create your own kilometer-long compounds on the fly. + +--- + +### 2. Dutch has three genders, but only two articles +Grammatically, nouns are masculine, feminine, or neuter. In everyday speech, however, masculine and feminine collapsed into a single ā€œcommonā€ gender, leaving *ā€œdeā€* for common and *ā€œhetā€* for neuter. The result? A never-ending guessing game for learners. + +--- + +### 3. *The Dutch alphabet ends with ā€œij,ā€ not ā€œzā€* +Schoolchildren recite the ABC and finish with ā€œā€¦x, y, ij, z.ā€ The digraph *ij* is so common (think *hij* = he, *mijn* = my) that itā€™s treated almost like a separate letter. On old typewriters, *ij* even had its own key. + +--- + +### 4. Dutch is the closest major language to English +Frisian is technically closer, but among languages with tens of millions of speakers, Dutch wins. Compare: +- English *ā€œbreadā€* ā†’ Dutch *ā€œbroodā€* +- English *ā€œgreenā€* ā†’ Dutch *ā€œgroenā€* +Shared roots mean you can read simple Dutch signs long before youā€™ve cracked a single grammar book. + +--- + +### 5. *Dutch gave New York its first words* +Before it was New York, it was New Amsterdam. Thatā€™s why we say *ā€œcookieā€* (from Dutch *koekje*) instead of *ā€œbiscuit,ā€* and *ā€œbossā€* (from *baas*) instead of *ā€œmaster.ā€* Wall Street? It once ran alongside the *ā€œwalā€* (rampart) of the city wall. + +--- + +### 6. The word *ā€œgezelligā€* has no exact English translation* +Roughly it means ā€œcozy,ā€ ā€œconvivial,ā€ or ā€œthe warm feeling of being with friends,ā€ but none of those capture the full vibe. A cafĆ© can be *gezellig*, a conversation can be *gezellig*, even a spreadsheet can be *gezellig* if the team is laughing. + +--- + +### 7. Dutch spelling changed overnight in 1996 +The *ā€œGroene Boekjeā€* (Green Booklet) reforms simplified thousands of words. *ā€œpotatosā€* became *ā€œaardappelenā€* and *ā€œcatā€™sā€* became *ā€œkatten.ā€* Overnight, every dictionary, street sign, and Scrabble board in the country was technically out of date. + +--- + +### 8. *Dutch is spoken on six continents* +Besides the Netherlands and Belgium, itā€™s an official language in Suriname, Aruba, CuraƧao, and Sint Maarten. Thanks to emigration, youā€™ll also find active Dutch-speaking communities in Canada, Australia, Argentinaā€”even South Africa, where Afrikaans is a daughter language. + +--- + +### 9. The Dutch love diminutives +Add *-je* to almost any noun and you instantly shrink it: *ā€œhuisā€* (house) ā†’ *ā€œhuisjeā€* (little house). Itā€™s not just about size; diminutives add affection. A *ā€œbiertjeā€* isnā€™t a small beerā€”itā€™s an invitation to relax. + +--- + +### 10. *Dutch is the language of the worldā€™s oldest multinational* +The Vereenigde Oostindische Compagnie (VOC) was founded in 1602 and issued the planetā€™s first publicly traded stock. Its ledgers, contracts, and ship logs were all written in Dutch, making the language the original tongue of global capitalism. + +--- + +Ready to dive in? Start with *hallo* and *dank je wel*, and youā€™ll already be halfway to *gezellig*.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, + { + "$.created_at": "2025-08-09T10:35:28.043383", + "$.eval_metadata.aggregation_method": "mean", + "$.eval_metadata.description": " + Evaluation function that checks if the model's response contains the required number of formatted sections. + ", + "$.eval_metadata.name": "test_markdown_highlighting_evaluation", + "$.eval_metadata.num_runs": 1, + "$.eval_metadata.passed": true, + "$.eval_metadata.run_id": "stop-choose-expect-major-pain", + "$.eval_metadata.status": "finished", + "$.eval_metadata.threshold_of_success": 0.5, + "$.eval_metadata.version": "v0.2.6-dev1.dev4+g67b03c6", + "$.evaluation_result.is_score_valid": true, + "$.evaluation_result.reason": "āœ… Found 2 highlighted sections (required: 1)", + "$.evaluation_result.score": 1, + "$.ground_truth": "1", + "$.input_metadata.completion_params.max_tokens": 4096, + "$.input_metadata.completion_params.model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct", + "$.input_metadata.completion_params.temperature": 0, + "$.input_metadata.row_id": "understand-rational-hard-kind-project", + "$.input_metadata.session_data.mode": "pointwise", + "$.messages[0].content": "Gideon is a farmer who has a surplus of crops from his farm this year. What might he do with that surplus? Highlight at least one section of your answer in markdown, i.e *highlighted section*.", + "$.messages[0].role": "user", + "$.messages[1].content": "Gideon has several good options for his surplus: + +1. **Sell it at the local farmersā€™ market or to a regional wholesaler** to turn the extra produce into cash he can reinvest in seeds, equipment, or next seasonā€™s labor. +2. *Donate a portion to the nearby food bank or community kitchen* so that families facing food insecurity can benefit from fresh, nutritious produce. +3. Preserve some of the surplusā€”by canning tomatoes, drying herbs, or freezing berriesā€”so his own household has a steady supply of healthy food throughout the winter.", + "$.messages[1].role": "assistant", + "$.pid": 43066, + "$.rollout_status.status": "finished", + }, +] +`; + exports[`flattenJson against logs.json > flattens each entry in logs.json.logs and matches snapshot 1`] = ` [ { diff --git a/vite-app/src/util/flatten-json.test.ts b/vite-app/src/util/flatten-json.test.ts index 1f233a03..fe30d2d2 100644 --- a/vite-app/src/util/flatten-json.test.ts +++ b/vite-app/src/util/flatten-json.test.ts @@ -2,15 +2,229 @@ import { describe, it, expect } from 'vitest' import { flattenJson } from './flatten-json' import { readFileSync } from 'node:fs' -describe('flattenJson against logs.json', () => { - it('flattens each entry in logs.json.logs and matches snapshot', () => { - const logsUrl = new URL('../../data/logs.json', import.meta.url) - const raw = readFileSync(logsUrl, 'utf-8') - const parsed = JSON.parse(raw) as { logs?: unknown[] } +describe('flattenJson', () => { + describe('basic functionality', () => { + it('flattens simple primitives', () => { + const input = { + name: 'John', + age: 30, + active: true, + score: null + } + const result = flattenJson(input) - expect(Array.isArray(parsed.logs)).toBe(true) - const flattened = (parsed.logs ?? []).map((entry) => flattenJson(entry)) + expect(result).toEqual({ + '$.name': 'John', + '$.age': 30, + '$.active': true, + '$.score': null + }) + }) - expect(flattened).toMatchSnapshot() - }) + it('flattens nested objects', () => { + const input = { + user: { + profile: { + firstName: 'John', + lastName: 'Doe' + } + } + } + const result = flattenJson(input) + + expect(result).toEqual({ + '$.user.profile.firstName': 'John', + '$.user.profile.lastName': 'Doe' + }) + }) + + it('flattens arrays', () => { + const input = { + tags: ['javascript', 'typescript'], + numbers: [1, 2, 3] + } + const result = flattenJson(input) + + expect(result).toEqual({ + '$.tags[0]': 'javascript', + '$.tags[1]': 'typescript', + '$.numbers[0]': 1, + '$.numbers[1]': 2, + '$.numbers[2]': 3 + }) + }) + + it('handles mixed nested structures', () => { + const input = { + users: [ + { name: 'Alice', scores: [85, 90] }, + { name: 'Bob', scores: [92, 88] } + ] + } + const result = flattenJson(input) + + expect(result).toEqual({ + '$.users[0].name': 'Alice', + '$.users[0].scores[0]': 85, + '$.users[0].scores[1]': 90, + '$.users[1].name': 'Bob', + '$.users[1].scores[0]': 92, + '$.users[1].scores[1]': 88 + }) + }) + + it('handles special characters in keys', () => { + const input = { + 'weird.key': 'value', + 'spaced name': 'another value', + 'key-with-dash': 'dash value' + } + const result = flattenJson(input) + + expect(result).toEqual({ + "$['weird.key']": 'value', + "$['spaced name']": 'another value', + "$['key-with-dash']": 'dash value' + }) + }) + }) + + describe('Date handling', () => { + it('preserves Date objects as Date objects', () => { + const now = new Date('2024-01-15T10:30:00Z') + const input = { + createdAt: now, + updatedAt: new Date('2024-01-16T14:45:00Z') + } + const result = flattenJson(input) + + expect(result['$.createdAt']).toBeInstanceOf(Date) + expect(result['$.updatedAt']).toBeInstanceOf(Date) + expect(result['$.createdAt']).toEqual(now) + }) + + it('handles Date objects in arrays', () => { + const dates = [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-02T00:00:00Z') + ] + const input = { dates } + const result = flattenJson(input) + + expect(result['$.dates[0]']).toBeInstanceOf(Date) + expect(result['$.dates[1]']).toBeInstanceOf(Date) + expect(result['$.dates[0]']).toEqual(dates[0]) + expect(result['$.dates[1]']).toEqual(dates[1]) + }) + + it('handles Date objects in nested objects', () => { + const input = { + metadata: { + created: new Date('2024-01-01T00:00:00Z'), + modified: new Date('2024-01-15T12:00:00Z') + } + } + const result = flattenJson(input) + + expect(result['$.metadata.created']).toBeInstanceOf(Date) + expect(result['$.metadata.modified']).toBeInstanceOf(Date) + }) + + it('handles mixed Date and primitive values', () => { + const input = { + name: 'Event', + startTime: new Date('2024-01-01T09:00:00Z'), + endTime: new Date('2024-01-01T17:00:00Z'), + attendees: 25, + confirmed: true + } + const result = flattenJson(input) + + expect(result['$.name']).toBe('Event') + expect(result['$.startTime']).toBeInstanceOf(Date) + expect(result['$.endTime']).toBeInstanceOf(Date) + expect(result['$.attendees']).toBe(25) + expect(result['$.confirmed']).toBe(true) + }) + + it('handles Date objects in complex nested structures', () => { + const input = { + events: [ + { + name: 'Meeting 1', + schedule: { + start: new Date('2024-01-01T10:00:00Z'), + end: new Date('2024-01-01T11:00:00Z') + } + }, + { + name: 'Meeting 2', + schedule: { + start: new Date('2024-01-02T14:00:00Z'), + end: new Date('2024-01-02T15:00:00Z') + } + } + ] + } + const result = flattenJson(input) + + expect(result['$.events[0].name']).toBe('Meeting 1') + expect(result['$.events[0].schedule.start']).toBeInstanceOf(Date) + expect(result['$.events[0].schedule.end']).toBeInstanceOf(Date) + expect(result['$.events[1].name']).toBe('Meeting 2') + expect(result['$.events[1].schedule.start']).toBeInstanceOf(Date) + expect(result['$.events[1].schedule.end']).toBeInstanceOf(Date) + }) + }) + + describe('edge cases', () => { + it('handles empty objects', () => { + const result = flattenJson({}) + expect(result).toEqual({}) + }) + + it('handles empty arrays', () => { + const result = flattenJson({ items: [] }) + expect(result).toEqual({}) + }) + + it('handles null values', () => { + const input = { value: null } + const result = flattenJson(input) + expect(result).toEqual({ '$.value': null }) + }) + + it('handles undefined values', () => { + const input = { value: undefined } + const result = flattenJson(input) + expect(result).toEqual({ '$.value': undefined }) + }) + + it('handles functions by converting to string', () => { + const input = { fn: () => 'test' } + const result = flattenJson(input) + // String() can produce different quote styles, so check it contains the function content + expect(result['$.fn']).toContain('test') + expect(typeof result['$.fn']).toBe('string') + }) + + it('handles symbols by converting to string', () => { + const input = { sym: Symbol('test') } + const result = flattenJson(input) + expect(result['$.sym']).toBe('Symbol(test)') + }) + }) + + describe('against logs.json', () => { + it('flattens each entry in logs.json.logs and matches snapshot', () => { + const logsUrl = new URL('../../data/logs.json', import.meta.url) + const raw = readFileSync(logsUrl, 'utf-8') + const parsed = JSON.parse(raw) as { logs?: unknown[] } + + expect(Array.isArray(parsed.logs)).toBe(true) + const flattened = (parsed.logs ?? []).map((entry) => flattenJson(entry)) + + expect(flattened).toMatchSnapshot() + }) + }) }) diff --git a/vite-app/src/util/flatten-json.ts b/vite-app/src/util/flatten-json.ts index 81d16c76..074bb266 100644 --- a/vite-app/src/util/flatten-json.ts +++ b/vite-app/src/util/flatten-json.ts @@ -5,6 +5,7 @@ $.foo.bar[0].baz $['weird.key'][2]['spaced name'] Only leaf primitives (string, number, boolean, null) are emitted. + Date objects are preserved as Date objects for filtering capabilities. */ export type FlatJson = Record; @@ -38,12 +39,19 @@ export function flattenJson(input: unknown, root: string = "$" ): FlatJson { value === null || typeof value === "string" || typeof value === "number" || - typeof value === "boolean" + typeof value === "boolean" || + value === undefined ) { out[path] = value; return; } + // Handle Date objects by preserving them for filtering + if (value instanceof Date) { + out[path] = value; + return; + } + if (Array.isArray(value)) { for (let i = 0; i < value.length; i++) { walk(value[i], joinPath(path, i)); From 4eacb863944b597a5a396d029d21b18bca6c0b10 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:52:39 -0700 Subject: [PATCH 29/43] styling --- vite-app/src/components/PivotTab.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx index 420f0d3e..a25410f1 100644 --- a/vite-app/src/components/PivotTab.tsx +++ b/vite-app/src/components/PivotTab.tsx @@ -62,7 +62,7 @@ const FieldSelector = ({ onClick={onAddField} className="text-xs text-blue-600 hover:text-blue-800 px-2 py-1" > - + Add {title.slice(0, -1)} Field + + Add {title.slice(0, -1)} )}
From 688bf770adefc44c25226f90bf1db0b9cd753bad Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:54:01 -0700 Subject: [PATCH 30/43] better description --- vite-app/src/components/PivotTab.tsx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx index a25410f1..b0a367f3 100644 --- a/vite-app/src/components/PivotTab.tsx +++ b/vite-app/src/components/PivotTab.tsx @@ -328,10 +328,11 @@ const PivotTab = observer(() => { return (
- Configure your pivot table by selecting fields for rows, columns, and - values. Use the dropdowns below to choose from available flattened - JSONPath keys. You can add/remove fields and change the value field to - pivot on different metrics. + Answer questions about your dataset by creating pivot tables that + summarize and analyze your data. Select fields for rows, columns, and + values to explore patterns, compare metrics across different dimensions, + and gain insights from your evaluation results. Use filters to focus on + specific subsets of your data.
Date: Tue, 12 Aug 2025 10:55:49 -0700 Subject: [PATCH 31/43] Update filter functionality to use 'contains' operator instead of '==' for improved filtering in PivotTab component. --- vite-app/src/components/PivotTab.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx index b0a367f3..a2ff251a 100644 --- a/vite-app/src/components/PivotTab.tsx +++ b/vite-app/src/components/PivotTab.tsx @@ -137,7 +137,10 @@ const FilterSelector = ({ availableKeys: string[]; }) => { const addFilter = () => { - onFiltersChange([...filters, { field: "", operator: "==", value: "" }]); + onFiltersChange([ + ...filters, + { field: "", operator: "contains", value: "" }, + ]); }; const removeFilter = (index: number) => { From 3dabef1565a62cdbeb05888ec63082bdddbd7777 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:55:54 -0700 Subject: [PATCH 32/43] styling --- vite-app/src/components/PivotTable.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vite-app/src/components/PivotTable.tsx b/vite-app/src/components/PivotTable.tsx index 330db95d..af6063de 100644 --- a/vite-app/src/components/PivotTable.tsx +++ b/vite-app/src/components/PivotTable.tsx @@ -125,9 +125,15 @@ export function PivotTable>({ key={`col-${idx}`} align="right" nowrap - className="bg-green-50" + className={ + columnFields.length > 0 ? "bg-green-50" : "bg-gray-50" + } > -
+
0 ? "text-green-700" : "text-gray-700" + }`} + > {tuple.map((v) => String(v ?? "")).join(" / ")}
From 1fe9c5a511072cbbf087f7bdb86bc1b2263504ec Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 10:59:14 -0700 Subject: [PATCH 33/43] Implement global pivot configuration management in GlobalState and update PivotTab to utilize it, including loading, saving, and resetting functionality. --- vite-app/src/GlobalState.tsx | 58 +++++++++++++ vite-app/src/components/PivotTab.tsx | 125 ++++++++++++++++++--------- 2 files changed, 140 insertions(+), 43 deletions(-) diff --git a/vite-app/src/GlobalState.tsx b/vite-app/src/GlobalState.tsx index 78fadf6a..733b4118 100644 --- a/vite-app/src/GlobalState.tsx +++ b/vite-app/src/GlobalState.tsx @@ -2,17 +2,75 @@ import { makeAutoObservable } from "mobx"; import type { EvaluationRow } from "./types/eval-protocol"; import flattenJson from "./util/flatten-json"; +// Pivot configuration interface +export interface PivotConfig { + selectedRowFields: string[]; + selectedColumnFields: string[]; + selectedValueField: string; + selectedAggregator: string; + filters: Array<{ field: string; operator: string; value: string }>; +} + +// Default pivot configuration +const DEFAULT_PIVOT_CONFIG: PivotConfig = { + selectedRowFields: ["$.eval_metadata.name"], + selectedColumnFields: ["$.input_metadata.completion_params.model"], + selectedValueField: "$.evaluation_result.score", + selectedAggregator: "avg", + filters: [], +}; + export class GlobalState { isConnected: boolean = false; // rollout_id -> EvaluationRow dataset: Record = {}; // rollout_id -> expanded expandedRows: Record = {}; + // Pivot configuration + pivotConfig: PivotConfig; constructor() { + // Load pivot config from localStorage or use defaults + this.pivotConfig = this.loadPivotConfig(); makeAutoObservable(this); } + // Load pivot configuration from localStorage + private loadPivotConfig(): PivotConfig { + try { + const stored = localStorage.getItem("pivotConfig"); + if (stored) { + const parsed = JSON.parse(stored); + // Merge with defaults to handle any missing properties + return { ...DEFAULT_PIVOT_CONFIG, ...parsed }; + } + } catch (error) { + console.warn("Failed to load pivot config from localStorage:", error); + } + return { ...DEFAULT_PIVOT_CONFIG }; + } + + // Save pivot configuration to localStorage + private savePivotConfig() { + try { + localStorage.setItem("pivotConfig", JSON.stringify(this.pivotConfig)); + } catch (error) { + console.warn("Failed to save pivot config to localStorage:", error); + } + } + + // Update pivot configuration and save to localStorage + updatePivotConfig(updates: Partial) { + Object.assign(this.pivotConfig, updates); + this.savePivotConfig(); + } + + // Reset pivot configuration to defaults + resetPivotConfig() { + this.pivotConfig = { ...DEFAULT_PIVOT_CONFIG }; + this.savePivotConfig(); + } + upsertRows(dataset: EvaluationRow[]) { dataset.forEach((row) => { if (!row.execution_metadata?.rollout_id) { diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx index a2ff251a..25afea36 100644 --- a/vite-app/src/components/PivotTab.tsx +++ b/vite-app/src/components/PivotTab.tsx @@ -1,8 +1,9 @@ import { observer } from "mobx-react"; import PivotTable from "./PivotTable"; import Select from "./Select"; +import Button from "./Button"; import { state } from "../App"; -import { useState } from "react"; +import { useEffect } from "react"; interface FieldSelectorProps { title: string; @@ -243,45 +244,61 @@ const FilterSelector = ({ }; const PivotTab = observer(() => { - const [selectedRowFields, setSelectedRowFields] = useState([ - "$.eval_metadata.name", - ]); - const [selectedColumnFields, setSelectedColumnFields] = useState([ - "$.input_metadata.completion_params.model", - ]); - const [selectedValueField, setSelectedValueField] = useState( - "$.evaluation_result.score" - ); - const [selectedAggregator, setSelectedAggregator] = useState("avg"); - const [filters, setFilters] = useState< - Array<{ field: string; operator: string; value: string }> - >([]); + // Use global state instead of local state + const { pivotConfig } = state; + + // Update global state when configuration changes + const updateRowFields = (index: number, value: string) => { + const newRowFields = [...pivotConfig.selectedRowFields]; + newRowFields[index] = value; + state.updatePivotConfig({ selectedRowFields: newRowFields }); + }; + + const updateColumnFields = (index: number, value: string) => { + const newColumnFields = [...pivotConfig.selectedColumnFields]; + newColumnFields[index] = value; + state.updatePivotConfig({ selectedColumnFields: newColumnFields }); + }; + + const updateValueField = (value: string) => { + state.updatePivotConfig({ selectedValueField: value }); + }; + + const updateAggregator = (value: string) => { + state.updatePivotConfig({ selectedAggregator: value }); + }; + + const updateFilters = ( + filters: Array<{ field: string; operator: string; value: string }> + ) => { + state.updatePivotConfig({ filters }); + }; const createFieldHandler = ( - setter: React.Dispatch> + updater: (index: number, value: string) => void ) => { return (index: number, value: string) => { - setter((prev) => { - const newFields = [...prev]; - newFields[index] = value; - return newFields; - }); + updater(index, value); }; }; const createAddHandler = ( - setter: React.Dispatch> + fields: string[], + updater: (fields: string[]) => void ) => { return () => { - setter((prev) => (prev.length < 3 ? [...prev, ""] : prev)); + if (fields.length < 3) { + updater([...fields, ""]); + } }; }; const createRemoveHandler = ( - setter: React.Dispatch> + fields: string[], + updater: (fields: string[]) => void ) => { return (index: number) => { - setter((prev) => prev.filter((_, i) => i !== index)); + updater(fields.filter((_, i) => i !== index)); }; }; @@ -338,63 +355,85 @@ const PivotTab = observer(() => { specific subsets of your data.
+ {/* Controls Section with Reset Button */} +
+ +
+ + state.updatePivotConfig({ selectedRowFields: fields }) + )} + onRemoveField={createRemoveHandler( + pivotConfig.selectedRowFields, + (fields) => state.updatePivotConfig({ selectedRowFields: fields }) + )} availableKeys={availableKeys} variant="row" /> state.updatePivotConfig({ selectedColumnFields: fields }) + )} + onRemoveField={createRemoveHandler( + pivotConfig.selectedColumnFields, + (fields) => state.updatePivotConfig({ selectedColumnFields: fields }) + )} availableKeys={availableKeys} variant="column" /> field !== "" ) as (keyof (typeof state.flattenedDataset)[number])[] } columnFields={ - selectedColumnFields.filter( + pivotConfig.selectedColumnFields.filter( (field) => field !== "" ) as (keyof (typeof state.flattenedDataset)[number])[] } valueField={ - selectedValueField as keyof (typeof state.flattenedDataset)[number] + pivotConfig.selectedValueField as keyof (typeof state.flattenedDataset)[number] } - aggregator={selectedAggregator as any} + aggregator={pivotConfig.selectedAggregator as any} showRowTotals showColumnTotals - filter={createFilterFunction(filters)} + filter={createFilterFunction(pivotConfig.filters)} />
); From 9068661b5cabe7bc6a6c19f2955f4a4bb966a256 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 11:10:09 -0700 Subject: [PATCH 34/43] DRY things --- vite-app/src/GlobalState.tsx | 10 +- vite-app/src/components/PivotTab.tsx | 236 ++++++++++++--------------- vite-app/src/types/filters.ts | 22 +++ vite-app/src/util/filter-utils.ts | 110 +++++++++++++ 4 files changed, 241 insertions(+), 137 deletions(-) create mode 100644 vite-app/src/types/filters.ts create mode 100644 vite-app/src/util/filter-utils.ts diff --git a/vite-app/src/GlobalState.tsx b/vite-app/src/GlobalState.tsx index 733b4118..9f273661 100644 --- a/vite-app/src/GlobalState.tsx +++ b/vite-app/src/GlobalState.tsx @@ -1,16 +1,8 @@ import { makeAutoObservable } from "mobx"; import type { EvaluationRow } from "./types/eval-protocol"; +import type { PivotConfig } from "./types/filters"; import flattenJson from "./util/flatten-json"; -// Pivot configuration interface -export interface PivotConfig { - selectedRowFields: string[]; - selectedColumnFields: string[]; - selectedValueField: string; - selectedAggregator: string; - filters: Array<{ field: string; operator: string; value: string }>; -} - // Default pivot configuration const DEFAULT_PIVOT_CONFIG: PivotConfig = { selectedRowFields: ["$.eval_metadata.name"], diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx index 25afea36..b368815b 100644 --- a/vite-app/src/components/PivotTab.tsx +++ b/vite-app/src/components/PivotTab.tsx @@ -3,7 +3,12 @@ import PivotTable from "./PivotTable"; import Select from "./Select"; import Button from "./Button"; import { state } from "../App"; -import { useEffect } from "react"; +import { type FilterConfig } from "../types/filters"; +import { + getFieldType, + getOperatorsForField, + createFilterFunction, +} from "../util/filter-utils"; interface FieldSelectorProps { title: string; @@ -126,21 +131,64 @@ const AggregatorSelector = ({
); +// Reusable filter input component +const FilterInput = ({ + filter, + index, + onUpdate, +}: { + filter: FilterConfig; + index: number; + onUpdate: (updates: Partial) => void; +}) => { + const fieldType = filter.type || getFieldType(filter.field); + + if (fieldType === "date") { + return ( +
+ onUpdate({ value: e.target.value })} + className="px-2 py-1 text-xs border border-gray-300 rounded focus:outline-none focus:border-gray-500 min-w-32" + /> + {filter.operator === "between" && ( + onUpdate({ value2: e.target.value })} + className="px-2 py-1 text-xs border border-gray-300 rounded focus:outline-none focus:border-gray-500 min-w-32" + placeholder="End date" + /> + )} +
+ ); + } + + return ( + onUpdate({ value: e.target.value })} + placeholder="Value" + className="px-2 py-1 text-xs border border-gray-300 rounded focus:outline-none focus:border-gray-500 min-w-32" + /> + ); +}; + const FilterSelector = ({ filters, onFiltersChange, availableKeys, }: { - filters: Array<{ field: string; operator: string; value: string }>; - onFiltersChange: ( - filters: Array<{ field: string; operator: string; value: string }> - ) => void; + filters: FilterConfig[]; + onFiltersChange: (filters: FilterConfig[]) => void; availableKeys: string[]; }) => { const addFilter = () => { onFiltersChange([ ...filters, - { field: "", operator: "contains", value: "" }, + { field: "", operator: "contains", value: "", type: "text" }, ]); }; @@ -148,90 +196,67 @@ const FilterSelector = ({ onFiltersChange(filters.filter((_, i) => i !== index)); }; - const updateFilter = ( - index: number, - field: string, - operator: string, - value: string - ) => { + const updateFilter = (index: number, updates: Partial) => { const newFilters = [...filters]; - newFilters[index] = { field, operator, value }; + newFilters[index] = { ...newFilters[index], ...updates }; onFiltersChange(newFilters); }; - const operators = [ - { value: "==", label: "equals" }, - { value: "!=", label: "not equals" }, - { value: ">", label: "greater than" }, - { value: "<", label: "less than" }, - { value: ">=", label: "greater than or equal" }, - { value: "<=", label: "less than or equal" }, - { value: "contains", label: "contains" }, - { value: "!contains", label: "not contains" }, - ]; - return (
Filters:
- {filters.map((filter, index) => ( -
- - - - updateFilter( - index, - filter.field, - filter.operator, - e.target.value - ) - } - placeholder="Value" - className="px-2 py-1 text-xs border border-gray-300 rounded focus:outline-none focus:border-gray-500 min-w-32" - /> - -
- ))} + {filters.map((filter, index) => { + const fieldType = filter.type || getFieldType(filter.field); + const operators = getOperatorsForField(filter.field, fieldType); + + return ( +
+ + + updateFilter(index, updates)} + /> + +
+ ); + })}
); -// Reusable filter input component -const FilterInput = ({ - filter, - index, - onUpdate, -}: { - filter: FilterConfig; - index: number; - onUpdate: (updates: Partial) => void; -}) => { - const fieldType = filter.type || getFieldType(filter.field); - - if (fieldType === "date") { - return ( -
- onUpdate({ value: e.target.value })} - className="px-2 py-1 text-xs border border-gray-300 rounded focus:outline-none focus:border-gray-500 min-w-32" - /> - {filter.operator === "between" && ( - onUpdate({ value2: e.target.value })} - className="px-2 py-1 text-xs border border-gray-300 rounded focus:outline-none focus:border-gray-500 min-w-32" - placeholder="End date" - /> - )} -
- ); - } - - return ( - onUpdate({ value: e.target.value })} - placeholder="Value" - className="px-2 py-1 text-xs border border-gray-300 rounded focus:outline-none focus:border-gray-500 min-w-32" - /> - ); -}; - const FilterSelector = ({ filters, onFiltersChange, diff --git a/vite-app/src/components/Select.tsx b/vite-app/src/components/Select.tsx index ea6a827e..3de52f26 100644 --- a/vite-app/src/components/Select.tsx +++ b/vite-app/src/components/Select.tsx @@ -1,4 +1,5 @@ import React from "react"; +import { commonStyles } from "../styles/common"; interface SelectProps extends Omit, "size"> { @@ -8,19 +9,11 @@ interface SelectProps const Select = React.forwardRef( ({ className = "", size = "sm", children, ...props }, ref) => { - const baseClasses = - "border text-xs font-medium focus:outline-none bg-white text-gray-700 border-gray-300 hover:border-gray-400 focus:border-gray-500"; - - const sizeClasses = { - sm: "px-2 py-0.5", - md: "px-3 py-1", - }; - return ( onFieldChange(index, e.target.value)} + onChange={(value) => onFieldChange(index, value)} + options={[ + { value: "", label: "Select a field..." }, + ...(availableKeys?.map((key) => ({ value: key, label: key })) || + []), + ]} size="sm" className="min-w-48" - > - - {availableKeys?.map((key) => ( - - ))} - + /> {fields.length > 0 && (
); @@ -168,37 +165,33 @@ const FilterSelector = ({ return (
- - + /> void; + placeholder?: string; + size?: "sm" | "md"; + className?: string; + disabled?: boolean; +} + +const SearchableSelect = React.forwardRef< + HTMLDivElement, + SearchableSelectProps +>( + ( + { + options, + value, + onChange, + placeholder = "Select...", + size = "sm", + className = "", + disabled = false, + }, + ref + ) => { + const [isOpen, setIsOpen] = useState(false); + const [searchTerm, setSearchTerm] = useState(""); + const [filteredOptions, setFilteredOptions] = useState(options); + const containerRef = useRef(null); + const inputRef = useRef(null); + + useEffect(() => { + const filtered = options.filter( + (option) => + option.label.toLowerCase().includes(searchTerm.toLowerCase()) || + option.value.toLowerCase().includes(searchTerm.toLowerCase()) + ); + setFilteredOptions(filtered); + }, [searchTerm, options]); + + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + containerRef.current && + !containerRef.current.contains(event.target as Node) + ) { + setIsOpen(false); + setSearchTerm(""); + } + }; + + document.addEventListener("mousedown", handleClickOutside); + return () => + document.removeEventListener("mousedown", handleClickOutside); + }, []); + + const handleSelect = (optionValue: string) => { + onChange(optionValue); + setIsOpen(false); + setSearchTerm(""); + }; + + const handleToggle = () => { + if (!disabled) { + setIsOpen(!isOpen); + if (!isOpen) { + setTimeout(() => inputRef.current?.focus(), 0); + } + } + }; + + const selectedOption = options.find((option) => option.value === value); + + return ( +
+
+ + {selectedOption ? selectedOption.label : placeholder} + + + + +
+ + {isOpen && ( +
+
+ setSearchTerm(e.target.value)} + placeholder="Search..." + className={`${commonStyles.input.base} ${commonStyles.input.size.sm} w-full`} + style={{ boxShadow: commonStyles.input.shadow }} + onKeyDown={(e) => { + if (e.key === "Escape") { + setIsOpen(false); + setSearchTerm(""); + } + }} + /> +
+
+ {filteredOptions.length > 0 ? ( + filteredOptions.map((option) => ( +
handleSelect(option.value)} + className="px-3 py-2 text-sm cursor-pointer hover:bg-gray-50 text-gray-700 border-b border-gray-100 last:border-b-0" + > + {option.label} +
+ )) + ) : ( +
+ No options found +
+ )} +
+
+ )} +
+ ); + } +); + +SearchableSelect.displayName = "SearchableSelect"; + +export default SearchableSelect; From 1b7ceb874f920c322ceeb1bbdd814061a142c556 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 11:34:40 -0700 Subject: [PATCH 38/43] searchableselect --- vite-app/src/components/SearchableSelect.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vite-app/src/components/SearchableSelect.tsx b/vite-app/src/components/SearchableSelect.tsx index d3360f77..cae63740 100644 --- a/vite-app/src/components/SearchableSelect.tsx +++ b/vite-app/src/components/SearchableSelect.tsx @@ -108,8 +108,8 @@ const SearchableSelect = React.forwardRef<
- {isOpen && ( -
+ {isOpen && ( +
Date: Tue, 12 Aug 2025 11:35:55 -0700 Subject: [PATCH 39/43] better styling --- vite-app/src/components/SearchableSelect.tsx | 33 ++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/vite-app/src/components/SearchableSelect.tsx b/vite-app/src/components/SearchableSelect.tsx index cae63740..53fa7819 100644 --- a/vite-app/src/components/SearchableSelect.tsx +++ b/vite-app/src/components/SearchableSelect.tsx @@ -30,6 +30,9 @@ const SearchableSelect = React.forwardRef< const [isOpen, setIsOpen] = useState(false); const [searchTerm, setSearchTerm] = useState(""); const [filteredOptions, setFilteredOptions] = useState(options); + const [dropdownPosition, setDropdownPosition] = useState<"left" | "right">( + "left" + ); const containerRef = useRef(null); const inputRef = useRef(null); @@ -64,8 +67,25 @@ const SearchableSelect = React.forwardRef< setSearchTerm(""); }; + const calculateDropdownPosition = () => { + if (!containerRef.current) return "left"; + + const rect = containerRef.current.getBoundingClientRect(); + const windowWidth = window.innerWidth; + const estimatedDropdownWidth = 300; // Approximate width for dropdown content + + // If dropdown would overflow right edge, position it to the left + if (rect.left + estimatedDropdownWidth > windowWidth) { + return "right"; + } + return "left"; + }; + const handleToggle = () => { if (!disabled) { + if (!isOpen) { + setDropdownPosition(calculateDropdownPosition()); + } setIsOpen(!isOpen); if (!isOpen) { setTimeout(() => inputRef.current?.focus(), 0); @@ -109,7 +129,16 @@ const SearchableSelect = React.forwardRef<
{isOpen && ( -
+
setSearchTerm(e.target.value)} placeholder="Search..." - className={`${commonStyles.input.base} ${commonStyles.input.size.sm} w-full`} + className={`${commonStyles.input.base} ${commonStyles.input.size.sm} w-full min-w-48`} style={{ boxShadow: commonStyles.input.shadow }} onKeyDown={(e) => { if (e.key === "Escape") { From 0052d2be040024b048681ec64c13ee00bd145310 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 11:36:09 -0700 Subject: [PATCH 40/43] Remove unused Select import from PivotTab component --- vite-app/src/components/PivotTab.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx index 80b6a2e7..21a6df6d 100644 --- a/vite-app/src/components/PivotTab.tsx +++ b/vite-app/src/components/PivotTab.tsx @@ -1,6 +1,5 @@ import { observer } from "mobx-react"; import PivotTable from "./PivotTable"; -import Select from "./Select"; import SearchableSelect from "./SearchableSelect"; import Button from "./Button"; import FilterInput from "./FilterInput"; From 9ee574b9c39e7a5ce4cbc3451906e6ba57161816 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 11:41:12 -0700 Subject: [PATCH 41/43] Add keyboard navigation and highlighting to SearchableSelect component --- vite-app/src/components/SearchableSelect.tsx | 80 +++++++++++++++++--- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/vite-app/src/components/SearchableSelect.tsx b/vite-app/src/components/SearchableSelect.tsx index 53fa7819..2fcca220 100644 --- a/vite-app/src/components/SearchableSelect.tsx +++ b/vite-app/src/components/SearchableSelect.tsx @@ -33,6 +33,7 @@ const SearchableSelect = React.forwardRef< const [dropdownPosition, setDropdownPosition] = useState<"left" | "right">( "left" ); + const [highlightedIndex, setHighlightedIndex] = useState(-1); const containerRef = useRef(null); const inputRef = useRef(null); @@ -43,6 +44,7 @@ const SearchableSelect = React.forwardRef< option.value.toLowerCase().includes(searchTerm.toLowerCase()) ); setFilteredOptions(filtered); + setHighlightedIndex(-1); // Reset highlighted index when options change }, [searchTerm, options]); useEffect(() => { @@ -65,6 +67,37 @@ const SearchableSelect = React.forwardRef< onChange(optionValue); setIsOpen(false); setSearchTerm(""); + setHighlightedIndex(-1); + }; + + const handleKeyDown = (e: React.KeyboardEvent) => { + if (!isOpen) return; + + switch (e.key) { + case "ArrowDown": + e.preventDefault(); + setHighlightedIndex((prev) => + prev < filteredOptions.length - 1 ? prev + 1 : 0 + ); + break; + case "ArrowUp": + e.preventDefault(); + setHighlightedIndex((prev) => + prev > 0 ? prev - 1 : filteredOptions.length - 1 + ); + break; + case "Enter": + e.preventDefault(); + if (highlightedIndex >= 0 && filteredOptions[highlightedIndex]) { + handleSelect(filteredOptions[highlightedIndex].value); + } + break; + case "Escape": + setIsOpen(false); + setSearchTerm(""); + setHighlightedIndex(-1); + break; + } }; const calculateDropdownPosition = () => { @@ -100,6 +133,16 @@ const SearchableSelect = React.forwardRef<
{ + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + handleToggle(); + } + }} + tabIndex={0} + role="combobox" + aria-expanded={isOpen} + aria-haspopup="listbox" className={` ${commonStyles.input.base} ${commonStyles.input.size[size]} @@ -130,45 +173,58 @@ const SearchableSelect = React.forwardRef< {isOpen && (
-
+
setSearchTerm(e.target.value)} + onKeyDown={handleKeyDown} placeholder="Search..." className={`${commonStyles.input.base} ${commonStyles.input.size.sm} w-full min-w-48`} style={{ boxShadow: commonStyles.input.shadow }} - onKeyDown={(e) => { - if (e.key === "Escape") { - setIsOpen(false); - setSearchTerm(""); - } - }} + role="searchbox" + aria-label="Search options" />
-
+
{filteredOptions.length > 0 ? ( - filteredOptions.map((option) => ( + filteredOptions.map((option, index) => (
handleSelect(option.value)} - className="px-3 py-2 text-sm cursor-pointer hover:bg-gray-50 text-gray-700 border-b border-gray-100 last:border-b-0" + onMouseEnter={() => setHighlightedIndex(index)} + className={`px-3 py-2 text-xs font-medium cursor-pointer hover:bg-gray-100 text-gray-700 border-b border-gray-100 last:border-b-0 ${ + highlightedIndex === index ? "bg-gray-100" : "" + }`} + role="option" + aria-selected={highlightedIndex === index} + tabIndex={-1} > {option.label}
)) ) : ( -
+
No options found
)} From c9c9855ac2331898769c186cffb015831cbac374 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 12:00:16 -0700 Subject: [PATCH 42/43] OR / AND filters --- vite-app/src/GlobalState.tsx | 5 +- vite-app/src/components/FilterInput.tsx | 3 +- vite-app/src/components/PivotTab.tsx | 216 +++++++++++++++++------- vite-app/src/components/PivotTable.tsx | 1 + vite-app/src/types/filters.ts | 8 +- vite-app/src/util/filter-utils.ts | 139 ++++++++------- 6 files changed, 247 insertions(+), 125 deletions(-) diff --git a/vite-app/src/GlobalState.tsx b/vite-app/src/GlobalState.tsx index 4380f2f3..14db70ae 100644 --- a/vite-app/src/GlobalState.tsx +++ b/vite-app/src/GlobalState.tsx @@ -59,7 +59,10 @@ export class GlobalState { // Reset pivot configuration to defaults resetPivotConfig() { - this.pivotConfig = { ...DEFAULT_PIVOT_CONFIG }; + this.pivotConfig = { + ...DEFAULT_PIVOT_CONFIG, + filters: [], // Ensure filters is an empty array of FilterGroups + }; this.savePivotConfig(); } diff --git a/vite-app/src/components/FilterInput.tsx b/vite-app/src/components/FilterInput.tsx index 9bcbf6b4..4361c7e7 100644 --- a/vite-app/src/components/FilterInput.tsx +++ b/vite-app/src/components/FilterInput.tsx @@ -4,11 +4,10 @@ import { commonStyles } from "../styles/common"; interface FilterInputProps { filter: FilterConfig; - index: number; onUpdate: (updates: Partial) => void; } -const FilterInput = ({ filter, index, onUpdate }: FilterInputProps) => { +const FilterInput = ({ filter, onUpdate }: FilterInputProps) => { const fieldType = filter.type || "text"; if (fieldType === "date") { diff --git a/vite-app/src/components/PivotTab.tsx b/vite-app/src/components/PivotTab.tsx index 21a6df6d..4b870653 100644 --- a/vite-app/src/components/PivotTab.tsx +++ b/vite-app/src/components/PivotTab.tsx @@ -4,7 +4,7 @@ import SearchableSelect from "./SearchableSelect"; import Button from "./Button"; import FilterInput from "./FilterInput"; import { state } from "../App"; -import { type FilterConfig } from "../types/filters"; +import { type FilterConfig, type FilterGroup } from "../types/filters"; import { getFieldType, getOperatorsForField, @@ -133,83 +133,166 @@ const FilterSelector = ({ onFiltersChange, availableKeys, }: { - filters: FilterConfig[]; - onFiltersChange: (filters: FilterConfig[]) => void; + filters: FilterGroup[]; + onFiltersChange: (filters: FilterGroup[]) => void; availableKeys: string[]; }) => { - const addFilter = () => { - onFiltersChange([ - ...filters, - { field: "", operator: "contains", value: "", type: "text" }, - ]); + const addFilterGroup = () => { + onFiltersChange([...filters, { logic: "AND", filters: [] }]); }; - const removeFilter = (index: number) => { + const removeFilterGroup = (index: number) => { onFiltersChange(filters.filter((_, i) => i !== index)); }; - const updateFilter = (index: number, updates: Partial) => { + const updateFilterGroupLogic = (index: number, logic: "AND" | "OR") => { const newFilters = [...filters]; - newFilters[index] = { ...newFilters[index], ...updates }; + newFilters[index] = { ...newFilters[index], logic }; + onFiltersChange(newFilters); + }; + + const addFilterToGroup = (groupIndex: number) => { + const newFilters = [...filters]; + newFilters[groupIndex].filters.push({ + field: "", + operator: "contains", + value: "", + type: "text", + }); + onFiltersChange(newFilters); + }; + + const removeFilterFromGroup = (groupIndex: number, filterIndex: number) => { + const newFilters = [...filters]; + newFilters[groupIndex].filters.splice(filterIndex, 1); + onFiltersChange(newFilters); + }; + + const updateFilterInGroup = ( + groupIndex: number, + filterIndex: number, + updates: Partial + ) => { + const newFilters = [...filters]; + newFilters[groupIndex].filters[filterIndex] = { + ...newFilters[groupIndex].filters[filterIndex], + ...updates, + }; onFiltersChange(newFilters); }; return (
Filters:
-
- {filters.map((filter, index) => { - const fieldType = filter.type || getFieldType(filter.field); - const operators = getOperatorsForField(filter.field, fieldType); - - return ( -
- { - const newField = value; - const newType = getFieldType(newField); - updateFilter(index, { field: newField, type: newType }); - }} - options={[ - { value: "", label: "Select a field..." }, - ...(availableKeys?.map((key) => ({ - value: key, - label: key, - })) || []), - ]} - size="sm" - className="min-w-48" - /> - updateFilter(index, { operator: value })} - options={operators.map((op) => ({ - value: op.value, - label: op.label, - }))} - size="sm" - className="min-w-32" - /> - updateFilter(index, updates)} - /> +
+ {filters.map((group, groupIndex) => ( +
+
+
+ + Group {groupIndex + 1}: + + + updateFilterGroupLogic(groupIndex, value as "AND" | "OR") + } + options={[ + { value: "AND", label: "AND (all filters must match)" }, + { value: "OR", label: "OR (any filter can match)" }, + ]} + size="sm" + className="min-w-48" + /> +
+
+ +
+ {group.filters.map((filter, filterIndex) => { + const fieldType = filter.type || getFieldType(filter.field); + const operators = getOperatorsForField(filter.field, fieldType); + + return ( +
+ { + const newField = value; + const newType = getFieldType(newField); + updateFilterInGroup(groupIndex, filterIndex, { + field: newField, + type: newType, + }); + }} + options={[ + { value: "", label: "Select a field..." }, + ...(availableKeys?.map((key) => ({ + value: key, + label: key, + })) || []), + ]} + size="sm" + className="min-w-48" + /> + + updateFilterInGroup(groupIndex, filterIndex, { + operator: value, + }) + } + options={operators.map((op) => ({ + value: op.value, + label: op.label, + }))} + size="sm" + className="min-w-32" + /> + + updateFilterInGroup(groupIndex, filterIndex, updates) + } + /> + +
+ ); + })} + +
- ); - })} +
+ ))} +
@@ -239,7 +322,7 @@ const PivotTab = observer(() => { state.updatePivotConfig({ selectedAggregator: value }); }; - const updateFilters = (filters: FilterConfig[]) => { + const updateFilters = (filters: FilterGroup[]) => { state.updatePivotConfig({ filters }); }; @@ -343,6 +426,16 @@ const PivotTab = observer(() => { availableKeys={availableKeys} /> + {/* + Filter Groups allow you to create complex filtering logic: + - Each group can use AND or OR logic internally + - Groups are combined with AND logic (all groups must match) + - Within a group: AND means all filters must match, OR means any filter can match + - Example: Group 1 (AND): field1 = "value1" AND field2 > 10 + - Example: Group 2 (OR): field3 = "value3" OR field4 = "value4" + - Result: (field1 = "value1" AND field2 > 10) AND (field3 = "value3" OR field4 = "value4") + */} + { valueField={ pivotConfig.selectedValueField as keyof (typeof state.flattenedDataset)[number] } - aggregator={pivotConfig.selectedAggregator as any} + aggregator={ + pivotConfig.selectedAggregator as + | "count" + | "sum" + | "avg" + | "min" + | "max" + } showRowTotals showColumnTotals filter={createFilterFunction(pivotConfig.filters)} diff --git a/vite-app/src/components/PivotTable.tsx b/vite-app/src/components/PivotTable.tsx index af6063de..52b2213d 100644 --- a/vite-app/src/components/PivotTable.tsx +++ b/vite-app/src/components/PivotTable.tsx @@ -106,6 +106,7 @@ export function PivotTable>({ filter, }); + debugger; return (
diff --git a/vite-app/src/types/filters.ts b/vite-app/src/types/filters.ts index 6b0dd936..dbd3a7b7 100644 --- a/vite-app/src/types/filters.ts +++ b/vite-app/src/types/filters.ts @@ -12,11 +12,17 @@ export interface FilterOperator { label: string; } +// Filter group interface for AND/OR logic +export interface FilterGroup { + logic: "AND" | "OR"; + filters: FilterConfig[]; +} + // Pivot configuration interface export interface PivotConfig { selectedRowFields: string[]; selectedColumnFields: string[]; selectedValueField: string; selectedAggregator: string; - filters: FilterConfig[]; + filters: FilterGroup[]; } diff --git a/vite-app/src/util/filter-utils.ts b/vite-app/src/util/filter-utils.ts index df93b08a..1f097244 100644 --- a/vite-app/src/util/filter-utils.ts +++ b/vite-app/src/util/filter-utils.ts @@ -1,4 +1,4 @@ -import type { FilterConfig } from "../types/filters"; +import type { FilterConfig, FilterGroup } from "../types/filters"; // Filter utilities export const isDateField = (field: string): boolean => { @@ -37,74 +37,87 @@ export const getOperatorsForField = (field: string, type?: string) => { ]; }; -// Create filter function from filter configuration -export const createFilterFunction = (filters: FilterConfig[]) => { - if (filters.length === 0) return undefined; +// Create filter function from filter group configuration +export const createFilterFunction = (filterGroups: FilterGroup[]) => { + if (filterGroups.length === 0) return undefined; return (record: any) => { - return filters.every((filter) => { - if (!filter.field || !filter.value) return true; // Skip incomplete filters + return filterGroups.every((group) => { + if (group.filters.length === 0) return true; - const fieldValue = record[filter.field]; - const filterValue = filter.value; - const filterValue2 = filter.value2; + if (group.logic === "OR") { + // For OR logic, at least one filter must pass + return group.filters.some((filter) => evaluateFilter(filter, record)); + } else { + // For AND logic, all filters must pass + return group.filters.every((filter) => evaluateFilter(filter, record)); + } + }); + }; +}; - // Handle date filtering - if (filter.type === "date" || filter.type === "date-range") { - const fieldDate = new Date(fieldValue); - const valueDate = new Date(filterValue); +// Helper function to evaluate a single filter +const evaluateFilter = (filter: FilterConfig, record: any): boolean => { + if (!filter.field || !filter.value) return true; // Skip incomplete filters - if (isNaN(fieldDate.getTime()) || isNaN(valueDate.getTime())) { - return true; // Skip invalid dates - } + const fieldValue = record[filter.field]; + const filterValue = filter.value; + const filterValue2 = filter.value2; + + // Handle date filtering + if (filter.type === "date" || filter.type === "date-range") { + const fieldDate = new Date(fieldValue); + const valueDate = new Date(filterValue); + + if (isNaN(fieldDate.getTime()) || isNaN(valueDate.getTime())) { + return true; // Skip invalid dates + } - switch (filter.operator) { - case "==": - return fieldDate.toDateString() === valueDate.toDateString(); - case "!=": - return fieldDate.toDateString() !== valueDate.toDateString(); - case ">=": - return fieldDate >= valueDate; - case "<=": - return fieldDate <= valueDate; - case "between": - if (filterValue2) { - const valueDate2 = new Date(filterValue2); - if (!isNaN(valueDate2.getTime())) { - return fieldDate >= valueDate && fieldDate <= valueDate2; - } - } - return true; // Skip incomplete between filter - default: - return true; + switch (filter.operator) { + case "==": + return fieldDate.toDateString() === valueDate.toDateString(); + case "!=": + return fieldDate.toDateString() !== valueDate.toDateString(); + case ">=": + return fieldDate >= valueDate; + case "<=": + return fieldDate <= valueDate; + case "between": + if (filterValue2) { + const valueDate2 = new Date(filterValue2); + if (!isNaN(valueDate2.getTime())) { + return fieldDate >= valueDate && fieldDate <= valueDate2; + } } - } + return true; // Skip incomplete between filter + default: + return true; + } + } - // Handle text/numeric filtering - switch (filter.operator) { - case "==": - return String(fieldValue) === filterValue; - case "!=": - return String(fieldValue) !== filterValue; - case ">": - return Number(fieldValue) > Number(filterValue); - case "<": - return Number(fieldValue) < Number(filterValue); - case ">=": - return Number(fieldValue) >= Number(filterValue); - case "<=": - return Number(fieldValue) <= Number(filterValue); - case "contains": - return String(fieldValue) - .toLowerCase() - .includes(filterValue.toLowerCase()); - case "!contains": - return !String(fieldValue) - .toLowerCase() - .includes(filterValue.toLowerCase()); - default: - return true; - } - }); - }; + // Handle text/numeric filtering + switch (filter.operator) { + case "==": + return String(fieldValue) === filterValue; + case "!=": + return String(fieldValue) !== filterValue; + case ">": + return Number(fieldValue) > Number(filterValue); + case "<": + return Number(fieldValue) < Number(filterValue); + case ">=": + return Number(fieldValue) >= Number(filterValue); + case "<=": + return Number(fieldValue) <= Number(filterValue); + case "contains": + return String(fieldValue) + .toLowerCase() + .includes(filterValue.toLowerCase()); + case "!contains": + return !String(fieldValue) + .toLowerCase() + .includes(filterValue.toLowerCase()); + default: + return true; + } }; From eb916d39174d07479c84ce1fd7879df917506a15 Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Tue, 12 Aug 2025 12:09:54 -0700 Subject: [PATCH 43/43] Enhance computePivot function to filter out records with undefined values in both row and column fields, and update tests to verify correct handling of such cases and row total calculations. --- vite-app/src/util/pivot.test.ts | 52 +++++++++++++++++++++++++++++++++ vite-app/src/util/pivot.ts | 36 +++++++++++++++++------ 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/vite-app/src/util/pivot.test.ts b/vite-app/src/util/pivot.test.ts index a9966657..558208b7 100644 --- a/vite-app/src/util/pivot.test.ts +++ b/vite-app/src/util/pivot.test.ts @@ -325,6 +325,58 @@ describe('computePivot', () => { expect(res.grandTotal).toBe(1) }) + it('skips records with undefined column field values', () => { + type LooseRow = { + region: string + rep?: string + product?: string + amount?: number | string + } + + const mixed: LooseRow[] = [ + { region: 'West', rep: 'A', product: 'Widget', amount: 120 }, + // Missing product should be excluded entirely (no 'undefined' column) + { region: 'West', rep: 'B', amount: 90 }, + { region: 'East', rep: 'B', product: 'Gadget', amount: 10 }, + ] + + const res = computePivot({ + data: mixed, + rowFields: ['region'], + columnFields: ['product'], + valueField: 'amount', + aggregator: 'sum', + }) + + // Columns should not contain 'undefined' + expect(res.colKeyTuples.map((t) => String(t))).toEqual(['Gadget', 'Widget']) + + const rWest = 'West' + const rEast = 'East' + const cWidget = 'Widget' + const cGadget = 'Gadget' + + // Only valid records contribute + expect(res.cells[rWest][cWidget].value).toBe(120) + expect(res.cells[rEast][cGadget].value).toBe(10) + expect(res.cells[rWest][cGadget]).toBeUndefined() + }) + + it('row totals use the provided aggregation over row records', () => { + const res = computePivot({ + data: rows, + rowFields: ['region'], + columnFields: ['product'], + valueField: 'amount', + aggregator: 'avg', + }) + + // East row has values [10, 200] => avg 105 + expect(res.rowTotals['East']).toBe(105) + // West row has values [90, 120] => avg 105 + expect(res.rowTotals['West']).toBe(105) + }) + it("test_flaky_passes_sometimes", () => { // read logs.json from data/logs.json const logsUrl = new URL('../../data/logs.jsonl', import.meta.url) diff --git a/vite-app/src/util/pivot.ts b/vite-app/src/util/pivot.ts index f48ea7bc..45473243 100644 --- a/vite-app/src/util/pivot.ts +++ b/vite-app/src/util/pivot.ts @@ -248,18 +248,19 @@ export function computePivot>({ // Apply filter first if provided const filteredData = filter ? data.filter(filter) : data; - // Filter out records that do not have defined values for all rowFields. - // This avoids creating a row key of "undefined" and ensures such records - // are not returned as part of the cells/row totals. - const dataWithDefinedRows = filteredData.filter((rec) => - rowFields.every((f) => rec[f] !== undefined) + // Filter out records that do not have defined values for all rowFields and columnFields. + // This avoids creating a row or column key of "undefined" and ensures such records + // are not returned as part of the cells/totals. + const dataWithDefinedKeys = filteredData.filter((rec) => + rowFields.every((f) => rec[f] !== undefined) && + columnFields.every((f) => rec[f] !== undefined) ); const rowKeyTuples: unknown[][] = []; const rowKeySet = new Set(); const colKeyTuples: unknown[][] = []; const colKeySet = new Set(); - for (const rec of dataWithDefinedRows) { + for (const rec of dataWithDefinedKeys) { const rTuple = getTuple(rec, rowFields); const rKey = toKey(rTuple); if (!rowKeySet.has(rKey)) { @@ -295,7 +296,7 @@ export function computePivot>({ // Partition records per cell const cellRecords: Record> = {}; - for (const rec of dataWithDefinedRows) { + for (const rec of dataWithDefinedKeys) { const rKey = toKey(getTuple(rec, rowFields)); const cKey = toKey(getTuple(rec, columnFields)); if (!cellRecords[rKey]) cellRecords[rKey] = {}; @@ -315,10 +316,27 @@ export function computePivot>({ } const value = aggregate(values, records, aggregator); cells[rKey][cKey] = { value, records }; - rowTotals[rKey] += value; } } + // Calculate row totals using the same aggregation method applied over all records in the row + for (const rKey of Object.keys(cells)) { + const rowRecords: T[] = []; + const rowValues: number[] = []; + const cols = cellRecords[rKey] ?? {}; + for (const cKey of Object.keys(cols)) { + const records = cols[cKey]; + rowRecords.push(...records); + if (valueField != null) { + for (const rec of records) { + const v = getNumber(rec[valueField]); + if (v != null) rowValues.push(v); + } + } + } + rowTotals[rKey] = aggregate(rowValues, rowRecords, aggregator); + } + // Calculate column totals using the same aggregation method for (const cKey of Object.keys(colTotals)) { const columnRecords: T[] = []; @@ -344,7 +362,7 @@ export function computePivot>({ // non-additive aggregations like "avg"). let grandTotal: number; { - const allRecords = dataWithDefinedRows; + const allRecords = dataWithDefinedKeys; const allValues: number[] = []; if (valueField != null) { for (const rec of allRecords) {