From 5dc482a21602e9c84963cc9fb2572f3fe036f8aa Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Mon, 6 Jul 2026 19:34:42 +0530 Subject: [PATCH 1/2] feat(core): add Firecrawl search integration --- packages/core/src/plugin/search/firecrawl.ts | 126 ++++++++++++++++++ packages/core/src/plugin/search/index.ts | 3 +- .../core/test/plugin/search-firecrawl.test.ts | 83 ++++++++++++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 packages/core/src/plugin/search/firecrawl.ts create mode 100644 packages/core/test/plugin/search-firecrawl.test.ts diff --git a/packages/core/src/plugin/search/firecrawl.ts b/packages/core/src/plugin/search/firecrawl.ts new file mode 100644 index 000000000000..594ff882a44a --- /dev/null +++ b/packages/core/src/plugin/search/firecrawl.ts @@ -0,0 +1,126 @@ +export * as SearchFirecrawl from "./firecrawl" + +import { define } from "@opencode-ai/plugin/v2/effect" +import { Duration, Effect, Schema, Scope } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import { collectBoundedResponseBody } from "../../tool/http-body" +import { SearchMcp } from "./mcp" + +export const endpoint = "https://api.firecrawl.dev/v2/search" + +const Request = Schema.Struct({ + query: Schema.String, + limit: Schema.optional(Schema.Number), +}) + +const WebResult = Schema.Struct({ + url: Schema.String, + title: Schema.optional(Schema.String), + description: Schema.optional(Schema.String), + markdown: Schema.optional(Schema.NullOr(Schema.String)), + position: Schema.optional(Schema.Number), + category: Schema.optional(Schema.String), +}) +const NewsResult = Schema.Struct({ + url: Schema.String, + title: Schema.optional(Schema.String), + snippet: Schema.optional(Schema.String), + date: Schema.optional(Schema.String), + markdown: Schema.optional(Schema.NullOr(Schema.String)), + position: Schema.optional(Schema.Number), +}) +const ImageResult = Schema.Struct({ + url: Schema.String, + title: Schema.optional(Schema.String), + imageUrl: Schema.String, + imageWidth: Schema.optional(Schema.Number), + imageHeight: Schema.optional(Schema.Number), + position: Schema.optional(Schema.Number), +}) +const Response = Schema.Struct({ + success: Schema.Literal(true), + data: Schema.Struct({ + web: Schema.optional(Schema.Array(WebResult)), + news: Schema.optional(Schema.Array(NewsResult)), + images: Schema.optional(Schema.Array(ImageResult)), + }), + warning: Schema.optional(Schema.NullOr(Schema.String)), + id: Schema.optional(Schema.String), + creditsUsed: Schema.optional(Schema.Number), +}) +const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json)) +const decodeResponse = Schema.decodeUnknownEffect(Response) + +const format = (response: typeof Response.Type) => + [ + ...(response.data.web ?? []).map((result) => + [`## ${result.title ?? result.url}`, `URL: ${result.url}`, result.description, result.markdown || undefined] + .filter((line) => line !== undefined) + .join("\n\n"), + ), + ...(response.data.news ?? []).map((result) => + [ + `## ${result.title ?? result.url}`, + `URL: ${result.url}`, + result.date ? `Date: ${result.date}` : undefined, + result.snippet, + result.markdown || undefined, + ] + .filter((line) => line !== undefined) + .join("\n\n"), + ), + ...(response.data.images ?? []).map((result) => + [`## ${result.title ?? result.url}`, `Source: ${result.url}`, `Image: ${result.imageUrl}`].join("\n\n"), + ), + ].join("\n\n") + +const search = ( + http: HttpClient.HttpClient, + input: { readonly query: string; readonly numResults?: number; readonly contextMaxCharacters?: number }, + apiKey?: string, +) => + Effect.gen(function* () { + const request = yield* HttpClientRequest.post(endpoint).pipe( + HttpClientRequest.acceptJson, + HttpClientRequest.setHeaders({ + ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), + }), + HttpClientRequest.schemaBodyJson(Request)({ query: input.query, limit: input.numResults }), + ) + const response = yield* HttpClient.filterStatusOk(http).execute(request) + const body = yield* collectBoundedResponseBody( + response, + SearchMcp.MAX_RESPONSE_BYTES, + () => new Error(`Firecrawl response exceeded ${SearchMcp.MAX_RESPONSE_BYTES} bytes`), + ) + const metadata = yield* decodeJson(body.toString("utf8")) + const result = yield* decodeResponse(metadata) + const text = format(result) + return { + text: input.contextMaxCharacters ? text.slice(0, input.contextMaxCharacters) : text, + metadata, + } + }).pipe( + Effect.timeoutOrElse({ + duration: Duration.seconds(25), + orElse: () => Effect.fail(new Error("Firecrawl search request timed out")), + }), + ) + +export const Plugin = define({ + id: "opencode.search.firecrawl", + effect: Effect.fn("SearchFirecrawl.Plugin")(function* (ctx) { + const http = yield* HttpClient.HttpClient + yield* ctx.integration.transform((draft) => { + draft.update("firecrawl", (integration) => (integration.name = "Firecrawl")) + draft.method.update({ integrationID: "firecrawl", method: { type: "key", label: "API key (optional)" } }) + draft.method.update({ integrationID: "firecrawl", method: { type: "env", names: ["FIRECRAWL_API_KEY"] } }) + draft.capability.search.update({ + integrationID: "firecrawl", + capability: { type: "search", connection: "optional" }, + execute: (input, context) => + search(http, input, context.credential?.type === "key" ? context.credential.key : undefined), + }) + }) + }), +}) diff --git a/packages/core/src/plugin/search/index.ts b/packages/core/src/plugin/search/index.ts index b816effe9b79..1c38f7869957 100644 --- a/packages/core/src/plugin/search/index.ts +++ b/packages/core/src/plugin/search/index.ts @@ -1,4 +1,5 @@ import { SearchExa } from "./exa" +import { SearchFirecrawl } from "./firecrawl" import { SearchParallel } from "./parallel" -export const SearchPlugins = [SearchExa.Plugin, SearchParallel.Plugin] as const +export const SearchPlugins = [SearchExa.Plugin, SearchFirecrawl.Plugin, SearchParallel.Plugin] as const diff --git a/packages/core/test/plugin/search-firecrawl.test.ts b/packages/core/test/plugin/search-firecrawl.test.ts new file mode 100644 index 000000000000..facfd025b80e --- /dev/null +++ b/packages/core/test/plugin/search-firecrawl.test.ts @@ -0,0 +1,83 @@ +import { beforeEach, describe, expect } from "bun:test" +import { Effect } from "effect" +import { Credential } from "@opencode-ai/core/credential" +import { Integration } from "@opencode-ai/core/integration" +import { SearchFirecrawl } from "@opencode-ai/core/plugin/search/firecrawl" +import { host, integrationHost } from "./host" +import { requests, resetSearchFixture, searchIntegrationTest } from "./search-fixture" + +beforeEach(() => { + resetSearchFixture( + JSON.stringify({ + success: true, + data: { + web: [ + { + url: "https://effect.website/", + title: "Effect", + description: "Build production TypeScript applications.", + position: 1, + }, + ], + }, + id: "search_1", + creditsUsed: 2, + }), + ) +}) + +const it = searchIntegrationTest + +describe("Firecrawl search integration", () => { + it.effect("registers Firecrawl and uses its keyless default response", () => + Effect.gen(function* () { + const integrations = yield* Integration.Service + yield* SearchFirecrawl.Plugin.effect(host({ integration: integrationHost(integrations) })) + const provider = yield* integrations.capability.search.get(Integration.ID.make("firecrawl")) + if (!provider) return yield* Effect.die("Expected Firecrawl search provider") + + const output = yield* provider.execute({ query: "effect", numResults: 3 }, {}) + expect(output).toEqual({ + text: "## Effect\n\nURL: https://effect.website/\n\nBuild production TypeScript applications.", + metadata: { + success: true, + data: { + web: [ + { + url: "https://effect.website/", + title: "Effect", + description: "Build production TypeScript applications.", + position: 1, + }, + ], + }, + id: "search_1", + creditsUsed: 2, + }, + }) + expect(requests).toEqual([ + { + url: SearchFirecrawl.endpoint, + headers: expect.not.objectContaining({ authorization: expect.anything() }), + body: { query: "effect", limit: 3 }, + }, + ]) + }), + ) + + it.effect("sends a configured Firecrawl key as a bearer credential", () => + Effect.gen(function* () { + const integrations = yield* Integration.Service + yield* SearchFirecrawl.Plugin.effect(host({ integration: integrationHost(integrations) })) + const provider = yield* integrations.capability.search.get(Integration.ID.make("firecrawl")) + if (!provider) return yield* Effect.die("Expected Firecrawl search provider") + + const output = yield* provider.execute( + { query: "effect" }, + { credential: Credential.Key.make({ type: "key", key: "firecrawl-secret" }) }, + ) + expect(requests[0]?.headers).toMatchObject({ authorization: "Bearer firecrawl-secret" }) + expect(JSON.stringify(output)).not.toContain("firecrawl-secret") + }), + ) +}) From f38e5589100bb02c44ef28e0dd0dc6bc57244d30 Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Mon, 6 Jul 2026 19:48:40 +0530 Subject: [PATCH 2/2] refactor(core): simplify Firecrawl adapter --- packages/core/src/plugin/search/firecrawl.ts | 40 +++++++-------- .../core/test/plugin/search-firecrawl.test.ts | 50 +++++++------------ 2 files changed, 38 insertions(+), 52 deletions(-) diff --git a/packages/core/src/plugin/search/firecrawl.ts b/packages/core/src/plugin/search/firecrawl.ts index 594ff882a44a..ed023b1df5b5 100644 --- a/packages/core/src/plugin/search/firecrawl.ts +++ b/packages/core/src/plugin/search/firecrawl.ts @@ -1,6 +1,7 @@ export * as SearchFirecrawl from "./firecrawl" import { define } from "@opencode-ai/plugin/v2/effect" +import type { Search } from "@opencode-ai/schema/search" import { Duration, Effect, Schema, Scope } from "effect" import { HttpClient, HttpClientRequest } from "effect/unstable/http" import { collectBoundedResponseBody } from "../../tool/http-body" @@ -8,36 +9,37 @@ import { SearchMcp } from "./mcp" export const endpoint = "https://api.firecrawl.dev/v2/search" -const Request = Schema.Struct({ +const FirecrawlRequest = Schema.Struct({ query: Schema.String, limit: Schema.optional(Schema.Number), }) -const WebResult = Schema.Struct({ +const ResultBase = { url: Schema.String, title: Schema.optional(Schema.String), - description: Schema.optional(Schema.String), - markdown: Schema.optional(Schema.NullOr(Schema.String)), position: Schema.optional(Schema.Number), +} +const PageResult = { + ...ResultBase, + markdown: Schema.optional(Schema.NullOr(Schema.String)), +} +const WebResult = Schema.Struct({ + ...PageResult, + description: Schema.optional(Schema.String), category: Schema.optional(Schema.String), }) const NewsResult = Schema.Struct({ - url: Schema.String, - title: Schema.optional(Schema.String), + ...PageResult, snippet: Schema.optional(Schema.String), date: Schema.optional(Schema.String), - markdown: Schema.optional(Schema.NullOr(Schema.String)), - position: Schema.optional(Schema.Number), }) const ImageResult = Schema.Struct({ - url: Schema.String, - title: Schema.optional(Schema.String), + ...ResultBase, imageUrl: Schema.String, imageWidth: Schema.optional(Schema.Number), imageHeight: Schema.optional(Schema.Number), - position: Schema.optional(Schema.Number), }) -const Response = Schema.Struct({ +const FirecrawlResponse = Schema.Struct({ success: Schema.Literal(true), data: Schema.Struct({ web: Schema.optional(Schema.Array(WebResult)), @@ -49,9 +51,9 @@ const Response = Schema.Struct({ creditsUsed: Schema.optional(Schema.Number), }) const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json)) -const decodeResponse = Schema.decodeUnknownEffect(Response) +const decodeResponse = Schema.decodeUnknownEffect(FirecrawlResponse) -const format = (response: typeof Response.Type) => +const formatResults = (response: typeof FirecrawlResponse.Type) => [ ...(response.data.web ?? []).map((result) => [`## ${result.title ?? result.url}`, `URL: ${result.url}`, result.description, result.markdown || undefined] @@ -76,16 +78,14 @@ const format = (response: typeof Response.Type) => const search = ( http: HttpClient.HttpClient, - input: { readonly query: string; readonly numResults?: number; readonly contextMaxCharacters?: number }, + input: Pick, apiKey?: string, ) => Effect.gen(function* () { const request = yield* HttpClientRequest.post(endpoint).pipe( HttpClientRequest.acceptJson, - HttpClientRequest.setHeaders({ - ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), - }), - HttpClientRequest.schemaBodyJson(Request)({ query: input.query, limit: input.numResults }), + HttpClientRequest.setHeaders(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), + HttpClientRequest.schemaBodyJson(FirecrawlRequest)({ query: input.query, limit: input.numResults }), ) const response = yield* HttpClient.filterStatusOk(http).execute(request) const body = yield* collectBoundedResponseBody( @@ -95,7 +95,7 @@ const search = ( ) const metadata = yield* decodeJson(body.toString("utf8")) const result = yield* decodeResponse(metadata) - const text = format(result) + const text = formatResults(result) return { text: input.contextMaxCharacters ? text.slice(0, input.contextMaxCharacters) : text, metadata, diff --git a/packages/core/test/plugin/search-firecrawl.test.ts b/packages/core/test/plugin/search-firecrawl.test.ts index facfd025b80e..4324332a6156 100644 --- a/packages/core/test/plugin/search-firecrawl.test.ts +++ b/packages/core/test/plugin/search-firecrawl.test.ts @@ -6,24 +6,24 @@ import { SearchFirecrawl } from "@opencode-ai/core/plugin/search/firecrawl" import { host, integrationHost } from "./host" import { requests, resetSearchFixture, searchIntegrationTest } from "./search-fixture" -beforeEach(() => { - resetSearchFixture( - JSON.stringify({ - success: true, - data: { - web: [ - { - url: "https://effect.website/", - title: "Effect", - description: "Build production TypeScript applications.", - position: 1, - }, - ], +const metadata = { + success: true, + data: { + web: [ + { + url: "https://effect.website/", + title: "Effect", + description: "Build production TypeScript applications.", + position: 1, }, - id: "search_1", - creditsUsed: 2, - }), - ) + ], + }, + id: "search_1", + creditsUsed: 2, +} + +beforeEach(() => { + resetSearchFixture(JSON.stringify(metadata)) }) const it = searchIntegrationTest @@ -39,21 +39,7 @@ describe("Firecrawl search integration", () => { const output = yield* provider.execute({ query: "effect", numResults: 3 }, {}) expect(output).toEqual({ text: "## Effect\n\nURL: https://effect.website/\n\nBuild production TypeScript applications.", - metadata: { - success: true, - data: { - web: [ - { - url: "https://effect.website/", - title: "Effect", - description: "Build production TypeScript applications.", - position: 1, - }, - ], - }, - id: "search_1", - creditsUsed: 2, - }, + metadata, }) expect(requests).toEqual([ {