diff --git a/packages/core/src/plugin/search/firecrawl.ts b/packages/core/src/plugin/search/firecrawl.ts new file mode 100644 index 000000000000..ed023b1df5b5 --- /dev/null +++ b/packages/core/src/plugin/search/firecrawl.ts @@ -0,0 +1,126 @@ +export * as SearchFirecrawl from "./firecrawl" + +import { define } from "@opencode-ai/plugin/v2/effect" +import type { Search } from "@opencode-ai/schema/search" +import { Duration, Effect, Schema, Scope } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import { collectBoundedResponseBody } from "../../tool/http-body" +import { SearchMcp } from "./mcp" + +export const endpoint = "https://api.firecrawl.dev/v2/search" + +const FirecrawlRequest = Schema.Struct({ + query: Schema.String, + limit: Schema.optional(Schema.Number), +}) + +const ResultBase = { + url: Schema.String, + title: Schema.optional(Schema.String), + position: Schema.optional(Schema.Number), +} +const PageResult = { + ...ResultBase, + markdown: Schema.optional(Schema.NullOr(Schema.String)), +} +const WebResult = Schema.Struct({ + ...PageResult, + description: Schema.optional(Schema.String), + category: Schema.optional(Schema.String), +}) +const NewsResult = Schema.Struct({ + ...PageResult, + snippet: Schema.optional(Schema.String), + date: Schema.optional(Schema.String), +}) +const ImageResult = Schema.Struct({ + ...ResultBase, + imageUrl: Schema.String, + imageWidth: Schema.optional(Schema.Number), + imageHeight: Schema.optional(Schema.Number), +}) +const FirecrawlResponse = Schema.Struct({ + success: Schema.Literal(true), + data: Schema.Struct({ + web: Schema.optional(Schema.Array(WebResult)), + news: Schema.optional(Schema.Array(NewsResult)), + images: Schema.optional(Schema.Array(ImageResult)), + }), + warning: Schema.optional(Schema.NullOr(Schema.String)), + id: Schema.optional(Schema.String), + creditsUsed: Schema.optional(Schema.Number), +}) +const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json)) +const decodeResponse = Schema.decodeUnknownEffect(FirecrawlResponse) + +const formatResults = (response: typeof FirecrawlResponse.Type) => + [ + ...(response.data.web ?? []).map((result) => + [`## ${result.title ?? result.url}`, `URL: ${result.url}`, result.description, result.markdown || undefined] + .filter((line) => line !== undefined) + .join("\n\n"), + ), + ...(response.data.news ?? []).map((result) => + [ + `## ${result.title ?? result.url}`, + `URL: ${result.url}`, + result.date ? `Date: ${result.date}` : undefined, + result.snippet, + result.markdown || undefined, + ] + .filter((line) => line !== undefined) + .join("\n\n"), + ), + ...(response.data.images ?? []).map((result) => + [`## ${result.title ?? result.url}`, `Source: ${result.url}`, `Image: ${result.imageUrl}`].join("\n\n"), + ), + ].join("\n\n") + +const search = ( + http: HttpClient.HttpClient, + input: Pick, + apiKey?: string, +) => + Effect.gen(function* () { + const request = yield* HttpClientRequest.post(endpoint).pipe( + HttpClientRequest.acceptJson, + HttpClientRequest.setHeaders(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), + HttpClientRequest.schemaBodyJson(FirecrawlRequest)({ query: input.query, limit: input.numResults }), + ) + const response = yield* HttpClient.filterStatusOk(http).execute(request) + const body = yield* collectBoundedResponseBody( + response, + SearchMcp.MAX_RESPONSE_BYTES, + () => new Error(`Firecrawl response exceeded ${SearchMcp.MAX_RESPONSE_BYTES} bytes`), + ) + const metadata = yield* decodeJson(body.toString("utf8")) + const result = yield* decodeResponse(metadata) + const text = formatResults(result) + return { + text: input.contextMaxCharacters ? text.slice(0, input.contextMaxCharacters) : text, + metadata, + } + }).pipe( + Effect.timeoutOrElse({ + duration: Duration.seconds(25), + orElse: () => Effect.fail(new Error("Firecrawl search request timed out")), + }), + ) + +export const Plugin = define({ + id: "opencode.search.firecrawl", + effect: Effect.fn("SearchFirecrawl.Plugin")(function* (ctx) { + const http = yield* HttpClient.HttpClient + yield* ctx.integration.transform((draft) => { + draft.update("firecrawl", (integration) => (integration.name = "Firecrawl")) + draft.method.update({ integrationID: "firecrawl", method: { type: "key", label: "API key (optional)" } }) + draft.method.update({ integrationID: "firecrawl", method: { type: "env", names: ["FIRECRAWL_API_KEY"] } }) + draft.capability.search.update({ + integrationID: "firecrawl", + capability: { type: "search", connection: "optional" }, + execute: (input, context) => + search(http, input, context.credential?.type === "key" ? context.credential.key : undefined), + }) + }) + }), +}) diff --git a/packages/core/src/plugin/search/index.ts b/packages/core/src/plugin/search/index.ts index b816effe9b79..1c38f7869957 100644 --- a/packages/core/src/plugin/search/index.ts +++ b/packages/core/src/plugin/search/index.ts @@ -1,4 +1,5 @@ import { SearchExa } from "./exa" +import { SearchFirecrawl } from "./firecrawl" import { SearchParallel } from "./parallel" -export const SearchPlugins = [SearchExa.Plugin, SearchParallel.Plugin] as const +export const SearchPlugins = [SearchExa.Plugin, SearchFirecrawl.Plugin, SearchParallel.Plugin] as const diff --git a/packages/core/test/plugin/search-firecrawl.test.ts b/packages/core/test/plugin/search-firecrawl.test.ts new file mode 100644 index 000000000000..4324332a6156 --- /dev/null +++ b/packages/core/test/plugin/search-firecrawl.test.ts @@ -0,0 +1,69 @@ +import { beforeEach, describe, expect } from "bun:test" +import { Effect } from "effect" +import { Credential } from "@opencode-ai/core/credential" +import { Integration } from "@opencode-ai/core/integration" +import { SearchFirecrawl } from "@opencode-ai/core/plugin/search/firecrawl" +import { host, integrationHost } from "./host" +import { requests, resetSearchFixture, searchIntegrationTest } from "./search-fixture" + +const metadata = { + success: true, + data: { + web: [ + { + url: "https://effect.website/", + title: "Effect", + description: "Build production TypeScript applications.", + position: 1, + }, + ], + }, + id: "search_1", + creditsUsed: 2, +} + +beforeEach(() => { + resetSearchFixture(JSON.stringify(metadata)) +}) + +const it = searchIntegrationTest + +describe("Firecrawl search integration", () => { + it.effect("registers Firecrawl and uses its keyless default response", () => + Effect.gen(function* () { + const integrations = yield* Integration.Service + yield* SearchFirecrawl.Plugin.effect(host({ integration: integrationHost(integrations) })) + const provider = yield* integrations.capability.search.get(Integration.ID.make("firecrawl")) + if (!provider) return yield* Effect.die("Expected Firecrawl search provider") + + const output = yield* provider.execute({ query: "effect", numResults: 3 }, {}) + expect(output).toEqual({ + text: "## Effect\n\nURL: https://effect.website/\n\nBuild production TypeScript applications.", + metadata, + }) + expect(requests).toEqual([ + { + url: SearchFirecrawl.endpoint, + headers: expect.not.objectContaining({ authorization: expect.anything() }), + body: { query: "effect", limit: 3 }, + }, + ]) + }), + ) + + it.effect("sends a configured Firecrawl key as a bearer credential", () => + Effect.gen(function* () { + const integrations = yield* Integration.Service + yield* SearchFirecrawl.Plugin.effect(host({ integration: integrationHost(integrations) })) + const provider = yield* integrations.capability.search.get(Integration.ID.make("firecrawl")) + if (!provider) return yield* Effect.die("Expected Firecrawl search provider") + + const output = yield* provider.execute( + { query: "effect" }, + { credential: Credential.Key.make({ type: "key", key: "firecrawl-secret" }) }, + ) + expect(requests[0]?.headers).toMatchObject({ authorization: "Bearer firecrawl-secret" }) + expect(JSON.stringify(output)).not.toContain("firecrawl-secret") + }), + ) +})