Feat/ai targets and toolsets

README.md

@@ -84,6 +84,29 @@ this:
 
 </details>
 
+## Toolsets
+
+Tools are organized into toolsets. You can selectively enable specific toolsets by passing a
+comma-separated list via the `toolsets` query parameter:
+
+```
+    "Decodo MCP Server": {
+      "url": "https://mcp.decodo.com/mcp?toolsets=web,ai",
+      "headers": {
+        "Authorization": "Basic <your_auth_token>"
+      }
+    }
+```
+
+When no toolsets are specified, all tools are registered.
+
+| Toolset        | Tools                                                          |
+| -------------- | -------------------------------------------------------------- |
+| `web`          | `scrape_as_markdown`, `screenshot`, `google_search_parsed`     |
+| `ecommerce`    | `amazon_search_parsed`                                         |
+| `social_media` | `reddit_post`, `reddit_subreddit`                              |
+| `ai`           | `chatgpt`, `perplexity`                       |
+
 ## Tools
 
 The server exposes the following tools:
@@ -96,6 +119,8 @@ The server exposes the following tools:
 | `amazon_search_parsed` | Scrapes Amazon Search for a given query, and returns parsed results.                       | Scrape Amazon Search for toothbrushes.                                                  |
 | `reddit_post`          | Scrapes a specific Reddit post for a given query, and returns parsed results.              | Scrape the following Reddit post: https://www.reddit.com/r/horseracing/comments/1nsrn3/ |
 | `reddit_subreddit`     | Scrapes a specific Reddit subreddit for a given query, and returns parsed results.         | Scrape the top 5 posts on r/Python this week.                                           |
+| `chatgpt`              | Search and interact with ChatGPT for AI-powered responses and conversations.               | Ask ChatGPT to explain quantum computing in simple terms.                               |
+| `perplexity`           | Search and interact with Perplexity for AI-powered responses and conversations.            | Ask Perplexity what the latest trends in web development are.                           |
 
 ## Parameters
 
@@ -107,7 +132,8 @@ The following parameters are inferred from user prompts:
 | `geo`          | Sets the country from which the request will originate.                                              |
 | `locale`       | Sets the locale of the request.                                                                      |
 | `tokenLimit`   | Truncates the response content up to this limit. Useful if the context window is small.              |
-| `fullResponse` | Skips automatic truncation and returns full content. If context window is small, may throw warnings. |
+| `prompt`       | Prompt to send to AI tools (`chatgpt`, `perplexity`).                                                |
+| `search`       | Activates ChatGPT's web search functionality (`chatgpt` only).                                       |
 
 ## Examples
 

src/__tests__/chatgpt-tool.test.ts

@@ -0,0 +1,76 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { ScraperApiClient } from '../clients/scraper-api-client';
+import { ChatGPTTool } from '../tools/chatgpt-tool';
+import { SCRAPER_API_TARGETS, TOOLSET } from '../constants';
+
+jest.mock('@modelcontextprotocol/sdk/server/mcp.js');
+jest.mock('../clients/scraper-api-client');
+
+describe('ChatGPTTool', () => {
+  let server: jest.Mocked<McpServer>;
+  let sapiClient: jest.Mocked<ScraperApiClient>;
+  const auth = 'dGVzdDp0ZXN0';
+
+  beforeEach(() => {
+    server = new McpServer({ name: 'test', version: '1.0' }) as jest.Mocked<McpServer>;
+    server.registerTool = jest.fn();
+    sapiClient = new ScraperApiClient() as jest.Mocked<ScraperApiClient>;
+  });
+
+  it('has ai toolset', () => {
+    expect(ChatGPTTool.toolset).toBe(TOOLSET.AI);
+  });
+
+  it('registers with correct tool name', () => {
+    ChatGPTTool.register({ server, sapiClient, getAuthToken: () => auth });
+
+    expect(server.registerTool).toHaveBeenCalledWith(
+      'chatgpt',
+      expect.any(Object),
+      expect.any(Function)
+    );
+  });
+
+  it('calls scrape with CHATGPT target and parse: true', async () => {
+    const mockData = { response: 'Hello! How can I help you?' };
+    sapiClient.scrape = jest.fn().mockResolvedValue({ data: mockData });
+
+    ChatGPTTool.register({ server, sapiClient, getAuthToken: () => auth });
+
+    const handler = (server.registerTool as jest.Mock).mock.calls[0][2];
+    const result = await handler({ prompt: 'What is TypeScript?' });
+
+    expect(sapiClient.scrape).toHaveBeenCalledWith({
+      auth,
+      scrapingParams: expect.objectContaining({
+        prompt: 'What is TypeScript?',
+        target: SCRAPER_API_TARGETS.CHATGPT,
+        parse: true,
+      }),
+    });
+
+    expect(result.content).toHaveLength(1);
+    expect(result.content[0].type).toBe('text');
+    expect(JSON.parse(result.content[0].text)).toEqual(mockData);
+  });
+
+  it('passes search parameter when provided', async () => {
+    const mockData = { response: 'Search results...' };
+    sapiClient.scrape = jest.fn().mockResolvedValue({ data: mockData });
+
+    ChatGPTTool.register({ server, sapiClient, getAuthToken: () => auth });
+
+    const handler = (server.registerTool as jest.Mock).mock.calls[0][2];
+    await handler({ prompt: 'Latest news', search: true });
+
+    expect(sapiClient.scrape).toHaveBeenCalledWith({
+      auth,
+      scrapingParams: expect.objectContaining({
+        prompt: 'Latest news',
+        search: true,
+        target: SCRAPER_API_TARGETS.CHATGPT,
+        parse: true,
+      }),
+    });
+  });
+});

src/__tests__/mocks/amazon-search-parsed.json

@@ -0,0 +1,73 @@
+{
+  "results": [
+    {
+      "content": {
+        "results": {
+          "url": "https://www.amazon.com/s?k=laptop",
+          "page": 1,
+          "query": "laptop",
+          "results": {
+            "organic": [
+              {
+                "pos": 1,
+                "url": "/dp/B0CX23V2ZK",
+                "asin": "B0CX23V2ZK",
+                "price": 799.99,
+                "title": "Laptop 15.6 inch, 16GB RAM",
+                "rating": 4.5,
+                "currency": "USD",
+                "is_prime": true,
+                "reviews_count": 1234
+              },
+              {
+                "pos": 2,
+                "url": "/dp/B0D1234567",
+                "asin": "B0D1234567",
+                "price": 599.99,
+                "title": "Budget Laptop 14 inch, 8GB RAM",
+                "rating": 4.2,
+                "currency": "USD",
+                "is_prime": false,
+                "reviews_count": 567
+              }
+            ]
+          },
+          "suggested": [
+            {
+              "title": "laptop stand",
+              "url": "/s?k=laptop+stand"
+            },
+            {
+              "title": "laptop bag",
+              "url": "/s?k=laptop+bag"
+            }
+          ],
+          "amazons_choices": [
+            {
+              "pos": 1,
+              "url": "/dp/B0AMAZONCHOICE",
+              "asin": "B0AMAZONCHOICE",
+              "price": 699.99,
+              "title": "Amazon's Choice Laptop"
+            }
+          ],
+          "refinements": {
+            "brands": ["HP", "Dell", "Lenovo", "Apple", "ASUS"],
+            "price_ranges": ["$200 - $400", "$400 - $600", "$600 - $800", "$800+"]
+          },
+          "parse_status_code": 12000
+        },
+        "errors": [],
+        "status_code": 12000,
+        "task_id": "7341006309614950402"
+      },
+      "headers": {},
+      "status_code": 200,
+      "url": "https://www.amazon.com/s?k=laptop",
+      "query": "laptop",
+      "task_id": "7341006309614950402",
+      "created_at": "2025-06-18 07:38:13",
+      "updated_at": "2025-06-18 07:38:15"
+    }
+  ]
+}

src/__tests__/perplexity-tool.test.ts

@@ -0,0 +1,76 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { ScraperApiClient } from '../clients/scraper-api-client';
+import { PerplexityTool } from '../tools/perplexity-tool';
+import { SCRAPER_API_TARGETS, TOOLSET } from '../constants';
+
+jest.mock('@modelcontextprotocol/sdk/server/mcp.js');
+jest.mock('../clients/scraper-api-client');
+
+describe('PerplexityTool', () => {
+  let server: jest.Mocked<McpServer>;
+  let sapiClient: jest.Mocked<ScraperApiClient>;
+  const auth = 'dGVzdDp0ZXN0';
+
+  beforeEach(() => {
+    server = new McpServer({ name: 'test', version: '1.0' }) as jest.Mocked<McpServer>;
+    server.registerTool = jest.fn();
+    sapiClient = new ScraperApiClient() as jest.Mocked<ScraperApiClient>;
+  });
+
+  it('has ai toolset', () => {
+    expect(PerplexityTool.toolset).toBe(TOOLSET.AI);
+  });
+
+  it('registers with correct tool name', () => {
+    PerplexityTool.register({ server, sapiClient, getAuthToken: () => auth });
+
+    expect(server.registerTool).toHaveBeenCalledWith(
+      'perplexity',
+      expect.any(Object),
+      expect.any(Function)
+    );
+  });
+
+  it('calls scrape with PERPLEXITY target and parse: true', async () => {
+    const mockData = { answer: 'Perplexity response with sources', sources: ['url1', 'url2'] };
+    sapiClient.scrape = jest.fn().mockResolvedValue({ data: mockData });
+
+    PerplexityTool.register({ server, sapiClient, getAuthToken: () => auth });
+
+    const handler = (server.registerTool as jest.Mock).mock.calls[0][2];
+    const result = await handler({ prompt: 'What is MCP?' });
+
+    expect(sapiClient.scrape).toHaveBeenCalledWith({
+      auth,
+      scrapingParams: expect.objectContaining({
+        prompt: 'What is MCP?',
+        target: SCRAPER_API_TARGETS.PERPLEXITY,
+        parse: true,
+      }),
+    });
+
+    expect(result.content).toHaveLength(1);
+    expect(result.content[0].type).toBe('text');
+    expect(JSON.parse(result.content[0].text)).toEqual(mockData);
+  });
+
+  it('passes geo parameter when provided', async () => {
+    const mockData = { answer: 'Local response' };
+    sapiClient.scrape = jest.fn().mockResolvedValue({ data: mockData });
+
+    PerplexityTool.register({ server, sapiClient, getAuthToken: () => auth });
+
+    const handler = (server.registerTool as jest.Mock).mock.calls[0][2];
+    await handler({ prompt: 'Weather today', geo: 'United States' });
+
+    expect(sapiClient.scrape).toHaveBeenCalledWith({
+      auth,
+      scrapingParams: expect.objectContaining({
+        prompt: 'Weather today',
+        geo: 'United States',
+        target: SCRAPER_API_TARGETS.PERPLEXITY,
+        parse: true,
+      }),
+    });
+  });
+});

src/__tests__/reddit-post-tool.test.ts

@@ -1,75 +1,55 @@
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { ScraperApiClient } from '../clients/scraper-api-client';
 import { RedditPostTool } from '../tools/reddit-post-tool';
-import { ScrapingMCPParams } from '../types';
+import { SCRAPER_API_TARGETS, TOOLSET } from '../constants';
 
 jest.mock('@modelcontextprotocol/sdk/server/mcp.js');
 jest.mock('../clients/scraper-api-client');
 
-const MockedMcpServer = McpServer as jest.MockedClass<typeof McpServer>;
-const MockedScraperApiClient = ScraperApiClient as jest.MockedClass<typeof ScraperApiClient>;
-
 describe('RedditPostTool', () => {
   let server: jest.Mocked<McpServer>;
   let sapiClient: jest.Mocked<ScraperApiClient>;
-  let registeredHandler: (params: ScrapingMCPParams) => Promise<unknown>;
   const auth = 'dGVzdDp0ZXN0';
 
   beforeEach(() => {
-    server = new MockedMcpServer({ name: 'test', version: '0.0.0' }) as jest.Mocked<McpServer>;
-    sapiClient = new MockedScraperApiClient() as jest.Mocked<ScraperApiClient>;
+    server = new McpServer({ name: 'test', version: '1.0' }) as jest.Mocked<McpServer>;
+    server.registerTool = jest.fn();
+    sapiClient = new ScraperApiClient() as jest.Mocked<ScraperApiClient>;
+  });
 
-    server.registerTool = jest.fn((_name, _config, handler) => {
-      registeredHandler = handler as typeof registeredHandler;
-      return server;
-    });
+  it('has social_media toolset', () => {
+    expect(RedditPostTool.toolset).toBe(TOOLSET.SOCIAL_MEDIA);
+  });
 
+  it('registers with correct tool name', () => {
     RedditPostTool.register({ server, sapiClient, getAuthToken: () => auth });
-  });
 
-  it('registers a tool named "reddit_post"', () => {
     expect(server.registerTool).toHaveBeenCalledWith(
       'reddit_post',
-      expect.objectContaining({ description: expect.stringContaining('Reddit') }),
+      expect.any(Object),
       expect.any(Function)
     );
   });
 
-  it('returns a text content block with pretty-printed JSON', async () => {
-    const mockData = { title: 'Test post', comments: [{ body: 'Great post!' }] };
+  it('calls scrape with REDDIT_POST target', async () => {
+    const mockData = { title: 'Test post', comments: [] };
     sapiClient.scrape = jest.fn().mockResolvedValue({ data: mockData });
 
-    const result = (await registeredHandler({
-      url: 'https://www.reddit.com/r/test/comments/abc123/',
-    })) as { content: { type: string; text: string }[] };
-
-    expect(result.content).toHaveLength(1);
-    expect(result.content[0].type).toBe('text');
-    expect(result.content[0].text).toBe(JSON.stringify(mockData, null, 2));
-  });
-
-  it('passes reddit_post target to the scraper', async () => {
-    sapiClient.scrape = jest.fn().mockResolvedValue({ data: {} });
+    RedditPostTool.register({ server, sapiClient, getAuthToken: () => auth });
 
-    const postUrl = 'https://www.reddit.com/r/horseracing/comments/1nsrn3/';
-    await registeredHandler({ url: postUrl });
+    const handler = (server.registerTool as jest.Mock).mock.calls[0][2];
+    const result = await handler({ url: 'https://reddit.com/r/test/comments/abc' });
 
     expect(sapiClient.scrape).toHaveBeenCalledWith({
       auth,
       scrapingParams: expect.objectContaining({
-        url: postUrl,
-        target: 'reddit_post',
+        url: 'https://reddit.com/r/test/comments/abc',
+        target: SCRAPER_API_TARGETS.REDDIT_POST,
       }),
     });
-  });
 
-  it('propagates scraper errors', async () => {
-    sapiClient.scrape = jest
-      .fn()
-      .mockRejectedValue(new Error('Scraper API request failed (401): Authentication failed.'));
-
-    await expect(
-      registeredHandler({ url: 'https://www.reddit.com/r/test/comments/abc123/' })
-    ).rejects.toThrow('Scraper API request failed (401): Authentication failed.');
+    expect(result.content).toHaveLength(1);
+    expect(result.content[0].type).toBe('text');
+    expect(JSON.parse(result.content[0].text)).toEqual(mockData);
   });
 });