diff --git a/apps/application/workflow/nodes/data_source_web_node/__init__.py b/apps/application/workflow/nodes/data_source_web_node/__init__.py new file mode 100644 index 00000000000..2887a81f640 --- /dev/null +++ b/apps/application/workflow/nodes/data_source_web_node/__init__.py @@ -0,0 +1,10 @@ +# coding=utf-8 +""" +@project: MaxKB +@Author: 虎虎虎 +@file: __init__.py +@date: 2026/9/16 +@desc: Web 站点数据源节点(知识库工作流起始节点之一) +""" + +from .data_source_web_node import DataSourceWebNode diff --git a/apps/application/workflow/nodes/data_source_web_node/data_source_web_node.py b/apps/application/workflow/nodes/data_source_web_node/data_source_web_node.py new file mode 100644 index 00000000000..f6c7ab0a444 --- /dev/null +++ b/apps/application/workflow/nodes/data_source_web_node/data_source_web_node.py @@ -0,0 +1,107 @@ +# coding=utf-8 +""" +@project: MaxKB +@Author: 虎虎虎 +@file: data_source_web_node.py +@date: 2026/9/16 +@desc: Web 站点数据源节点:知识库工作流的起始节点之一,按根地址抓取站点内容写入 document_list 供下游读取 +""" + +import traceback + +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from application.workflow.common import WorkflowType +from application.workflow.i_node import CancelledException, INode +from common.utils.fork import ChildLink, Fork, ForkManage +from common.utils.logger import maxkb_logger + + +class DataSourceWebNodeParamsSerializer(serializers.Serializer): + source_url = serializers.CharField(required=True, label=_("Web source url")) + selector = serializers.CharField( + required=False, allow_blank=True, allow_null=True, label=_("Web knowledge selector") + ) + + +class DataSourceWebNode(INode): + serializer_class = DataSourceWebNodeParamsSerializer + supported_workflow_type_list = [WorkflowType.KNOWLEDGE] + type = "data-source-web-node" + + @staticmethod + def get_form_list(node): + return [ + { + "field": "source_url", + "input_type": "TextInput", + "attrs": {"placeholder": _("Please enter the Web root address")}, + "label": _("Web source url"), + "required": True, + }, + { + "field": "selector", + "input_type": "TextInput", + "attrs": {"placeholder": _("The default is body, you can enter .classname/#idname/tagname")}, + "label": _("Web knowledge selector"), + "required": False, + }, + ] + + def _get_collect_handler(self, document_list): + def handler(child_link: ChildLink, response: Fork.Response): + if response.status == 200: + try: + document_name = ( + child_link.tag.text + if child_link.tag is not None and len(child_link.tag.text.strip()) > 0 + else child_link.url + ) + document_list.append({"name": document_name.strip(), "content": response.content}) + except Exception as e: + maxkb_logger.error(f"{str(e)}:{traceback.format_exc()}") + # 已取消则抛出 CancelledException,由引擎结束流程 + self._check_cancelled() + + return handler + + def execute(self): + workflow_params = self.get_workflow_parameters() + data_source = workflow_params.get("data_source") or {} + + serializer = self.serializer_class(data=data_source) + serializer.is_valid(raise_exception=True) + source_url = serializer.validated_data.get("source_url") + selector = serializer.validated_data.get("selector") or "body" + + document_list = [] + collect_handler = self._get_collect_handler(document_list) + + try: + ForkManage(source_url, selector.split(" ") if selector else []).fork(3, set(), collect_handler) + except CancelledException: + raise + except Exception as e: + maxkb_logger.error( + _("data source web node:{node_id} error{error}{traceback}").format( + node_id=self.get_node_id(), error=str(e), traceback=traceback.format_exc() + ) + ) + + self.write_context("document_list", document_list) + self.write_context("source_url", source_url) + self.write_context("selector", selector) + + def get_details(self, index: int = 0, position: dict = None, old_details: dict = None, **kwargs): + details = super().get_details(index, position, old_details, **kwargs) + details.update( + { + "document_list": self.get_context("document_list"), + "source_url": self.get_context("source_url"), + "selector": self.get_context("selector"), + "knowledge_base": self.get_workflow_parameters().get("knowledge_base"), + "enableException": self.node.properties.get("enableException"), + } + ) + return details diff --git a/ui/src/views/details-demo/index.vue b/ui/src/views/details-demo/index.vue index f69a79c5fec..8f31b795ed6 100644 --- a/ui/src/views/details-demo/index.vue +++ b/ui/src/views/details-demo/index.vue @@ -167,12 +167,22 @@ const detail = ref([ is_submit: true, }), + // ===== Web 数据源 ===== + base(WorkflowNodeType.DataSourceWebNode, 16, 'Web 数据源', { + selector: '.content', + source_url: 'https://example.com/docs', + document_list: [ + { name: '前言', content: '本文介绍 MaxKB 的安装步骤与系统要求。' }, + { name: '快速开始', content: '下载镜像后,通过 docker compose 一键启动。' }, + ], + }), + // ===== 循环 ===== { type: WorkflowNodeType.LoopNode, name: '循环', status: 200, - index: 16, + index: 17, run_time: 4.567, loop_type: 'ARRAY', loop_node_data: { diff --git a/ui/src/workflow-canvas/nodes/data-source-web-node/index.ts b/ui/src/workflow-canvas/nodes/data-source-web-node/index.ts index 45c957004c8..a7e475e9a17 100644 --- a/ui/src/workflow-canvas/nodes/data-source-web-node/index.ts +++ b/ui/src/workflow-canvas/nodes/data-source-web-node/index.ts @@ -1,4 +1,5 @@ import DataSourceWebNode from './index.vue' +import DataSourceWebNodeDetail from './details/index.vue' import { WorkflowNodeModel, WorkflowNodeView } from '@/workflow-canvas/core/workflow-node' import { WorkflowNodeType } from '@/workflow-canvas/types' @@ -8,4 +9,4 @@ class DataSourceWebNodeView extends WorkflowNodeView { } } -export default { type: WorkflowNodeType.DataSourceWebNode, model: WorkflowNodeModel, view: DataSourceWebNodeView } +export default { type: WorkflowNodeType.DataSourceWebNode, model: WorkflowNodeModel, view: DataSourceWebNodeView, details: DataSourceWebNodeDetail } diff --git a/ui/src/workflow-canvas/nodes/search-knowledge-node/details/index.vue b/ui/src/workflow-canvas/nodes/search-knowledge-node/details/index.vue index a3b6b365f3e..fda1cb5d527 100644 --- a/ui/src/workflow-canvas/nodes/search-knowledge-node/details/index.vue +++ b/ui/src/workflow-canvas/nodes/search-knowledge-node/details/index.vue @@ -17,6 +17,10 @@ interface SearchParagraph { document_name?: string knowledge_name?: string knowledge_type?: string + meta?: { + source_file_id?: string + source_url?: string + } similarity?: number title?: string }