diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py index 511fde692..a6e8768ef 100644 --- a/crawl4ai/deep_crawling/bff_strategy.py +++ b/crawl4ai/deep_crawling/bff_strategy.py @@ -36,7 +36,7 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy): def __init__( self, max_depth: int, - filter_chain: FilterChain = FilterChain(), + filter_chain: Optional[FilterChain] = None, url_scorer: Optional[URLScorer] = None, include_external: bool = False, score_threshold: float = -infinity, @@ -49,7 +49,7 @@ def __init__( should_cancel: Optional[Callable[[], Union[bool, Awaitable[bool]]]] = None, ): self.max_depth = max_depth - self.filter_chain = filter_chain + self.filter_chain = filter_chain if filter_chain is not None else FilterChain() self.url_scorer = url_scorer self.include_external = include_external self.score_threshold = score_threshold diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py index dfb759272..d7d3fa532 100644 --- a/crawl4ai/deep_crawling/bfs_strategy.py +++ b/crawl4ai/deep_crawling/bfs_strategy.py @@ -25,7 +25,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy): def __init__( self, max_depth: int, - filter_chain: FilterChain = FilterChain(), + filter_chain: Optional[FilterChain] = None, url_scorer: Optional[URLScorer] = None, include_external: bool = False, score_threshold: float = -infinity, @@ -38,7 +38,7 @@ def __init__( should_cancel: Optional[Callable[[], Union[bool, Awaitable[bool]]]] = None, ): self.max_depth = max_depth - self.filter_chain = filter_chain + self.filter_chain = filter_chain if filter_chain is not None else FilterChain() self.url_scorer = url_scorer self.include_external = include_external self.score_threshold = score_threshold diff --git a/tests/deep_crawling/test_default_filter_chain_isolation.py b/tests/deep_crawling/test_default_filter_chain_isolation.py new file mode 100644 index 000000000..512a907f7 --- /dev/null +++ b/tests/deep_crawling/test_default_filter_chain_isolation.py @@ -0,0 +1,42 @@ +"""Regression tests for the mutable-default-argument bug in deep-crawl strategy +constructors: filter_chain defaulted to a single FilterChain() instance shared +across every strategy created without an explicit filter_chain=, leaking +FilterChain.stats counters and add_filter() mutations across unrelated +strategy instances. +""" + +from crawl4ai.deep_crawling.bff_strategy import BestFirstCrawlingStrategy +from crawl4ai.deep_crawling.bfs_strategy import BFSDeepCrawlStrategy +from crawl4ai.deep_crawling.dfs_strategy import DFSDeepCrawlStrategy + + +def test_bfs_strategy_instances_get_independent_default_filter_chains(): + a = BFSDeepCrawlStrategy(max_depth=2) + b = BFSDeepCrawlStrategy(max_depth=3) + assert a.filter_chain is not b.filter_chain + + +def test_dfs_strategy_instances_get_independent_default_filter_chains(): + a = DFSDeepCrawlStrategy(max_depth=2) + b = DFSDeepCrawlStrategy(max_depth=3) + assert a.filter_chain is not b.filter_chain + + +def test_best_first_strategy_instances_get_independent_default_filter_chains(): + a = BestFirstCrawlingStrategy(max_depth=2) + b = BestFirstCrawlingStrategy(max_depth=3) + assert a.filter_chain is not b.filter_chain + + +def test_adding_a_filter_to_one_default_chain_does_not_leak_to_another_instance(): + a = BFSDeepCrawlStrategy(max_depth=2) + b = BFSDeepCrawlStrategy(max_depth=3) + + class _DummyFilter: + def apply(self, url): + return True + + a.filter_chain.add_filter(_DummyFilter()) + + assert len(a.filter_chain.filters) == 1 + assert len(b.filter_chain.filters) == 0