diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py index dfb759272..70529a92c 100644 --- a/crawl4ai/deep_crawling/bfs_strategy.py +++ b/crawl4ai/deep_crawling/bfs_strategy.py @@ -70,8 +70,6 @@ async def can_process_url(self, url: str, depth: int) -> bool: raise ValueError("Missing scheme or netloc") if parsed.scheme not in ("http", "https"): raise ValueError("Invalid scheme") - if "." not in parsed.netloc: - raise ValueError("Invalid domain") except Exception as e: self.logger.warning(f"Invalid URL: {url}, error: {e}") return False diff --git a/tests/deep_crawling/test_deep_crawl_resume.py b/tests/deep_crawling/test_deep_crawl_resume.py index c6d84180d..c30afb35b 100644 --- a/tests/deep_crawling/test_deep_crawl_resume.py +++ b/tests/deep_crawling/test_deep_crawl_resume.py @@ -628,6 +628,14 @@ async def test_filter_chain_still_works(self): assert await strategy.can_process_url("https://example.com/blog/post1", 1) == True assert await strategy.can_process_url("https://other.com/blog/post1", 1) == False + @pytest.mark.asyncio + async def test_single_label_hostname_is_valid(self): + """Single-label hostnames can reach the configured filter chain.""" + filter_chain = FilterChain([DomainFilter(allowed_domains=["intranet"])]) + strategy = BFSDeepCrawlStrategy(max_depth=1, filter_chain=filter_chain) + + assert await strategy.can_process_url("https://intranet/docs", 1) is True + @pytest.mark.asyncio async def test_url_scorer_still_works(self): """URL scoring integration unchanged."""