fix(scrape): improve error handling and response propagation for scraping failures

docs/API.md

@@ -252,6 +252,21 @@ Scrape a single URL with flexible extraction options.
 }
 ```
 
+**Error behavior:**
+
+Real scraping failures (HTTP 4xx/5xx responses, connection errors,
+robots.txt blocks, browser timeouts) are returned as
+`500 Internal Server Error` — never as HTTP 200 with
+`success: true`. The `detail` message carries the original failure
+reason (for example `HTTP 404`).
+
+| Scenario                                    | Status | `detail`                          |
+| ------------------------------------------- | ------ | --------------------------------- |
+| Scraping failure (404, connection, robots)  | 500    | Original reason (e.g. `HTTP 404`) |
+| SSRF-protected target (metadata/private IP) | 500    | SSRF protection reason            |
+| Malformed request body                      | 422    | FastAPI validation error          |
+| Missing or invalid API key                  | 401    | Authentication error              |
+
 #### POST /api/scrape/batch
 
 Scrape multiple URLs in batch.
@@ -296,6 +311,13 @@ Scrape multiple URLs in batch.
 }
 ```
 
+**Batch error behavior:**
+
+- Mixed batches (some URLs succeed, some fail) return HTTP 200 with
+  `success: true`; failed URLs are listed individually in `failed_urls`.
+- If **every** URL fails, the endpoint returns HTTP 500 and `detail`
+  summarizes the per-URL errors.
+
 ### Attack Simulation
 
 #### POST /api/attack

src/ciberwebscan/api/routes/scrape.py

@@ -95,6 +95,8 @@ def scrape_url(
             status_code=status.HTTP_400_BAD_REQUEST,
             detail=f"Invalid request: {e}",
         ) from e
+    except HTTPException:
+        raise
     except Exception as e:
         logger.error(f"Error scraping URL {request.url}: {e}", exc_info=True)
         raise HTTPException(
@@ -193,6 +195,8 @@ def scrape_batch(
             status_code=status.HTTP_400_BAD_REQUEST,
             detail=f"Invalid request: {e}",
         ) from e
+    except HTTPException:
+        raise
     except Exception as e:
         logger.error(f"Error in batch scraping: {e}", exc_info=True)
         raise HTTPException(

src/ciberwebscan/cli/commands/scrape.py

@@ -17,6 +17,7 @@
     print_error,
     print_header,
     print_info,
+    print_warning,
 )
 from ciberwebscan.cli.validators import (
     ValidationError,
@@ -386,6 +387,8 @@ def scrape_batch(
                     print_info(f"  [{i}] {r.url} - {r.status_code}")
                 if len(result.data) > 5:
                     print_info(f"  ... and {len(result.data) - 5} more")
+            for warning in result.warnings:
+                print_warning(warning)
             exit_code = format_service_result(result, json_output=False)
 
         sys.exit(exit_code)

src/ciberwebscan/services/scrape_service.py

@@ -345,15 +345,19 @@ def scrape_multiple(
                 )
                 single_result = self.scrape(opts)
 
-                if single_result.success and single_result.data:
-                    if isinstance(single_result.data, ScrapeResult):
-                        results.append(single_result.data)
-                else:
-                    errors.append(f"{url}: {single_result.error}")
-                    result.warnings.append(f"Failed to scrape {url}")
+                if not single_result.success:
+                    reason = single_result.error or "Scraping failed"
+                    errors.append(f"{url}: {reason}")
+                    result.warnings.append(f"Failed to scrape {url}: {reason}")
+                elif isinstance(single_result.data, ScrapeResult):
+                    results.append(single_result.data)
 
             result.data = results
-            result.success = len(results) > 0
+            if results:
+                result.success = True
+            elif errors:
+                result.error = "; ".join(errors)
+                result.error_code = "SCRAPE_ERROR"
 
             # Export all results if requested
             if options.export and results:
@@ -395,6 +399,15 @@ def _scrape_static(self, url: str, options: ScrapeOptions) -> ScrapeResult:
                 cookies=options.cookies or None,
             ).scrape(url, config)
 
+            if not core_result.success:
+                details: dict[str, Any] = {"url": url}
+                if core_result.status_code is not None:
+                    details["status_code"] = core_result.status_code
+                raise ExecutionError(
+                    core_result.error or "Static scraping failed",
+                    details=details,
+                )
+
             # Map core ScrapeResult to export ScrapeResult
             from ciberwebscan.export.models import (
                 FormInfo,
@@ -442,6 +455,8 @@ def _scrape_static(self, url: str, options: ScrapeOptions) -> ScrapeResult:
                 elapsed_ms=(core_result.elapsed_time or 0.0) * 1000,
             )
 
+        except ExecutionError:
+            raise
         except Exception as e:
             raise ExecutionError(f"Static scraping failed: {e}") from e
 
@@ -465,6 +480,16 @@ def _scrape_dynamic(self, url: str, options: ScrapeOptions) -> ScrapeResult:
             # Run the async scrape method safely from sync context
             core_result = run_async(self.dynamic_scraper.scrape(url, config))
 
+            if not getattr(core_result, "success", True):
+                details: dict[str, Any] = {"url": url}
+                status_code = getattr(core_result, "status_code", None)
+                if status_code is not None:
+                    details["status_code"] = status_code
+                raise ExecutionError(
+                    getattr(core_result, "error", None) or "Dynamic scraping failed",
+                    details=details,
+                )
+
             # dynamic scraper returns DynamicScrapeResult-like dataclass
             from ciberwebscan.export.models import (
                 FormInfo,
@@ -513,6 +538,8 @@ def _scrape_dynamic(self, url: str, options: ScrapeOptions) -> ScrapeResult:
                 elapsed_ms=(getattr(core_result, "elapsed_time", 0.0) or 0.0) * 1000,
             )
 
+        except ExecutionError:
+            raise
         except Exception as e:
             raise ExecutionError(f"Dynamic scraping failed: {e}") from e
 

tests/unit/api/routes/test_scrape_routes.py

@@ -4,7 +4,7 @@
 
 from __future__ import annotations
 
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, Mock, patch
 
 import pytest
 from fastapi.testclient import TestClient
@@ -160,3 +160,87 @@ def test_post_scrape_batch_service_failure_returns_500(self, client: TestClient)
     def test_post_scrape_batch_empty_urls_returns_422(self, client: TestClient):
         response = client.post("/api/scrape/batch", json={"urls": []})
         assert response.status_code == 422
+
+
+def _mock_http_response(
+    status: int, text: str = "<html><body>ok</body></html>"
+) -> Mock:
+    """Build a minimal HTTP response stub for the real StaticScraper."""
+    response = Mock()
+    response.status_code = status
+    response.headers = {"content-type": "text/html"}
+    response.text = text
+    response.cookies.items.return_value = {}
+    return response
+
+
+class TestScrapeFailurePropagation:
+    """Real scrape failures must surface as HTTP 500, never HTTP 200.
+
+    These tests run the real ``ScrapeService`` (only the HTTP client is
+    mocked) to regression-test BUG-005 end to end through the API layer.
+    """
+
+    def test_real_scrape_failure_returns_500_with_clean_detail(
+        self, client: TestClient
+    ):
+        with patch("ciberwebscan.core.client.HTTPClient") as mock_http_cls:
+            mock_http_cls.return_value.get.return_value = _mock_http_response(
+                404, "<html>Not Found</html>"
+            )
+            response = client.post(
+                "/api/scrape",
+                json={
+                    "url": "https://example.com/missing",
+                    "check_robots": False,
+                },
+            )
+
+        assert response.status_code == 500
+        detail = response.json()["detail"]
+        assert "404" in detail
+        assert "Scraping failed: 500" not in detail
+
+    def test_real_batch_mixed_returns_partial_success(self, client: TestClient):
+        ok_response = _mock_http_response(200)
+        missing_response = _mock_http_response(404, "<html>Not Found</html>")
+        with patch("ciberwebscan.core.client.HTTPClient") as mock_http_cls:
+            mock_http_cls.return_value.get.side_effect = lambda url, **kwargs: (
+                ok_response if url.endswith("/ok") else missing_response
+            )
+            response = client.post(
+                "/api/scrape/batch",
+                json={
+                    "urls": [
+                        "https://example.com/ok",
+                        "https://example.com/missing",
+                    ]
+                },
+            )
+
+        assert response.status_code == 200
+        body = response.json()
+        assert body["success"] is True
+        assert body["data"]["total_success"] == 1
+        assert body["data"]["total_failed"] == 1
+        assert body["data"]["failed_urls"][0]["url"] == "https://example.com/missing"
+
+    def test_real_batch_all_failures_returns_500(self, client: TestClient):
+        with patch("ciberwebscan.core.client.HTTPClient") as mock_http_cls:
+            mock_http_cls.return_value.get.return_value = _mock_http_response(
+                404, "<html>Not Found</html>"
+            )
+            response = client.post(
+                "/api/scrape/batch",
+                json={
+                    "urls": [
+                        "https://example.com/a",
+                        "https://example.com/b",
+                    ]
+                },
+            )
+
+        assert response.status_code == 500
+        detail = response.json()["detail"]
+        assert "404" in detail
+        assert "Batch scraping failed: 500" not in detail

tests/unit/cli/test_commands.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import re
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, Mock, patch
 
 import pytest
 from typer.testing import CliRunner
@@ -20,6 +20,23 @@ def _strip_ansi(text: str) -> str:
     return re.sub(r"\x1b\[[0-9;]*m", "", text)
 
 
+def _cli_output(result) -> str:
+    """Combined stdout/stderr from a CliRunner result (click >= 8.2)."""
+    return f"{result.output}{result.stderr}"
+
+
+def _mock_http_response(
+    status: int, text: str = "<html><body>ok</body></html>"
+) -> Mock:
+    """Build a minimal HTTP response stub for the real StaticScraper."""
+    response = Mock()
+    response.status_code = status
+    response.headers = {"content-type": "text/html"}
+    response.text = text
+    response.cookies.items.return_value = {}
+    return response
+
+
 class TestAppBasic:
     """Tests for basic CLI app functionality."""
 
@@ -92,6 +109,72 @@ def test_scrape_batch_help(self):
         result = runner.invoke(app, ["scrape", "batch", "--help"])
         assert result.exit_code == 0
 
+    def test_scrape_url_http_404_exits_nonzero(self):
+        """A real HTTP 404 scrape must exit with a non-zero code."""
+        with patch("ciberwebscan.core.client.HTTPClient") as mock_http_cls:
+            mock_http_cls.return_value.get.return_value = _mock_http_response(
+                404, "<html>Not Found</html>"
+            )
+            result = runner.invoke(
+                app,
+                [
+                    "scrape",
+                    "url",
+                    "https://example.com/missing",
+                    "--no-check-robots",
+                    "-q",
+                ],
+            )
+
+        assert result.exit_code == 1
+        assert "404" in _strip_ansi(_cli_output(result))
+
+    def test_scrape_batch_all_failures_exits_nonzero(self):
+        """A fully failed batch must exit with a non-zero code."""
+        with patch("ciberwebscan.core.client.HTTPClient") as mock_http_cls:
+            mock_http_cls.return_value.get.return_value = _mock_http_response(
+                404, "<html>Not Found</html>"
+            )
+            result = runner.invoke(
+                app,
+                [
+                    "scrape",
+                    "batch",
+                    "https://example.com/a",
+                    "https://example.com/b",
+                    "--no-check-robots",
+                    "-q",
+                ],
+            )
+
+        assert result.exit_code == 1
+        assert "404" in _strip_ansi(_cli_output(result))
+
+    def test_scrape_batch_partial_failure_shows_warnings(self):
+        """Partial batches surface failed URLs instead of looking all-good."""
+        ok_response = _mock_http_response(200)
+        missing_response = _mock_http_response(404, "<html>Not Found</html>")
+        with patch("ciberwebscan.core.client.HTTPClient") as mock_http_cls:
+            mock_http_cls.return_value.get.side_effect = lambda url, **kwargs: (
+                ok_response if url.endswith("/ok") else missing_response
+            )
+            result = runner.invoke(
+                app,
+                [
+                    "scrape",
+                    "batch",
+                    "https://example.com/ok",
+                    "https://example.com/missing",
+                    "--no-check-robots",
+                ],
+            )
+
+        assert result.exit_code == 0
+        output = _strip_ansi(_cli_output(result))
+        assert "1 successful" in output
+        assert "Failed to scrape" in output
+        assert "404" in output
+
 
 class TestAnalyzeCommand:
     """Tests for analyze command."""