fix: 브라우저 오류 화면 감지와 응답 대기 진단

CHANGELOG.md

@@ -9,6 +9,8 @@ OpinetClient를 native async 하나로 통합했다. 모든 조회·디버그에
 ## [Unreleased]
 
 ### Fixed
+- HTTP 200의 서비스 이용 불가 HTML을 정상 검색 화면으로 오인하지 않고 조기 실패시킨다.
+  브라우저 응답 대기 오류는 단계와 최근 요청 종류로 진단하며 비밀 URL·본문은 기록하지 않는다.
 - 서로 다른 검색지역의 같은 주유소 ID를 전역으로 병합해 유효 가격과 주유소/충전소
   출처가 후행 빈 값으로 소실되지 않도록 수정. 가격·갱신 시각은 같은 입력의 쌍으로
   선택하며 대표 지역은 최초 검색 문맥으로 명시.

docs/browser-collection.md

@@ -59,6 +59,15 @@ HTTP 오류·접근 거부·CAPTCHA 응답은 빈 결과로 바꾸지 않고 실
 차단·CAPTCHA 응답에 대한 자동 재시도를 뜻하지 않는다. 더 짧거나 긴 제한이 필요한 호출자는
 생성자 인자로 명시할 수 있다.
 
+HTTP 200이어도 `The service is not available.` 오류 HTML이면 즉시 실패한다.
+탭 전환·지역 선택·검색의 응답 대기 실패는 `OpinetServerError`에 단계·숫자 지역 코드와
+최근 8개 요청/응답/전송 실패 이벤트를 남긴다. 인증 URL의 query, 헤더, 본문, 원 예외 문자열은
+포함하지 않는다. 이 진단은 추가 요청·자동 재시도 없이 같은 화면 동작만 관찰한다.
+HTTP 401/403/429와 취소는 기존 예외 의미를 유지한다. 오류 화면 감지가 과거 모든
+timeout 원인의 해결을 뜻하지는 않으며, 실제 수집 복구는 별도로 확인해야 한다.
+현재 이 안전 진단 범위는 응답 대기 구간이다. 초기 navigation과 응답 이후 DOM/파싱
+오류까지 같은 진단으로 정제한다는 보장은 없으며, 소비자는 기존 오류 마스킹도 유지해야 한다.
+
 ## 반복 수집과 부하 분산
 
 기본 `OpinetBrowserThrottle`은 화면 조작 사이에 0.25~1.25초의 무작위 대기를

src/opinet/experimental/browser.py

@@ -12,18 +12,21 @@
 import importlib
 import random
 import time
+from collections import deque
 from collections.abc import Awaitable, Callable, Mapping, Sequence
+from contextlib import contextmanager
 from dataclasses import dataclass, field, replace
 from datetime import datetime, timedelta
 from types import MappingProxyType
 from typing import Any, Literal, TypeAlias
+from typing import Iterator
 from urllib.parse import urlsplit
 from zoneinfo import ZoneInfo
 
 from .._convert import strip_or_none, to_bool_yn, to_float_or_none
 from ..codes import ProductCode, StationType, is_alddle
 from ..coords import katec_to_wgs84
-from ..exceptions import OpinetAuthError, OpinetRateLimitError, OpinetServerError
+from ..exceptions import OpinetAuthError, OpinetError, OpinetRateLimitError, OpinetServerError
 
 BrowserStationKind: TypeAlias = Literal["station", "lpg"]
 BrowserQueryLevel: TypeAlias = Literal["sigungu", "dong"]
@@ -195,10 +198,54 @@ async def _reject_blocked_response(response: Any, *, page: Any = None, timeout_m
         if str(page.url) != current_url:
             raise OpinetServerError("Opinet browser navigation changed while reading document") from exc
     body = body[:20_000].lower()
+    if "the service is not available." in body:
+        raise OpinetServerError("Opinet browser returned a service-unavailable page")
     if any(marker in body for marker in _BLOCKED_PAGE_MARKERS):
         raise OpinetServerError("Opinet browser response appears to be an access-block page")
 
 
+@contextmanager
+def _observe_browser_action(page: Any, phase: str) -> Iterator[None]:
+    """실패 단계와 최근 요청 종류만 남기고 URL·본문·헤더는 노출하지 않는다."""
+    events: deque[str] = deque(maxlen=8)
+    endpoints = {_PAGE_ENDPOINT, _SEARCH_ENDPOINT, "sigunguGisSelect.do", "geocodeUtmkSelect.do"}
+
+    def describe(request: Any) -> str:
+        method = request.method if request.method in {"GET", "POST"} else "other"
+        endpoint = _matching_endpoint(str(request.url), tuple(endpoints))
+        source = endpoint if _is_allowed_opinet_url(str(request.url)) and endpoint else "other"
+        return f"{method}:{source}"
+
+    def requested(request: Any) -> None:
+        if request.resource_type in {"document", "xhr", "fetch"}:
+            events.append(f"request:{describe(request)}")
+
+    def responded(response: Any) -> None:
+        if response.request.resource_type in {"document", "xhr", "fetch"}:
+            events.append(f"response:{describe(response.request)}:{int(response.status)}")
+
+    def failed(request: Any) -> None:
+        if request.resource_type in {"document", "xhr", "fetch"}:
+            events.append(f"requestfailed:{describe(request)}")
+
+    handlers = {"request": requested, "response": responded, "requestfailed": failed}
+    for event, handler in handlers.items():
+        page.on(event, handler)
+    try:
+        yield
+    except OpinetError:
+        raise
+    except Exception as exc:
+        # 원 예외 문자열은 인증 URL이나 요청 본문을 포함할 수 있으므로 복제하지 않는다.
+        raise OpinetServerError(
+            f"Opinet browser action failed: phase={phase}; cause={type(exc).__name__}; "
+            f"events={','.join(events) or 'none'}"
+        ) from None
+    finally:
+        for event, handler in handlers.items():
+            page.remove_listener(event, handler)
+
+
 def _freeze_raw_value(value: Any) -> Any:
     if isinstance(value, Mapping):
         return MappingProxyType({str(key): _freeze_raw_value(item) for key, item in value.items()})
@@ -857,11 +904,12 @@ async def _activate_tab(self, page: Any, *, station_kind: BrowserStationKind, se
             {"on", "active", "selected"}.intersection(class_name.split())
         )
         if not is_active:
-            async with page.expect_response(
-                lambda response: _is_post_response(response, _PAGE_ENDPOINT),
-                timeout=self.timeout_ms,
-            ) as response_info:
-                await locator.click(timeout=self.timeout_ms)
+            with _observe_browser_action(page, f"tab:{station_kind}"):
+                async with page.expect_response(
+                    lambda response: _is_post_response(response, _PAGE_ENDPOINT),
+                    timeout=self.timeout_ms,
+                ) as response_info:
+                    await locator.click(timeout=self.timeout_ms)
             response = await response_info.value
             _validate_response(response, _PAGE_ENDPOINT)
             await _reject_blocked_response(response, page=page, timeout_ms=self.timeout_ms)
@@ -973,11 +1021,14 @@ async def _select_value(self, page: Any, selector: str, value: str) -> None:
         if region_endpoints is None:
             await locator.select_option(value=value)
         else:
-            async with page.expect_response(
-                lambda response: any(_is_post_response(response, endpoint) for endpoint in region_endpoints),
-                timeout=self.timeout_ms,
-            ) as response_info:
-                await locator.select_option(value=value)
+            # 지역 코드만 진단에 포함한다. 임의 문자열·URL은 기록하지 않는다.
+            target = value if value.isascii() and value.isdecimal() and len(value) <= 10 else "non-numeric"
+            with _observe_browser_action(page, f"select:{selector}:target={target}"):
+                async with page.expect_response(
+                    lambda response: any(_is_post_response(response, endpoint) for endpoint in region_endpoints),
+                    timeout=self.timeout_ms,
+                ) as response_info:
+                    await locator.select_option(value=value)
             response = await response_info.value
             response_endpoint = _matching_endpoint(str(response.url), region_endpoints)
             if response_endpoint is None:
@@ -1118,12 +1169,15 @@ async def _search_region(
                 )
         await self._pause(page)
 
-        async with page.expect_response(
-            lambda response: _is_post_response(response, _SEARCH_ENDPOINT)
-            or _is_post_response(response, _PAGE_ENDPOINT),
-            timeout=self.timeout_ms,
-        ) as response_info:
-            await page.get_by_role("link", name="조회", exact=True).click(timeout=self.timeout_ms)
+        region_code = region.sigungu_value
+        target = region_code if region_code.isascii() and region_code.isdecimal() and len(region_code) <= 10 else "non-numeric"
+        with _observe_browser_action(page, f"search:{station_kind}:target={target}"):
+            async with page.expect_response(
+                lambda response: _is_post_response(response, _SEARCH_ENDPOINT)
+                or _is_post_response(response, _PAGE_ENDPOINT),
+                timeout=self.timeout_ms,
+            ) as response_info:
+                await page.get_by_role("link", name="조회", exact=True).click(timeout=self.timeout_ms)
         response = await response_info.value
         response_endpoint = _SEARCH_ENDPOINT if _has_endpoint(str(response.url), _SEARCH_ENDPOINT) else _PAGE_ENDPOINT
         _validate_response(response, response_endpoint)

tests/test_browser_playwright.py

@@ -0,0 +1,39 @@
+"""외부 통신 없이 실제 Chromium의 응답 대기 실패를 재현한다."""
+
+import pytest
+
+from opinet.exceptions import OpinetServerError
+from opinet.experimental.browser import OpinetBrowserCollector, OpinetBrowserThrottle
+
+
+@pytest.mark.asyncio
+async def test_tab_get_response_mismatch_reports_phase_without_retry():
+    api = pytest.importorskip("playwright.async_api", reason="browser extra가 필요한 검증")
+    requests = []
+    async with api.async_playwright() as playwright:
+        browser = await playwright.chromium.launch()
+        try:
+            page = await browser.new_page()
+
+            async def respond(route):
+                requests.append(route.request.method)
+                await route.fulfill(content_type="text/html", body=(
+                    '<button id="LPG_BTN" onclick="fetch(\'/searRgSelect.do?certkey=secret\')">LPG</button>'
+                ))
+
+            await page.route("**/*", respond)
+            await page.goto("https://www.opinet.co.kr/searRgSelect.do")
+            collector = OpinetBrowserCollector(
+                timeout_ms=500,
+                throttle=OpinetBrowserThrottle(action_min_seconds=0, action_max_seconds=0),
+            )
+            with pytest.raises(OpinetServerError) as raised:
+                await collector._activate_tab(page, station_kind="lpg", selector="#LPG_BTN")
+            message = str(raised.value)
+            assert "phase=tab:lpg" in message
+            assert "cause=TimeoutError" in message
+            assert "response:GET:searRgSelect.do:200" in message
+            assert "secret" not in message
+            assert requests == ["GET", "GET"]
+        finally:
+            await browser.close()

tests/test_experimental_browser.py

@@ -464,7 +464,81 @@ async def click(self, **_kwargs):
         self.clicked += 1
 
 
-class _FakeRegionPage:
+class _FakeEvents:
+    def on(self, event, handler):
+        pass
+
+    def remove_listener(self, event, handler):
+        pass
+
+
+@pytest.mark.parametrize("phase", ["tab:lpg", "select:#SIGUNGU_NM0", "search:station"])
+def test_browser_action_failure_has_bounded_redacted_network_context(phase):
+    handlers = {}
+    page = SimpleNamespace(on=handlers.__setitem__, remove_listener=lambda event, handler: handlers.pop(event))
+    request = SimpleNamespace(
+        url="https://www.opinet.co.kr/searRgSelect.do?certkey=secret",
+        method="GET", resource_type="document",
+    )
+    with pytest.raises(OpinetServerError) as raised:
+        with browser_module._observe_browser_action(page, phase):
+            for _ in range(20):
+                handlers["request"](request)
+            handlers["response"](SimpleNamespace(request=request, status=503))
+            handlers["requestfailed"](request)
+            request.url = "https://external.invalid/private-secret-path"
+            request.method = "DELETE"
+            handlers["request"](request)
+            request.resource_type = "image"
+            handlers["request"](request)
+            handlers["response"](SimpleNamespace(request=request, status=200))
+            for _ in range(12):
+                handlers["requestfailed"](request)
+            raise TimeoutError("secret response body and certkey")
+    message = str(raised.value)
+    assert f"phase={phase}" in message
+    assert "cause=TimeoutError" in message
+    assert "response:GET:searRgSelect.do:503" in message
+    assert "requestfailed:GET:searRgSelect.do" in message
+    assert "request:other:other" in message
+    assert "secret" not in message
+    assert len(message.split("events=")[1].split(",")) == 8
+    assert not handlers
+
+
+@pytest.mark.parametrize("error", [
+    browser_module.OpinetAuthError("denied"), browser_module.OpinetRateLimitError("limited"),
+    asyncio.CancelledError(),
+])
+def test_browser_action_preserves_provider_errors_and_cancellation(error):
+    handlers = {}
+    page = SimpleNamespace(on=handlers.__setitem__, remove_listener=lambda event, handler: handlers.pop(event))
+    with pytest.raises(type(error)) as raised:
+        with browser_module._observe_browser_action(page, "search:lpg"):
+            raise error
+    assert raised.value is error
+    assert not handlers
+
+
+def test_browser_action_no_response_diagnostic():
+    with pytest.raises(OpinetServerError, match="events=none"):
+        with browser_module._observe_browser_action(_FakeEvents(), "tab:lpg"):
+            raise TimeoutError()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("value,target", [("11680", "11680"), ("secret=value", "non-numeric")])
+async def test_selection_timeout_reports_only_numeric_region_target(value, target):
+    class NoResponsePage(_FakeRegionPage):
+        def expect_response(self, predicate, timeout):
+            raise TimeoutError("secret=value")
+
+    with pytest.raises(OpinetServerError, match=f"target={target}") as raised:
+        await OpinetBrowserCollector()._select_value(NoResponsePage(), "#SIGUNGU_NM0", value)
+    assert "secret=value" not in str(raised.value)
+
+
+class _FakeRegionPage(_FakeEvents):
     def __init__(self):
         self.current = {"#SIDO_NM0": "", "#SIGUNGU_NM0": "", "#DONG_NM": ""}
         self.locators = {}
@@ -579,6 +653,19 @@ async def __aexit__(self, _exc_type, _exc, _traceback):
         return False
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("body", [
+    '<html><title>error.jsp</title><body>The service is not available.</body></html>',
+    '<html><body>THE SERVICE IS NOT AVAILABLE.</body></html>',
+])
+async def test_http_200_service_unavailable_is_not_a_search_page(body):
+    response = _FakeResponse(
+        "https://www.opinet.co.kr/searRgSelect.do", content_type="text/html", text_body=body,
+    )
+    with pytest.raises(OpinetServerError, match="service-unavailable"):
+        await browser_module._reject_blocked_response(response)
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("body,blocked", [("<html>지역 검색</html>", False), ("<html>CAPTCHA</html>", True)])
 async def test_lost_navigation_response_checks_current_document(body, blocked):
@@ -639,7 +726,7 @@ async def text(self):
         await browser_module._reject_blocked_response(response, page=SimpleNamespace())
 
 
-class _FakeSearchPage:
+class _FakeSearchPage(_FakeEvents):
     def __init__(self, response, records=()):
         self.response = response
         self.records = records
@@ -768,7 +855,7 @@ async def test_browser_rejects_block_page_and_only_malformed_dom(region):
         await collector._read_dom_stations(page, region=region, station_kind="station")
 
 
-class _FakeTabPage:
+class _FakeTabPage(_FakeEvents):
     def __init__(self):
         self.tabs = {
             "#OS_BTN": _FakeLocator(attributes={"class": "on", "aria-selected": "true"}),