From b87d7769db8d00b6e4cc61f511e82697da79e610 Mon Sep 17 00:00:00 2001 From: digitie <964189+digitie@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:14:19 +0900 Subject: [PATCH 1/2] fix: fail early on unavailable pages and diagnose browser response waits --- CHANGELOG.md | 2 + docs/browser-collection.md | 7 +++ src/opinet/experimental/browser.py | 83 ++++++++++++++++++++++++------ tests/test_browser_playwright.py | 39 ++++++++++++++ tests/test_experimental_browser.py | 79 ++++++++++++++++++++++++++-- 5 files changed, 190 insertions(+), 20 deletions(-) create mode 100644 tests/test_browser_playwright.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ca4b4d..2d4a164 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ OpinetClient를 native async 하나로 통합했다. 모든 조회·디버그에 ## [Unreleased] ### Fixed +- HTTP 200의 서비스 이용 불가 HTML을 정상 검색 화면으로 오인하지 않고 조기 실패시킨다. + 브라우저 응답 대기 오류는 단계와 최근 요청 종류로 진단하며 비밀 URL·본문은 기록하지 않는다. - 서로 다른 검색지역의 같은 주유소 ID를 전역으로 병합해 유효 가격과 주유소/충전소 출처가 후행 빈 값으로 소실되지 않도록 수정. 가격·갱신 시각은 같은 입력의 쌍으로 선택하며 대표 지역은 최초 검색 문맥으로 명시. diff --git a/docs/browser-collection.md b/docs/browser-collection.md index b8b7a63..fbbc674 100644 --- a/docs/browser-collection.md +++ b/docs/browser-collection.md @@ -59,6 +59,13 @@ HTTP 오류·접근 거부·CAPTCHA 응답은 빈 결과로 바꾸지 않고 실 차단·CAPTCHA 응답에 대한 자동 재시도를 뜻하지 않는다. 더 짧거나 긴 제한이 필요한 호출자는 생성자 인자로 명시할 수 있다. +HTTP 200이어도 `The service is not available.` 오류 HTML이면 즉시 실패한다. +탭 전환·지역 선택·검색의 응답 대기 실패는 `OpinetServerError`에 단계와 최근 8개의 +요청/응답/전송 실패 종류를 남긴다. 인증 URL의 query, 헤더, 본문, 원 예외 문자열은 +포함하지 않는다. 이 진단은 추가 요청·자동 재시도 없이 같은 화면 동작만 관찰한다. +HTTP 401/403/429와 취소는 기존 예외 의미를 유지한다. 오류 화면 감지가 과거 모든 +timeout 원인의 해결을 뜻하지는 않으며, 실제 수집 복구는 별도로 확인해야 한다. + ## 반복 수집과 부하 분산 기본 `OpinetBrowserThrottle`은 화면 조작 사이에 0.25~1.25초의 무작위 대기를 diff --git a/src/opinet/experimental/browser.py b/src/opinet/experimental/browser.py index 6ee900f..f59e0a0 100644 --- a/src/opinet/experimental/browser.py +++ b/src/opinet/experimental/browser.py @@ -12,18 +12,21 @@ import importlib import random import time +from collections import deque from collections.abc import Awaitable, Callable, Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass, field, replace from datetime import datetime, timedelta from types import MappingProxyType from typing import Any, Literal, TypeAlias +from typing import Iterator from urllib.parse import urlsplit from zoneinfo import ZoneInfo from .._convert import strip_or_none, to_bool_yn, to_float_or_none from ..codes import ProductCode, StationType, is_alddle from ..coords import katec_to_wgs84 -from ..exceptions import OpinetAuthError, OpinetRateLimitError, OpinetServerError +from ..exceptions import OpinetAuthError, OpinetError, OpinetRateLimitError, OpinetServerError BrowserStationKind: TypeAlias = Literal["station", "lpg"] BrowserQueryLevel: TypeAlias = Literal["sigungu", "dong"] @@ -195,10 +198,53 @@ async def _reject_blocked_response(response: Any, *, page: Any = None, timeout_m if str(page.url) != current_url: raise OpinetServerError("Opinet browser navigation changed while reading document") from exc body = body[:20_000].lower() + if "the service is not available." in body: + raise OpinetServerError("Opinet browser returned a service-unavailable page") if any(marker in body for marker in _BLOCKED_PAGE_MARKERS): raise OpinetServerError("Opinet browser response appears to be an access-block page") +@contextmanager +def _observe_browser_action(page: Any, phase: str) -> Iterator[None]: + """실패 단계와 최근 요청 종류만 남기고 URL·본문·헤더는 노출하지 않는다.""" + events: deque[str] = deque(maxlen=8) + endpoints = {_PAGE_ENDPOINT, _SEARCH_ENDPOINT, "sigunguGisSelect.do", "geocodeUtmkSelect.do"} + + def describe(request: Any) -> str: + method = request.method if request.method in {"GET", "POST"} else "other" + endpoint = _matching_endpoint(str(request.url), tuple(endpoints)) + source = endpoint if _is_allowed_opinet_url(str(request.url)) and endpoint else "other" + return f"{method}:{source}" + + def requested(request: Any) -> None: + if request.resource_type in {"document", "xhr", "fetch"}: + events.append(f"request:{describe(request)}") + + def responded(response: Any) -> None: + if response.request.resource_type in {"document", "xhr", "fetch"}: + events.append(f"response:{describe(response.request)}:{int(response.status)}") + + def failed(request: Any) -> None: + events.append(f"requestfailed:{describe(request)}") + + handlers = {"request": requested, "response": responded, "requestfailed": failed} + for event, handler in handlers.items(): + page.on(event, handler) + try: + yield + except OpinetError: + raise + except Exception as exc: + # 원 예외 문자열은 인증 URL이나 요청 본문을 포함할 수 있으므로 복제하지 않는다. + raise OpinetServerError( + f"Opinet browser action failed: phase={phase}; cause={type(exc).__name__}; " + f"events={','.join(events) or 'none'}" + ) from None + finally: + for event, handler in handlers.items(): + page.remove_listener(event, handler) + + def _freeze_raw_value(value: Any) -> Any: if isinstance(value, Mapping): return MappingProxyType({str(key): _freeze_raw_value(item) for key, item in value.items()}) @@ -857,11 +903,12 @@ async def _activate_tab(self, page: Any, *, station_kind: BrowserStationKind, se {"on", "active", "selected"}.intersection(class_name.split()) ) if not is_active: - async with page.expect_response( - lambda response: _is_post_response(response, _PAGE_ENDPOINT), - timeout=self.timeout_ms, - ) as response_info: - await locator.click(timeout=self.timeout_ms) + with _observe_browser_action(page, f"tab:{station_kind}"): + async with page.expect_response( + lambda response: _is_post_response(response, _PAGE_ENDPOINT), + timeout=self.timeout_ms, + ) as response_info: + await locator.click(timeout=self.timeout_ms) response = await response_info.value _validate_response(response, _PAGE_ENDPOINT) await _reject_blocked_response(response, page=page, timeout_ms=self.timeout_ms) @@ -973,11 +1020,12 @@ async def _select_value(self, page: Any, selector: str, value: str) -> None: if region_endpoints is None: await locator.select_option(value=value) else: - async with page.expect_response( - lambda response: any(_is_post_response(response, endpoint) for endpoint in region_endpoints), - timeout=self.timeout_ms, - ) as response_info: - await locator.select_option(value=value) + with _observe_browser_action(page, f"select:{selector}"): + async with page.expect_response( + lambda response: any(_is_post_response(response, endpoint) for endpoint in region_endpoints), + timeout=self.timeout_ms, + ) as response_info: + await locator.select_option(value=value) response = await response_info.value response_endpoint = _matching_endpoint(str(response.url), region_endpoints) if response_endpoint is None: @@ -1118,12 +1166,13 @@ async def _search_region( ) await self._pause(page) - async with page.expect_response( - lambda response: _is_post_response(response, _SEARCH_ENDPOINT) - or _is_post_response(response, _PAGE_ENDPOINT), - timeout=self.timeout_ms, - ) as response_info: - await page.get_by_role("link", name="조회", exact=True).click(timeout=self.timeout_ms) + with _observe_browser_action(page, f"search:{station_kind}"): + async with page.expect_response( + lambda response: _is_post_response(response, _SEARCH_ENDPOINT) + or _is_post_response(response, _PAGE_ENDPOINT), + timeout=self.timeout_ms, + ) as response_info: + await page.get_by_role("link", name="조회", exact=True).click(timeout=self.timeout_ms) response = await response_info.value response_endpoint = _SEARCH_ENDPOINT if _has_endpoint(str(response.url), _SEARCH_ENDPOINT) else _PAGE_ENDPOINT _validate_response(response, response_endpoint) diff --git a/tests/test_browser_playwright.py b/tests/test_browser_playwright.py new file mode 100644 index 0000000..d07946b --- /dev/null +++ b/tests/test_browser_playwright.py @@ -0,0 +1,39 @@ +"""외부 통신 없이 실제 Chromium의 응답 대기 실패를 재현한다.""" + +import pytest + +from opinet.exceptions import OpinetServerError +from opinet.experimental.browser import OpinetBrowserCollector, OpinetBrowserThrottle + + +@pytest.mark.asyncio +async def test_tab_get_response_mismatch_reports_phase_without_retry(): + api = pytest.importorskip("playwright.async_api", reason="browser extra가 필요한 검증") + requests = [] + async with api.async_playwright() as playwright: + browser = await playwright.chromium.launch() + try: + page = await browser.new_page() + + async def respond(route): + requests.append(route.request.method) + await route.fulfill(content_type="text/html", body=( + '' + )) + + await page.route("**/*", respond) + await page.goto("https://www.opinet.co.kr/searRgSelect.do") + collector = OpinetBrowserCollector( + timeout_ms=500, + throttle=OpinetBrowserThrottle(action_min_seconds=0, action_max_seconds=0), + ) + with pytest.raises(OpinetServerError) as raised: + await collector._activate_tab(page, station_kind="lpg", selector="#LPG_BTN") + message = str(raised.value) + assert "phase=tab:lpg" in message + assert "cause=TimeoutError" in message + assert "response:GET:searRgSelect.do:200" in message + assert "secret" not in message + assert requests == ["GET", "GET"] + finally: + await browser.close() diff --git a/tests/test_experimental_browser.py b/tests/test_experimental_browser.py index e917a60..f5eb186 100644 --- a/tests/test_experimental_browser.py +++ b/tests/test_experimental_browser.py @@ -464,7 +464,67 @@ async def click(self, **_kwargs): self.clicked += 1 -class _FakeRegionPage: +class _FakeEvents: + def on(self, event, handler): + pass + + def remove_listener(self, event, handler): + pass + + +@pytest.mark.parametrize("phase", ["tab:lpg", "select:#SIGUNGU_NM0", "search:station"]) +def test_browser_action_failure_has_bounded_redacted_network_context(phase): + handlers = {} + page = SimpleNamespace(on=handlers.__setitem__, remove_listener=lambda event, handler: handlers.pop(event)) + request = SimpleNamespace( + url="https://www.opinet.co.kr/searRgSelect.do?certkey=secret", + method="GET", resource_type="document", + ) + with pytest.raises(OpinetServerError) as raised: + with browser_module._observe_browser_action(page, phase): + for _ in range(20): + handlers["request"](request) + handlers["response"](SimpleNamespace(request=request, status=503)) + handlers["requestfailed"](request) + request.url = "https://external.invalid/private-secret-path" + request.method = "DELETE" + handlers["request"](request) + request.resource_type = "image" + handlers["request"](request) + handlers["response"](SimpleNamespace(request=request, status=200)) + raise TimeoutError("secret response body and certkey") + message = str(raised.value) + assert f"phase={phase}" in message + assert "cause=TimeoutError" in message + assert "response:GET:searRgSelect.do:503" in message + assert "requestfailed:GET:searRgSelect.do" in message + assert "request:other:other" in message + assert "secret" not in message + assert len(message.split("events=")[1].split(",")) == 8 + assert not handlers + + +@pytest.mark.parametrize("error", [ + browser_module.OpinetAuthError("denied"), browser_module.OpinetRateLimitError("limited"), + asyncio.CancelledError(), +]) +def test_browser_action_preserves_provider_errors_and_cancellation(error): + handlers = {} + page = SimpleNamespace(on=handlers.__setitem__, remove_listener=lambda event, handler: handlers.pop(event)) + with pytest.raises(type(error)) as raised: + with browser_module._observe_browser_action(page, "search:lpg"): + raise error + assert raised.value is error + assert not handlers + + +def test_browser_action_no_response_diagnostic(): + with pytest.raises(OpinetServerError, match="events=none"): + with browser_module._observe_browser_action(_FakeEvents(), "tab:lpg"): + raise TimeoutError() + + +class _FakeRegionPage(_FakeEvents): def __init__(self): self.current = {"#SIDO_NM0": "", "#SIGUNGU_NM0": "", "#DONG_NM": ""} self.locators = {} @@ -579,6 +639,19 @@ async def __aexit__(self, _exc_type, _exc, _traceback): return False +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + 'error.jspThe service is not available.', + 'THE SERVICE IS NOT AVAILABLE.', +]) +async def test_http_200_service_unavailable_is_not_a_search_page(body): + response = _FakeResponse( + "https://www.opinet.co.kr/searRgSelect.do", content_type="text/html", text_body=body, + ) + with pytest.raises(OpinetServerError, match="service-unavailable"): + await browser_module._reject_blocked_response(response) + + @pytest.mark.asyncio @pytest.mark.parametrize("body,blocked", [("지역 검색", False), ("CAPTCHA", True)]) async def test_lost_navigation_response_checks_current_document(body, blocked): @@ -639,7 +712,7 @@ async def text(self): await browser_module._reject_blocked_response(response, page=SimpleNamespace()) -class _FakeSearchPage: +class _FakeSearchPage(_FakeEvents): def __init__(self, response, records=()): self.response = response self.records = records @@ -768,7 +841,7 @@ async def test_browser_rejects_block_page_and_only_malformed_dom(region): await collector._read_dom_stations(page, region=region, station_kind="station") -class _FakeTabPage: +class _FakeTabPage(_FakeEvents): def __init__(self): self.tabs = { "#OS_BTN": _FakeLocator(attributes={"class": "on", "aria-selected": "true"}), From 5fa00463e8cd0c67aa935e8e2f0c3adef48c42e2 Mon Sep 17 00:00:00 2001 From: digitie <964189+digitie@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:18:44 +0900 Subject: [PATCH 2/2] fix: keep diagnostic events focused and identify numeric regions --- docs/browser-collection.md | 6 ++++-- src/opinet/experimental/browser.py | 11 ++++++++--- tests/test_experimental_browser.py | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/docs/browser-collection.md b/docs/browser-collection.md index fbbc674..ebbf03f 100644 --- a/docs/browser-collection.md +++ b/docs/browser-collection.md @@ -60,11 +60,13 @@ HTTP 오류·접근 거부·CAPTCHA 응답은 빈 결과로 바꾸지 않고 실 생성자 인자로 명시할 수 있다. HTTP 200이어도 `The service is not available.` 오류 HTML이면 즉시 실패한다. -탭 전환·지역 선택·검색의 응답 대기 실패는 `OpinetServerError`에 단계와 최근 8개의 -요청/응답/전송 실패 종류를 남긴다. 인증 URL의 query, 헤더, 본문, 원 예외 문자열은 +탭 전환·지역 선택·검색의 응답 대기 실패는 `OpinetServerError`에 단계·숫자 지역 코드와 +최근 8개 요청/응답/전송 실패 이벤트를 남긴다. 인증 URL의 query, 헤더, 본문, 원 예외 문자열은 포함하지 않는다. 이 진단은 추가 요청·자동 재시도 없이 같은 화면 동작만 관찰한다. HTTP 401/403/429와 취소는 기존 예외 의미를 유지한다. 오류 화면 감지가 과거 모든 timeout 원인의 해결을 뜻하지는 않으며, 실제 수집 복구는 별도로 확인해야 한다. +현재 이 안전 진단 범위는 응답 대기 구간이다. 초기 navigation과 응답 이후 DOM/파싱 +오류까지 같은 진단으로 정제한다는 보장은 없으며, 소비자는 기존 오류 마스킹도 유지해야 한다. ## 반복 수집과 부하 분산 diff --git a/src/opinet/experimental/browser.py b/src/opinet/experimental/browser.py index f59e0a0..348ad62 100644 --- a/src/opinet/experimental/browser.py +++ b/src/opinet/experimental/browser.py @@ -225,7 +225,8 @@ def responded(response: Any) -> None: events.append(f"response:{describe(response.request)}:{int(response.status)}") def failed(request: Any) -> None: - events.append(f"requestfailed:{describe(request)}") + if request.resource_type in {"document", "xhr", "fetch"}: + events.append(f"requestfailed:{describe(request)}") handlers = {"request": requested, "response": responded, "requestfailed": failed} for event, handler in handlers.items(): @@ -1020,7 +1021,9 @@ async def _select_value(self, page: Any, selector: str, value: str) -> None: if region_endpoints is None: await locator.select_option(value=value) else: - with _observe_browser_action(page, f"select:{selector}"): + # 지역 코드만 진단에 포함한다. 임의 문자열·URL은 기록하지 않는다. + target = value if value.isascii() and value.isdecimal() and len(value) <= 10 else "non-numeric" + with _observe_browser_action(page, f"select:{selector}:target={target}"): async with page.expect_response( lambda response: any(_is_post_response(response, endpoint) for endpoint in region_endpoints), timeout=self.timeout_ms, @@ -1166,7 +1169,9 @@ async def _search_region( ) await self._pause(page) - with _observe_browser_action(page, f"search:{station_kind}"): + region_code = region.sigungu_value + target = region_code if region_code.isascii() and region_code.isdecimal() and len(region_code) <= 10 else "non-numeric" + with _observe_browser_action(page, f"search:{station_kind}:target={target}"): async with page.expect_response( lambda response: _is_post_response(response, _SEARCH_ENDPOINT) or _is_post_response(response, _PAGE_ENDPOINT), diff --git a/tests/test_experimental_browser.py b/tests/test_experimental_browser.py index f5eb186..6cf7bd9 100644 --- a/tests/test_experimental_browser.py +++ b/tests/test_experimental_browser.py @@ -492,6 +492,8 @@ def test_browser_action_failure_has_bounded_redacted_network_context(phase): request.resource_type = "image" handlers["request"](request) handlers["response"](SimpleNamespace(request=request, status=200)) + for _ in range(12): + handlers["requestfailed"](request) raise TimeoutError("secret response body and certkey") message = str(raised.value) assert f"phase={phase}" in message @@ -524,6 +526,18 @@ def test_browser_action_no_response_diagnostic(): raise TimeoutError() +@pytest.mark.asyncio +@pytest.mark.parametrize("value,target", [("11680", "11680"), ("secret=value", "non-numeric")]) +async def test_selection_timeout_reports_only_numeric_region_target(value, target): + class NoResponsePage(_FakeRegionPage): + def expect_response(self, predicate, timeout): + raise TimeoutError("secret=value") + + with pytest.raises(OpinetServerError, match=f"target={target}") as raised: + await OpinetBrowserCollector()._select_value(NoResponsePage(), "#SIGUNGU_NM0", value) + assert "secret=value" not in str(raised.value) + + class _FakeRegionPage(_FakeEvents): def __init__(self): self.current = {"#SIDO_NM0": "", "#SIGUNGU_NM0": "", "#DONG_NM": ""}