'
+
+ def document(*, result=False):
+ wrong_region = result and mode == "wrong-region"
+ controls = (''
+ ''
+ ''
+ '')
+ body = link if result and mode not in ("dynamic", "empty") else ""
+ dynamic = ('') if result and mode == "dynamic" else ""
+ return (controls + ''
+ '
' + body + '
' + dynamic
+ + ('' if result else ""))
+
+ release_image = asyncio.Event()
+ image_requested = asyncio.Event()
+ async with api.async_playwright() as playwright:
+ browser = await playwright.chromium.launch()
+ try:
+ page = await browser.new_page()
+
+ async def respond(route):
+ if route.request.url.endswith("/slow-image.png"):
+ image_requested.set()
+ await release_image.wait()
+ await route.fulfill(content_type="image/png", body=b"")
+ else:
+ await route.fulfill(content_type="text/html; charset=utf-8", body=document(result=route.request.method == "POST"))
+
+ await page.route("**/*", respond)
+ await page.goto("https://www.opinet.co.kr/searRgSelect.do")
+ collector = OpinetBrowserCollector(timeout_ms=1500)
+ task = asyncio.create_task(collector._search_region(page, region, station_kind="station"))
+ try:
+ await asyncio.wait_for(image_requested.wait(), 3)
+ except TimeoutError:
+ if task.done():
+ task.result()
+ raise
+ if mode == "server":
+ stations = await asyncio.wait_for(task, 1)
+ assert [station.station_id for station in stations] == ["A0000001"]
+ assert stations[0].price_by_product
+ assert await page.evaluate("document.readyState") == "interactive"
+ elif mode in ("wrong-region", "wrong-tab"):
+ with pytest.raises(Exception, match="Timeout"):
+ await task
+ else:
+ # 빈 표를 DCL 시점에 정상 결과로 확정하면 동적 결과를 잃는다.
+ await asyncio.sleep(0.1)
+ assert not task.done()
+ release_image.set()
+ stations = await asyncio.wait_for(task, 2)
+ assert len(stations) == (1 if mode == "dynamic" else 0)
+ finally:
+ release_image.set()
+ await browser.close()
diff --git a/tests/test_experimental_browser.py b/tests/test_experimental_browser.py
index a52021d..950952b 100644
--- a/tests/test_experimental_browser.py
+++ b/tests/test_experimental_browser.py
@@ -419,6 +419,20 @@ def test_browser_collector_rejects_non_opinet_url():
OpinetBrowserCollector(query_level="invalid")
+@pytest.mark.asyncio
+async def test_default_action_delays_remain_random_and_bounded():
+ page = _FakeRegionPage()
+ collector = OpinetBrowserCollector(rng=random.Random(23))
+ for _ in range(100):
+ await collector._pause(page)
+ assert len(page.waits) == 100
+ assert min(page.waits) >= 50
+ assert max(page.waits) <= 150
+ assert len(set(page.waits)) > 20
+ # 전국 약230개 시군구의 두 탭에서 조작 대기가 시간을 지배하지 않는다.
+ assert sum(page.waits) / len(page.waits) * 1150 < 180_000
+
+
def test_browser_loader_reports_optional_dependency(monkeypatch):
def missing(_name):
raise ModuleNotFoundError("playwright")
@@ -736,6 +750,9 @@ def __init__(self, response, records=()):
self.selectors = {}
self.waits = []
+ async def evaluate(self, _script):
+ return 1.0
+
def locator(self, selector):
if selector == "table.tbl_type10 tbody tr":
return _FakeLocator(values=self.records)
From b2af98e7bd276130fc3cb2e56ef38dd32f60c13a Mon Sep 17 00:00:00 2001
From: digitie <964189+digitie@users.noreply.github.com>
Date: Sat, 24 Oct 2026 09:58:33 +0900
Subject: [PATCH 2/4] fix(browser): await selection bodies and validate empty
result context
---
docs/browser-collection.md | 2 ++
src/opinet/experimental/browser.py | 47 +++++++++++++++---------------
tests/test_browser_playwright.py | 11 +++----
tests/test_experimental_browser.py | 36 +++++++++++++++++++++++
4 files changed, 68 insertions(+), 28 deletions(-)
diff --git a/docs/browser-collection.md b/docs/browser-collection.md
index ed8866d..c5aeed9 100644
--- a/docs/browser-collection.md
+++ b/docs/browser-collection.md
@@ -5,6 +5,8 @@
기본 화면 조작 간격은 매번 독립적으로 뽑는 **0.05~0.15초 무작위 대기**다.
기존 0.25~1.25초보다 평균 대기는 0.75초에서 0.10초로 줄지만 랜덤성은 유지한다.
지역 선택 AJAX 완료·선택값 일치·검색 응답 검증은 생략하지 않는다.
+지역 선택은 헤더 수신뿐 아니라 응답 본문 완료까지 제한 시간 안에 기다린 뒤
+하위 목록과 선택값을 확인한다. 빈 결과도 선택 지역·활성 탭이 맞아야 확정한다.
HTML 검색 응답의 결과 표에 일반 주유소 원본 링크가 이미 있으면 새 문서·지역 선택값·
활성 탭·응답 원본 링크 전체와 DOM의 일치를 확인하고 이미지·지도 자원의 `load`를
diff --git a/src/opinet/experimental/browser.py b/src/opinet/experimental/browser.py
index 0c3214c..52943b3 100644
--- a/src/opinet/experimental/browser.py
+++ b/src/opinet/experimental/browser.py
@@ -1144,30 +1144,28 @@ async def _wait_for_results_dom(
rendered = _ServerResultLinks()
if response_body is not None:
rendered.feed(response_body)
- if rendered.links and not rendered.has_dynamic_fields and previous_document is not None and region is not None:
- # 새 문서·선택값·응답에 있던 모든 원본 필드의 일치를 확인한다. 이미지·지도
- # load가 늦어도 이미 완성된 결과를 읽는다. 빈/동적 결과는 아래 기존 대기를 쓴다.
- await page.wait_for_function(
- "({previous, values, links, tab}) => { "
- "if (performance.timeOrigin === previous || document.readyState === 'loading') return false; "
- "if (!values.every(([selector, value]) => document.querySelector(selector)?.value === value)) return false; "
- "const active = document.querySelector(tab); "
- "if (!active || !(active.getAttribute('aria-selected') === 'true' || "
- "['on', 'active', 'selected'].some(name => active.classList.contains(name)))) return false; "
- "const actual = new Set(Array.from(document.querySelectorAll('table.tbl_type10 a'), "
- "a => a.getAttribute('href')).filter(href => href && "
- "(href.startsWith('javascript:fn_osPop(') || href.startsWith('javascript:fnVolatInfowindow(')))); "
- "return actual.size === links.length && links.every(link => actual.has(link)); }",
- arg={"previous": previous_document,
- "values": [["#SIDO_NM0", region.sido_value], ["#SIGUNGU_NM0", region.sigungu_value],
- ["#DONG_NM", region.dong_value or ""]],
- "links": sorted(rendered.links), "tab": _TAB_SELECTORS[station_kind]},
- timeout=self.timeout_ms,
- )
- return
+ fast = bool(rendered.links and not rendered.has_dynamic_fields
+ and previous_document is not None and region is not None)
+ # 새 문서·선택값·탭은 빈 결과에도 확인한다. 원본 링크 증거가 있는 경우만
+ # 이미지·지도 load를 기다리지 않고 완성된 결과를 읽는다.
await page.wait_for_function(
- "() => document.readyState === 'complete' && "
- "document.querySelector('table.tbl_type10') !== null",
+ "({previous, values, links, tab, fast}) => { "
+ "if (previous !== null && performance.timeOrigin === previous) return false; "
+ "if (fast ? document.readyState === 'loading' : document.readyState !== 'complete') return false; "
+ "if (!document.querySelector('table.tbl_type10')) return false; "
+ "if (!values.every(([selector, value]) => document.querySelector(selector)?.value === value)) return false; "
+ "if (tab) { const active = document.querySelector(tab); "
+ "if (!active || !(active.getAttribute('aria-selected') === 'true' || "
+ "['on', 'active', 'selected'].some(name => active.classList.contains(name)))) return false; } "
+ "if (!fast) return true; "
+ "const actual = new Set(Array.from(document.querySelectorAll('table.tbl_type10 a'), "
+ "a => a.getAttribute('href')).filter(href => href && "
+ "(href.startsWith('javascript:fn_osPop(') || href.startsWith('javascript:fnVolatInfowindow(')))); "
+ "return actual.size === links.length && links.every(link => actual.has(link)); }",
+ arg={"previous": previous_document, "fast": fast,
+ "values": [["#SIDO_NM0", region.sido_value], ["#SIGUNGU_NM0", region.sigungu_value],
+ ["#DONG_NM", region.dong_value or ""]] if region is not None else [],
+ "links": sorted(rendered.links), "tab": _TAB_SELECTORS[station_kind] if region is not None else None},
timeout=self.timeout_ms,
)
@@ -1278,6 +1276,9 @@ async def _select_value(self, page: Any, selector: str, value: str, *, parent: s
raise OpinetServerError("unexpected Opinet region response URL")
_validate_response(response, response_endpoint)
await _reject_blocked_response(response, page=page, timeout_ms=self.timeout_ms)
+ # expect_response는 헤더 수신 시점이다. 짧은 랜덤 대기 전에 본문까지
+ # 받아야 하위 목록 갱신이 느린 응답을 이전 옵션으로 오인하지 않는다.
+ await asyncio.wait_for(response.finished(), timeout=self.timeout_ms / 1000)
await self._pause(page)
if dependent_selector is not None:
await page.wait_for_function(
diff --git a/tests/test_browser_playwright.py b/tests/test_browser_playwright.py
index 1c18260..b427447 100644
--- a/tests/test_browser_playwright.py
+++ b/tests/test_browser_playwright.py
@@ -44,7 +44,7 @@ async def respond(route):
@pytest.mark.asyncio
-@pytest.mark.parametrize("mode", ["server", "dynamic", "empty", "wrong-region", "wrong-tab"])
+@pytest.mark.parametrize("mode", ["server", "dynamic", "empty", "wrong-region", "wrong-tab", "empty-wrong-region", "empty-wrong-tab"])
async def test_html_results_wait_for_data_instead_of_unrelated_image(mode):
api = pytest.importorskip("playwright.async_api", reason="browser extra가 필요한 검증")
region = BrowserRegion(sido_value="11", sido_name="서울", sigungu_value="110", sigungu_name="테스트구")
@@ -53,12 +53,12 @@ async def test_html_results_wait_for_data_instead_of_unrelated_image(mode):
link = '
' if result else ""))
+
+ release_image = asyncio.Event()
+ image_requested = asyncio.Event()
+ async with api.async_playwright() as playwright:
+ browser = await playwright.chromium.launch()
+ try:
+ page = await browser.new_page()
+
+ async def respond(route):
+ if route.request.url.endswith("/slow-image.png"):
+ image_requested.set()
+ await release_image.wait()
+ await route.fulfill(content_type="image/png", body=b"")
+ else:
+ await route.fulfill(content_type="text/html; charset=utf-8", body=document(result=route.request.method == "POST"))
+
+ await page.route("**/*", respond)
+ await page.goto("https://www.opinet.co.kr/searRgSelect.do")
+ collector = OpinetBrowserCollector(timeout_ms=1500)
+ task = asyncio.create_task(collector._search_region(page, region, station_kind="station"))
+ try:
+ await asyncio.wait_for(image_requested.wait(), 3)
+ except TimeoutError:
+ if task.done():
+ task.result()
+ raise
+ if mode == "server":
+ stations = await asyncio.wait_for(task, 1)
+ assert [station.station_id for station in stations] == ["A0000001"]
+ assert stations[0].price_by_product
+ assert await page.evaluate("document.readyState") == "interactive"
+ elif mode in ("wrong-region", "wrong-tab"):
+ with pytest.raises(Exception, match="Timeout"):
+ await task
+ else:
+ # 빈 표를 DCL 시점에 정상 결과로 확정하면 동적 결과를 잃는다.
+ await asyncio.sleep(0.1)
+ assert not task.done()
+ release_image.set()
+ stations = await asyncio.wait_for(task, 2)
+ assert len(stations) == (1 if mode == "dynamic" else 0)
+ finally:
+ release_image.set()
+ await browser.close()
diff --git a/tests/test_experimental_browser.py b/tests/test_experimental_browser.py
index a52021d..950952b 100644
--- a/tests/test_experimental_browser.py
+++ b/tests/test_experimental_browser.py
@@ -419,6 +419,20 @@ def test_browser_collector_rejects_non_opinet_url():
OpinetBrowserCollector(query_level="invalid")
+@pytest.mark.asyncio
+async def test_default_action_delays_remain_random_and_bounded():
+ page = _FakeRegionPage()
+ collector = OpinetBrowserCollector(rng=random.Random(23))
+ for _ in range(100):
+ await collector._pause(page)
+ assert len(page.waits) == 100
+ assert min(page.waits) >= 50
+ assert max(page.waits) <= 150
+ assert len(set(page.waits)) > 20
+ # 전국 약230개 시군구의 두 탭에서 조작 대기가 시간을 지배하지 않는다.
+ assert sum(page.waits) / len(page.waits) * 1150 < 180_000
+
+
def test_browser_loader_reports_optional_dependency(monkeypatch):
def missing(_name):
raise ModuleNotFoundError("playwright")
@@ -736,6 +750,9 @@ def __init__(self, response, records=()):
self.selectors = {}
self.waits = []
+ async def evaluate(self, _script):
+ return 1.0
+
def locator(self, selector):
if selector == "table.tbl_type10 tbody tr":
return _FakeLocator(values=self.records)
From b2af98e7bd276130fc3cb2e56ef38dd32f60c13a Mon Sep 17 00:00:00 2001
From: digitie <964189+digitie@users.noreply.github.com>
Date: Sat, 24 Oct 2026 09:58:33 +0900
Subject: [PATCH 2/4] fix(browser): await selection bodies and validate empty
result context
---
docs/browser-collection.md | 2 ++
src/opinet/experimental/browser.py | 47 +++++++++++++++---------------
tests/test_browser_playwright.py | 11 +++----
tests/test_experimental_browser.py | 36 +++++++++++++++++++++++
4 files changed, 68 insertions(+), 28 deletions(-)
diff --git a/docs/browser-collection.md b/docs/browser-collection.md
index ed8866d..c5aeed9 100644
--- a/docs/browser-collection.md
+++ b/docs/browser-collection.md
@@ -5,6 +5,8 @@
기본 화면 조작 간격은 매번 독립적으로 뽑는 **0.05~0.15초 무작위 대기**다.
기존 0.25~1.25초보다 평균 대기는 0.75초에서 0.10초로 줄지만 랜덤성은 유지한다.
지역 선택 AJAX 완료·선택값 일치·검색 응답 검증은 생략하지 않는다.
+지역 선택은 헤더 수신뿐 아니라 응답 본문 완료까지 제한 시간 안에 기다린 뒤
+하위 목록과 선택값을 확인한다. 빈 결과도 선택 지역·활성 탭이 맞아야 확정한다.
HTML 검색 응답의 결과 표에 일반 주유소 원본 링크가 이미 있으면 새 문서·지역 선택값·
활성 탭·응답 원본 링크 전체와 DOM의 일치를 확인하고 이미지·지도 자원의 `load`를
diff --git a/src/opinet/experimental/browser.py b/src/opinet/experimental/browser.py
index 0c3214c..52943b3 100644
--- a/src/opinet/experimental/browser.py
+++ b/src/opinet/experimental/browser.py
@@ -1144,30 +1144,28 @@ async def _wait_for_results_dom(
rendered = _ServerResultLinks()
if response_body is not None:
rendered.feed(response_body)
- if rendered.links and not rendered.has_dynamic_fields and previous_document is not None and region is not None:
- # 새 문서·선택값·응답에 있던 모든 원본 필드의 일치를 확인한다. 이미지·지도
- # load가 늦어도 이미 완성된 결과를 읽는다. 빈/동적 결과는 아래 기존 대기를 쓴다.
- await page.wait_for_function(
- "({previous, values, links, tab}) => { "
- "if (performance.timeOrigin === previous || document.readyState === 'loading') return false; "
- "if (!values.every(([selector, value]) => document.querySelector(selector)?.value === value)) return false; "
- "const active = document.querySelector(tab); "
- "if (!active || !(active.getAttribute('aria-selected') === 'true' || "
- "['on', 'active', 'selected'].some(name => active.classList.contains(name)))) return false; "
- "const actual = new Set(Array.from(document.querySelectorAll('table.tbl_type10 a'), "
- "a => a.getAttribute('href')).filter(href => href && "
- "(href.startsWith('javascript:fn_osPop(') || href.startsWith('javascript:fnVolatInfowindow(')))); "
- "return actual.size === links.length && links.every(link => actual.has(link)); }",
- arg={"previous": previous_document,
- "values": [["#SIDO_NM0", region.sido_value], ["#SIGUNGU_NM0", region.sigungu_value],
- ["#DONG_NM", region.dong_value or ""]],
- "links": sorted(rendered.links), "tab": _TAB_SELECTORS[station_kind]},
- timeout=self.timeout_ms,
- )
- return
+ fast = bool(rendered.links and not rendered.has_dynamic_fields
+ and previous_document is not None and region is not None)
+ # 새 문서·선택값·탭은 빈 결과에도 확인한다. 원본 링크 증거가 있는 경우만
+ # 이미지·지도 load를 기다리지 않고 완성된 결과를 읽는다.
await page.wait_for_function(
- "() => document.readyState === 'complete' && "
- "document.querySelector('table.tbl_type10') !== null",
+ "({previous, values, links, tab, fast}) => { "
+ "if (previous !== null && performance.timeOrigin === previous) return false; "
+ "if (fast ? document.readyState === 'loading' : document.readyState !== 'complete') return false; "
+ "if (!document.querySelector('table.tbl_type10')) return false; "
+ "if (!values.every(([selector, value]) => document.querySelector(selector)?.value === value)) return false; "
+ "if (tab) { const active = document.querySelector(tab); "
+ "if (!active || !(active.getAttribute('aria-selected') === 'true' || "
+ "['on', 'active', 'selected'].some(name => active.classList.contains(name)))) return false; } "
+ "if (!fast) return true; "
+ "const actual = new Set(Array.from(document.querySelectorAll('table.tbl_type10 a'), "
+ "a => a.getAttribute('href')).filter(href => href && "
+ "(href.startsWith('javascript:fn_osPop(') || href.startsWith('javascript:fnVolatInfowindow(')))); "
+ "return actual.size === links.length && links.every(link => actual.has(link)); }",
+ arg={"previous": previous_document, "fast": fast,
+ "values": [["#SIDO_NM0", region.sido_value], ["#SIGUNGU_NM0", region.sigungu_value],
+ ["#DONG_NM", region.dong_value or ""]] if region is not None else [],
+ "links": sorted(rendered.links), "tab": _TAB_SELECTORS[station_kind] if region is not None else None},
timeout=self.timeout_ms,
)
@@ -1278,6 +1276,9 @@ async def _select_value(self, page: Any, selector: str, value: str, *, parent: s
raise OpinetServerError("unexpected Opinet region response URL")
_validate_response(response, response_endpoint)
await _reject_blocked_response(response, page=page, timeout_ms=self.timeout_ms)
+ # expect_response는 헤더 수신 시점이다. 짧은 랜덤 대기 전에 본문까지
+ # 받아야 하위 목록 갱신이 느린 응답을 이전 옵션으로 오인하지 않는다.
+ await asyncio.wait_for(response.finished(), timeout=self.timeout_ms / 1000)
await self._pause(page)
if dependent_selector is not None:
await page.wait_for_function(
diff --git a/tests/test_browser_playwright.py b/tests/test_browser_playwright.py
index 1c18260..b427447 100644
--- a/tests/test_browser_playwright.py
+++ b/tests/test_browser_playwright.py
@@ -44,7 +44,7 @@ async def respond(route):
@pytest.mark.asyncio
-@pytest.mark.parametrize("mode", ["server", "dynamic", "empty", "wrong-region", "wrong-tab"])
+@pytest.mark.parametrize("mode", ["server", "dynamic", "empty", "wrong-region", "wrong-tab", "empty-wrong-region", "empty-wrong-tab"])
async def test_html_results_wait_for_data_instead_of_unrelated_image(mode):
api = pytest.importorskip("playwright.async_api", reason="browser extra가 필요한 검증")
region = BrowserRegion(sido_value="11", sido_name="서울", sigungu_value="110", sigungu_name="테스트구")
@@ -53,12 +53,12 @@ async def test_html_results_wait_for_data_instead_of_unrelated_image(mode):
link = '