diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbf8e9f..fc9aacf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Poetry - run: curl -sSL https://install.python-poetry.org | python3 - + run: curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.3 python3 - - name: Configure Poetry to create virtualenvs in project root run: poetry config virtualenvs.in-project true diff --git a/proxyproviders/providers/webshare.py b/proxyproviders/providers/webshare.py index b8878f5..97f4473 100644 --- a/proxyproviders/providers/webshare.py +++ b/proxyproviders/providers/webshare.py @@ -92,7 +92,7 @@ def _fetch_proxies(self) -> List[Proxy]: all_proxies.extend([self._convert_to_proxy(proxy) for proxy in proxy_data]) - if data.get("next") is None: # no more pages + if "page" in self.search_params or data.get("next") is None: break return all_proxies diff --git a/proxyproviders/version.py b/proxyproviders/version.py index df44003..35fd71c 100644 --- a/proxyproviders/version.py +++ b/proxyproviders/version.py @@ -1,3 +1,3 @@ """Version information for proxyproviders package.""" -__version__ = "0.2.1" +__version__ = "0.2.2" diff --git a/pyproject.toml b/pyproject.toml index 824ecd4..a4363be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,9 +2,18 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" +[tool.poetry] +name = "proxyproviders" +version = "0.2.2" +description = "A unified interface for different proxy providers" +authors = ["David Teather "] + +[tool.poetry.dependencies] +python = ">=3.9" + [project] name = "proxyproviders" -version = "0.2.1" +version = "0.2.2" description = "A unified interface for different proxy providers" readme = "README.md" authors = [{ name = "David Teather", email = "contact.davidteather@gmail.com" }] diff --git a/setup.cfg b/setup.cfg index c8ba5ae..a1d69c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.1 +current_version = 0.2.2 [bumpversion:file:setup.cfg] search = version = {current_version} diff --git a/tests/providers/test_webshare.py b/tests/providers/test_webshare.py index 5cf24f5..627e218 100644 --- a/tests/providers/test_webshare.py +++ b/tests/providers/test_webshare.py @@ -182,6 +182,73 @@ def test_bad_proxy_format_response(mock_webshare): mock_webshare._fetch_proxies() +@responses.activate +def test_fetch_proxies_specific_page_stops_after_one_request(): + """ + Regression test: When 'page' is in search_params, _fetch_proxies() must + make exactly one API call and return only that page's results, even if the + response contains a 'next' URL indicating further pages exist. + """ + provider = Webshare( + api_key="test-api-key", search_params={"page": 2, "page_size": 25} + ) + + responses.add( + responses.GET, + "https://proxy.webshare.io/api/v2/proxy/list", + json={ + "results": [ + { + "id": "1", + "username": "user1", + "password": "pass1", + "proxy_address": "192.168.1.1", + "port": 8080, + "country_code": "US", + "city_name": "New York", + "created_at": "2023-05-10T12:34:56", + } + ], + "next": "https://proxy.webshare.io/api/v2/proxy/list?page=3&page_size=25", + }, + status=200, + ) + + proxies = provider._fetch_proxies() + + assert len(responses.calls) == 1 + assert len(proxies) == 1 + assert proxies[0].proxy_address == "192.168.1.1" + + +@responses.activate +def test_fetch_proxies_specific_page_sends_correct_params(): + """ + Regression test: Verify the page value from search_params is forwarded + to the API verbatim (not overridden by the internal loop counter). + """ + provider = Webshare( + api_key="test-api-key", search_params={"page": 5, "page_size": 10} + ) + + responses.add( + responses.GET, + "https://proxy.webshare.io/api/v2/proxy/list", + json={ + "results": [], + "next": "https://proxy.webshare.io/api/v2/proxy/list?page=6&page_size=10", + }, + status=200, + ) + + provider._fetch_proxies() + + assert len(responses.calls) == 1 + request_url = responses.calls[0].request.url + assert "page=5" in request_url + assert "page_size=10" in request_url + + @responses.activate def test_invalid_proxy_timestamp(mock_webshare): """