From f5c0c5b2e572ac1ee8bcd453789c74a9d91f4a6b Mon Sep 17 00:00:00 2001 From: Sn0wAlice Date: Tue, 22 Sep 2026 19:40:42 +0200 Subject: [PATCH 1/4] fix: [api] NameError on every query when requests_cache isn't installed --- pypdns/api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pypdns/api.py b/pypdns/api.py index f3454b3..3964158 100644 --- a/pypdns/api.py +++ b/pypdns/api.py @@ -344,12 +344,13 @@ def iter_query(self, q: str, if e := self._handle_dribble_errors(response): return e + if getattr(response, 'from_cache', False): + logger.debug("from cache query() q=[%s]", q) + for line in response.text.split('\n'): if len(line) == 0: continue try: - if isinstance(response, CachedResponse) and response.from_cache is True: - logger.debug("from cache query() q=[%s]", q) obj = json.loads(line) except Exception: logger.exception("except query() q=[%s]", q) @@ -376,13 +377,14 @@ def _query(self, q: str, sort_by: str = 'time_last', if response.status_code != 200: self._handle_http_error(response) errors = self._handle_dribble_errors(response) + if getattr(response, 'from_cache', False): + logger.debug("from cache query() q=[%s]", q) + to_return = [] for line in response.text.split('\n'): if len(line) == 0: continue try: - if isinstance(response, CachedResponse) and response.from_cache is True: - logger.debug("from cache query() q=[%s]", q) obj = json.loads(line) except Exception: logger.exception("except query() q=[%s]", q) From b69e61331d36c0986524f5ff824bad5f46c7d06b Mon Sep 17 00:00:00 2001 From: Sn0wAlice Date: Tue, 22 Sep 2026 19:43:37 +0200 Subject: [PATCH 2/4] new: [api] configurable page size in iter_query --- pypdns/api.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pypdns/api.py b/pypdns/api.py index 3964158..2f702d8 100644 --- a/pypdns/api.py +++ b/pypdns/api.py @@ -322,16 +322,23 @@ def __init__(self, url: str='https://www.circl.lu/pdns/query', def iter_query(self, q: str, filter_rrtype: str | None=None, - break_on_errors: bool=False) -> Generator[PDNSRecord, None, dict[str, str | int] | None]: + break_on_errors: bool=False, + *, page_size: int=50) -> Generator[PDNSRecord, None, dict[str, str | int] | None]: '''Iterate over all the recording matching your request, useful if there are a lot. Note: the order is non-deterministic. :param q: The query :param filter_rrtype: The filter, must be a valid RR Type or the response will be empty. :param break_on_errors: If there is an error, stop iterating and break immediately + :param page_size: How many records to fetch per request. The pages are fetched + sequentially, so a bigger page means fewer round trips on a big + response, at the cost of more memory per response. The server + may cap it to a lower value. ''' + if page_size < 1: + raise PDNSError('page_size must be a strictly positive integer.') cursor = -1 - query_headers = {'dribble-paginate-count': '50'} + query_headers = {'dribble-paginate-count': str(page_size)} if filter_rrtype: query_headers['dribble-filter-rrtype'] = filter_rrtype while True: From 2862cb0c21751ce899b417b1523ae7098f49990c Mon Sep 17 00:00:00 2001 From: Sn0wAlice Date: Tue, 22 Sep 2026 20:12:00 +0200 Subject: [PATCH 3/4] fix: [api] zone_time_last was never set, zone_time_first was overwritten --- pypdns/api.py | 113 +++++++++++++++++++------------------------------- 1 file changed, 42 insertions(+), 71 deletions(-) diff --git a/pypdns/api.py b/pypdns/api.py index 2f702d8..360aba0 100644 --- a/pypdns/api.py +++ b/pypdns/api.py @@ -27,6 +27,21 @@ sort_choice = ['count', 'rdata', 'rrname', 'rrtype', 'time_first', 'time_last'] +# Optional fields of a COF record, and the python type each one must have. +_optional_fields: dict[str, type] = { + 'count': int, + 'bailiwick': str, + 'sensor_id': str, + 'zone_time_first': int, + 'zone_time_last': int, + 'origin': str, + 'time_first_ms': int, + 'time_last_ms': int, + 'time_first_rfc3339': str, + 'time_last_rfc3339': str, + 'meta': dict, +} + class TypedPDNSRecord(TypedDict, total=False): '''A dict representing a Passive DNS record''' @@ -70,86 +85,42 @@ def record(self) -> TypedPDNSRecord: def __init_typed_record(self) -> TypedPDNSRecord: '''The record as a python dictionary''' - if not isinstance(self._raw_record['rrname'], str): - raise PDNSRecordTypeError('rrname', 'str', self._raw_record["rrname"]) + raw = self._raw_record + + if not isinstance(raw['rrname'], str): + raise PDNSRecordTypeError('rrname', 'str', raw['rrname']) - if not isinstance(self._raw_record['rrtype'], (str, int)): - raise PDNSRecordTypeError('rrtype', 'str, int', self._raw_record["rrtype"]) + if not isinstance(raw['rrtype'], (str, int)): + raise PDNSRecordTypeError('rrtype', 'str, int', raw['rrtype']) - if isinstance(self._raw_record['rrtype'], int): + if isinstance(raw['rrtype'], int): # Accordingly to the specs, the type can be a string OR an int. we normalize to str - rrtype: str = RdataType(self._raw_record['rrtype']).name + rrtype: str = RdataType(raw['rrtype']).name else: - rrtype = RdataType[self._raw_record['rrtype'].upper()].name + rrtype = RdataType[raw['rrtype'].upper()].name - if not isinstance(self._raw_record['rdata'], (str, list)): - raise PDNSRecordTypeError('rdata', 'str, list of string', self._raw_record["rdata"]) + if not isinstance(raw['rdata'], (str, list)): + raise PDNSRecordTypeError('rdata', 'str, list of string', raw['rdata']) - if not isinstance(self._raw_record['time_first'], int): - raise PDNSRecordTypeError('time_first', 'int', self._raw_record["time_first"]) + if not isinstance(raw['time_first'], int): + raise PDNSRecordTypeError('time_first', 'int', raw['time_first']) - if not isinstance(self._raw_record['time_last'], int): - raise PDNSRecordTypeError('time_last', 'int', self._raw_record["time_last"]) + if not isinstance(raw['time_last'], int): + raise PDNSRecordTypeError('time_last', 'int', raw['time_last']) - to_return: TypedPDNSRecord = {'rrname': self._raw_record['rrname'], + to_return: TypedPDNSRecord = {'rrname': raw['rrname'], 'rrtype': rrtype, - 'rdata': self._raw_record['rdata'], - 'time_first': self._raw_record["time_first"], - 'time_last': self._raw_record["time_last"]} - if 'count' in self._raw_record: - if not isinstance(self._raw_record['count'], int): - raise PDNSRecordTypeError('count', 'int', self._raw_record["count"]) - to_return['count'] = self._raw_record["count"] - - if 'bailiwick' in self._raw_record: - if not isinstance(self._raw_record['bailiwick'], str): - raise PDNSRecordTypeError('bailiwick', 'str', self._raw_record["bailiwick"]) - to_return['bailiwick'] = self._raw_record['bailiwick'] - - if 'sensor_id' in self._raw_record: - if not isinstance(self._raw_record['sensor_id'], str): - raise PDNSRecordTypeError('sensor_id', 'str', self._raw_record["sensor_id"]) - to_return['sensor_id'] = self._raw_record['sensor_id'] - - if 'zone_time_first' in self._raw_record: - if not isinstance(self._raw_record['zone_time_first'], int): - raise PDNSRecordTypeError('zone_time_first', 'int', self._raw_record["zone_time_first"]) - to_return['zone_time_first'] = self._raw_record['zone_time_first'] - - if 'zone_time_last' in self._raw_record: - if not isinstance(self._raw_record['zone_time_last'], int): - raise PDNSRecordTypeError('zone_time_last', 'int', self._raw_record["zone_time_last"]) - to_return['zone_time_first'] = self._raw_record["zone_time_last"] - - if 'origin' in self._raw_record: - if not isinstance(self._raw_record['origin'], str): - raise PDNSRecordTypeError('origin', 'str', self._raw_record["origin"]) - to_return['origin'] = self._raw_record["origin"] - - if 'time_first_ms' in self._raw_record: - if not isinstance(self._raw_record['time_first_ms'], int): - raise PDNSRecordTypeError('time_first_ms', 'int', self._raw_record["time_first_ms"]) - to_return['time_first_ms'] = self._raw_record["time_first_ms"] - - if 'time_last_ms' in self._raw_record: - if not isinstance(self._raw_record['time_last_ms'], int): - raise PDNSRecordTypeError('time_last_ms', 'int', self._raw_record["time_last_ms"]) - to_return['time_last_ms'] = self._raw_record['time_last_ms'] - - if 'time_first_rfc3339' in self._raw_record: - if not isinstance(self._raw_record['time_first_rfc3339'], str): - raise PDNSRecordTypeError('time_first_rfc3339', 'str', self._raw_record["time_first_rfc3339"]) - to_return['time_first_rfc3339'] = self._raw_record['time_first_rfc3339'] - - if 'time_last_rfc3339' in self._raw_record: - if not isinstance(self._raw_record['time_last_rfc3339'], str): - raise PDNSRecordTypeError('time_last_rfc3339', 'str', self._raw_record["time_last_rfc3339"]) - to_return['time_last_rfc3339'] = self._raw_record['time_last_rfc3339'] - - if 'meta' in self._raw_record: - if not isinstance(self._raw_record['meta'], dict): - raise PDNSRecordTypeError('meta', 'dict', self._raw_record["meta"]) - to_return['meta'] = self._raw_record['meta'] + 'rdata': raw['rdata'], + 'time_first': raw['time_first'], + 'time_last': raw['time_last']} + + for name, expected_type in _optional_fields.items(): + if name not in raw: + continue + value = raw[name] + if not isinstance(value, expected_type): + raise PDNSRecordTypeError(name, expected_type.__name__, value) + to_return[name] = value # type: ignore[literal-required] return to_return @property From 728aa1c89cce11893d429765b57a914e1e9928c3 Mon Sep 17 00:00:00 2001 From: Sn0wAlice Date: Tue, 22 Sep 2026 20:17:33 +0200 Subject: [PATCH 4/4] fix: [api] honour cache_file, cache_expire_after and timeout, stop patching requests globally --- pypdns/api.py | 54 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/pypdns/api.py b/pypdns/api.py index 360aba0..107a7aa 100644 --- a/pypdns/api.py +++ b/pypdns/api.py @@ -11,12 +11,12 @@ import requests from requests import Session, Response -from dns.rdatatype import RdataType +from dns.exception import DNSException +from dns.rdatatype import RdataType, to_text as rrtype_to_text from pypdns.errors import PDNSError, UnauthorizedError, ForbiddenError, RateLimitError, ServerError, PDNSRecordTypeError try: - import requests_cache from requests_cache import CachedSession from requests_cache.models import CachedResponse # type: ignore[attr-defined] HAS_CACHE = True @@ -27,6 +27,8 @@ sort_choice = ['count', 'rdata', 'rrname', 'rrtype', 'time_first', 'time_last'] +default_timeout = 15 + # Optional fields of a COF record, and the python type each one must have. _optional_fields: dict[str, type] = { 'count': int, @@ -90,14 +92,16 @@ def __init_typed_record(self) -> TypedPDNSRecord: if not isinstance(raw['rrname'], str): raise PDNSRecordTypeError('rrname', 'str', raw['rrname']) - if not isinstance(raw['rrtype'], (str, int)): - raise PDNSRecordTypeError('rrtype', 'str, int', raw['rrtype']) + rrtype_raw = raw['rrtype'] + if not isinstance(rrtype_raw, (str, int)): + raise PDNSRecordTypeError('rrtype', 'str, int', rrtype_raw) - if isinstance(raw['rrtype'], int): - # Accordingly to the specs, the type can be a string OR an int. we normalize to str - rrtype: str = RdataType(raw['rrtype']).name - else: - rrtype = RdataType[raw['rrtype'].upper()].name + try: + # Accordingly to the specs, the type can be a string OR an int. we normalize to str. + # make()/to_text() also cover the RFC 3597 "TYPE65535" spelling. + rrtype = rrtype_to_text(RdataType.make(rrtype_raw)) + except (DNSException, ValueError): + raise PDNSRecordTypeError('rrtype', 'valid RR type', rrtype_raw) if not isinstance(raw['rdata'], (str, list)): raise PDNSRecordTypeError('rdata', 'str, list of string', raw['rdata']) @@ -265,8 +269,9 @@ def __init__(self, url: str='https://www.circl.lu/pdns/query', self.session: CachedSession | Session if enable_cache is True: - requests_cache.install_cache(cache_file, backend='sqlite', expire_after=cache_expire_after) - self.session = requests_cache.CachedSession() + # NOT install_cache(): that one patches requests.Session for the whole process. + self.session = CachedSession(cache_name=cache_file, backend='sqlite', + expire_after=cache_expire_after) else: self.session = requests.Session() self.session.headers['user-agent'] = useragent if useragent else f'PyPDNS / {version("pypdns")}' @@ -294,13 +299,15 @@ def __init__(self, url: str='https://www.circl.lu/pdns/query', def iter_query(self, q: str, filter_rrtype: str | None=None, break_on_errors: bool=False, - *, page_size: int=50) -> Generator[PDNSRecord, None, dict[str, str | int] | None]: + *, page_size: int=50, + timeout: int=default_timeout) -> Generator[PDNSRecord, None, dict[str, str | int] | None]: '''Iterate over all the recording matching your request, useful if there are a lot. Note: the order is non-deterministic. :param q: The query :param filter_rrtype: The filter, must be a valid RR Type or the response will be empty. :param break_on_errors: If there is an error, stop iterating and break immediately + :param timeout: Timeout of each HTTP request, in seconds :param page_size: How many records to fetch per request. The pages are fetched sequentially, so a bigger page means fewer round trips on a big response, at the cost of more memory per response. The server @@ -315,7 +322,7 @@ def iter_query(self, q: str, while True: if cursor > 0: query_headers['dribble-paginate-cursor'] = str(cursor) - response: Response | CachedResponse = self.session.get(f'{self.url}/{q}', timeout=15, headers=query_headers) + response: Response | CachedResponse = self.session.get(f'{self.url}/{q}', timeout=timeout, headers=query_headers) if response.status_code != 200: self._handle_http_error(response) if break_on_errors: @@ -342,7 +349,8 @@ def iter_query(self, q: str, def _query(self, q: str, sort_by: str = 'time_last', *, - filter_rrtype: str | None=None) -> tuple[list[dict[str, str | int | bool | list[str] | dict[Any, Any] | None]], + filter_rrtype: str | None=None, + timeout: int=default_timeout) -> tuple[list[dict[str, str | int | bool | list[str] | dict[Any, Any] | None]], dict[str, str | int]]: '''Internal method running a non-paginated query, can be sorted.''' logger.debug("start query() q=[%s]", q) @@ -351,7 +359,7 @@ def _query(self, q: str, sort_by: str = 'time_last', query_headers = {} if filter_rrtype: query_headers['dribble-filter-rrtype'] = filter_rrtype - response: Response | CachedResponse = self.session.get(f'{self.url}/{q}', timeout=15, headers=query_headers) + response: Response | CachedResponse = self.session.get(f'{self.url}/{q}', timeout=timeout, headers=query_headers) if response.status_code != 200: self._handle_http_error(response) errors = self._handle_dribble_errors(response) @@ -368,7 +376,12 @@ def _query(self, q: str, sort_by: str = 'time_last', logger.exception("except query() q=[%s]", q) raise PDNSError(f'Unable to decode JSON object: {line}') to_return.append(obj) - to_return = sorted(to_return, key=lambda k: k[sort_by]) + try: + to_return.sort(key=lambda k: k[sort_by]) + except KeyError: + raise PDNSError(f'Unable to sort by "{sort_by}": some records do not have that key.') + except TypeError: + raise PDNSError(f'Unable to sort by "{sort_by}": records have mixed types for that key.') return to_return, errors @overload @@ -376,6 +389,7 @@ def rfc_query(self, q: str, /, *, sort_by: str = 'time_last', filter_rrtype: str | None= None, + timeout: int=default_timeout, with_errors: Literal[True]) -> tuple[list[PDNSRecord], dict[str, str | int]]: pass @@ -384,6 +398,7 @@ def rfc_query(self, q: str, /, *, sort_by: str = 'time_last', filter_rrtype: str | None= None, + timeout: int=default_timeout, with_errors: Literal[False]) -> list[PDNSRecord]: pass @@ -391,15 +406,17 @@ def rfc_query(self, q: str, /, *, sort_by: str = 'time_last', filter_rrtype: str | None= None, + timeout: int=default_timeout, with_errors: bool=False) -> list[PDNSRecord] | tuple[list[PDNSRecord], dict[str, str | int]]: '''Triggers a non-paginated query, can be sorted but will raise an error if the response is too big. :param q: The query :param sort_by: The key to use to sort the records :param filter_rrtype: The filter, must be a valid RR Type or the response will be enpty. + :param timeout: Timeout of the HTTP request, in seconds :param with_errors: Returns the errors (if any) ''' - records, errors = self._query(q, sort_by, filter_rrtype=filter_rrtype) + records, errors = self._query(q, sort_by, filter_rrtype=filter_rrtype, timeout=timeout) to_return_records = [PDNSRecord(record) for record in records] if not with_errors: return to_return_records @@ -410,7 +427,8 @@ def query(self, q: str, sort_by: str = 'time_last', timeout: int | None = None) It was a bad design decision years ago. Use rfc_query instead for something saner. This method is deprecated. ''' - records, errors = self._query(q, sort_by) + records, errors = self._query(q, sort_by, + timeout=timeout if timeout is not None else default_timeout) for record in records: record['time_first'] = datetime.fromtimestamp(record['time_first'], timezone.utc) # type: ignore record['time_last'] = datetime.fromtimestamp(record['time_last'], timezone.utc) # type: ignore