diff --git a/paimon-python/pypaimon/api/client.py b/paimon-python/pypaimon/api/client.py index 9e73c41723df..4e0214d17982 100644 --- a/paimon-python/pypaimon/api/client.py +++ b/paimon-python/pypaimon/api/client.py @@ -37,6 +37,7 @@ ServiceFailureException, ServiceUnavailableException) from pypaimon.api.typedef import RESTAuthParameter +from pypaimon.common import user_agent from pypaimon.common.json_util import JSON T = TypeVar('T', bound='RESTResponse') @@ -286,7 +287,7 @@ class HttpClient(RESTClient): _READ_TIMEOUT_SECONDS = 180 _MAX_RETRIES = 5 - def __init__(self, uri: str): + def __init__(self, uri: str, user_agent_value: Optional[str] = None): self.logger = logging.getLogger(self.__class__.__name__) self.uri = _normalize_uri(uri) self.error_handler = DefaultErrorHandler.get_instance() @@ -301,10 +302,15 @@ def __init__(self, uri: str): # and pass it explicitly on every call (see ``_execute_request``). self._timeout = (self._CONNECT_TIMEOUT_SECONDS, self._READ_TIMEOUT_SECONDS) + # A per-request User-Agent, e.g. from ``header.User-Agent``, still takes precedence. self.session.headers.update({ - 'Accept': 'application/json' + 'Accept': 'application/json', + 'User-Agent': user_agent_value or user_agent.rest_user_agent(), }) + def set_user_agent(self, value: str) -> None: + self.session.headers['User-Agent'] = value + def set_error_handler(self, error_handler: ErrorHandler) -> None: self.error_handler = error_handler diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py index aff1e8793bb4..f9ecb27faab0 100755 --- a/paimon-python/pypaimon/api/rest_api.py +++ b/paimon-python/pypaimon/api/rest_api.py @@ -50,6 +50,7 @@ from pypaimon.api.resource_paths import ResourcePaths from pypaimon.api.rest_util import RESTUtil from pypaimon.api.typedef import T +from pypaimon.common import user_agent from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions from pypaimon.common.identifier import Identifier @@ -85,7 +86,7 @@ def __init__(self, options: Union[Options, Dict[str, str]], config_required: boo raise ValueError("URI cannot be empty") self.logger = logging.getLogger(self.__class__.__name__) - self.client = HttpClient(uri) + self.client = HttpClient(uri, user_agent.rest_user_agent(options)) auth_provider = AuthProviderFactory.create_auth_provider(options) base_headers = RESTUtil.extract_prefix_map(options, self.HEADER_PREFIX) @@ -105,6 +106,7 @@ def __init__(self, options: Union[Options, Dict[str, str]], config_required: boo RESTAuthFunction(base_headers, auth_provider), ) options = config_response.merge(options) + self.client.set_user_agent(user_agent.rest_user_agent(options)) base_headers.update( RESTUtil.extract_prefix_map(options, self.HEADER_PREFIX) ) diff --git a/paimon-python/pypaimon/api/token_loader.py b/paimon-python/pypaimon/api/token_loader.py index 15e8035b0d8e..8f2affd220ef 100644 --- a/paimon-python/pypaimon/api/token_loader.py +++ b/paimon-python/pypaimon/api/token_loader.py @@ -27,6 +27,7 @@ from requests.exceptions import RequestException from pypaimon.api.client import ExponentialRetry +from pypaimon.common import user_agent from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions from pypaimon.common.json_util import JSON, json_field, json_ignore_field @@ -136,6 +137,7 @@ def __init__(self, connect_timeout: int = 180, read_timeout: int = 180, self.connect_timeout = connect_timeout self.read_timeout = read_timeout self.session = requests.Session() + self.session.headers['User-Agent'] = user_agent.rest_user_agent() # Add retry adapter retry_interceptor = ExponentialRetry(max_retries=3) diff --git a/paimon-python/pypaimon/build_info.py b/paimon-python/pypaimon/build_info.py index e034f356fd4e..22649caf7789 100644 --- a/paimon-python/pypaimon/build_info.py +++ b/paimon-python/pypaimon/build_info.py @@ -84,3 +84,11 @@ def _load_full_version(): def full_version(): """Return ``-`` for snapshot provenance.""" return _FULL_VERSION + + +def version(): + """Return the pypaimon version embedded at build time, or None when unknown.""" + prefix = "python-" + if not _FULL_VERSION.startswith(prefix): + return None + return _FULL_VERSION[len(prefix):].rsplit("-", 1)[0] or None diff --git a/paimon-python/pypaimon/common/options/config.py b/paimon-python/pypaimon/common/options/config.py index 1792dbd1df43..d6d627bd3e53 100644 --- a/paimon-python/pypaimon/common/options/config.py +++ b/paimon-python/pypaimon/common/options/config.py @@ -131,7 +131,17 @@ class CatalogOptions: "If not set, will be automatically selected based on endpoint host.") PREFIX = ConfigOptions.key("prefix").string_type().no_default_value().with_description("Prefix") HTTP_USER_AGENT_HEADER = ConfigOptions.key( - "header.HTTP_USER_AGENT").string_type().no_default_value().with_description("HTTP User Agent header") + "header.User-Agent").string_type().no_default_value().with_description( + "The User-Agent of REST catalog requests, replacing Paimon's unified one") + USER_AGENT_MODULE = ConfigOptions.key( + "user-agent.module").string_type().no_default_value().with_description( + "The module of Paimon's unified User-Agent, 'pypaimon/' by default") + USER_AGENT_FEATURES = ConfigOptions.key( + "user-agent.features").string_type().no_default_value().with_description( + "Space-separated features of Paimon's unified User-Agent") + USER_AGENT_EXTENDED = ConfigOptions.key( + "user-agent.extended").string_type().no_default_value().with_description( + "Free text appended to Paimon's unified User-Agent") SYNC_ALL_PROPERTIES = ConfigOptions.key("sync-all-properties").boolean_type().default_value(True).with_description( "Sync all table properties to the catalog metastore") RESOLVING_FILE_IO_ENABLED = ( diff --git a/paimon-python/pypaimon/common/user_agent.py b/paimon-python/pypaimon/common/user_agent.py new file mode 100644 index 000000000000..6bfac5677ff6 --- /dev/null +++ b/paimon-python/pypaimon/common/user_agent.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Paimon's unified User-Agent ``module(transport;features) extended``.""" + +import functools +from typing import Iterable, Optional + +import requests + +from pypaimon import build_info +from pypaimon.common.options import Options +from pypaimon.common.options.config import CatalogOptions + +NAME = "pypaimon" + + +@functools.lru_cache(maxsize=None) +def identity() -> str: + """``pypaimon/`` with the version embedded at build time, or bare ``pypaimon``.""" + version = build_info.version() + return "{}/{}".format(NAME, version) if version else NAME + + +def format_user_agent(module: str, transport: str, features: Iterable[str] = (), + extended: Optional[str] = None) -> str: + """Render ``module(transport;feature...) extended``, skipping empty parts.""" + user_agent = "{}({})".format(module, ";".join([transport] + [f for f in features if f])) + return "{} {}".format(user_agent, extended) if extended else user_agent + + +def rest_user_agent(options: Optional[Options] = None) -> str: + """The User-Agent of REST requests, built from the catalog's ``user-agent.*`` options.""" + data = options.to_map() if options is not None else {} + module = _value(data, CatalogOptions.USER_AGENT_MODULE.key()) or identity() + features = (_value(data, CatalogOptions.USER_AGENT_FEATURES.key()) or "").split() + return format_user_agent(module, "python-requests/" + requests.__version__, features, + _value(data, CatalogOptions.USER_AGENT_EXTENDED.key())) + + +def with_feature(options: Options, feature: str) -> None: + """Put ``feature`` first in the ``user-agent.features`` option.""" + key = CatalogOptions.USER_AGENT_FEATURES.key() + features = [f for f in (_value(options.to_map(), key) or "").split() if f != feature] + options.set(CatalogOptions.USER_AGENT_FEATURES, " ".join([feature] + features)) + + +def _value(data, key: str) -> Optional[str]: + value = data.get(key) + return str(value).strip() if value is not None and str(value).strip() else None diff --git a/paimon-python/pypaimon/filesystem/pvfs.py b/paimon-python/pypaimon/filesystem/pvfs.py index a56128ca4025..13030f08b8a0 100644 --- a/paimon-python/pypaimon/filesystem/pvfs.py +++ b/paimon-python/pypaimon/filesystem/pvfs.py @@ -34,6 +34,7 @@ from pypaimon.api.api_response import GetTableResponse, GetTableTokenResponse from pypaimon.api.client import AlreadyExistsException, NoSuchResourceException from pypaimon.api.rest_api import RESTApi +from pypaimon.common import user_agent from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions, OssOptions, PVFSOptions from pypaimon.common.identifier import Identifier @@ -149,7 +150,7 @@ class PaimonVirtualFileSystem(fsspec.AbstractFileSystem): def __init__(self, options: Union[Options, Dict[str, str]] = None, **kwargs): if isinstance(options, dict): options = Options(options) - options.set(CatalogOptions.HTTP_USER_AGENT_HEADER, 'PythonPVFS') + user_agent.with_feature(options, 'PythonPVFS') self.options = options self.warehouse = options.get(CatalogOptions.WAREHOUSE) cache_expired_time = ( diff --git a/paimon-python/pypaimon/tests/rest_user_agent_test.py b/paimon-python/pypaimon/tests/rest_user_agent_test.py new file mode 100644 index 000000000000..f2d254fb9659 --- /dev/null +++ b/paimon-python/pypaimon/tests/rest_user_agent_test.py @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import threading +import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer +from unittest import mock + +import requests + +from pypaimon import PaimonVirtualFileSystem, build_info +from pypaimon.api.rest_api import RESTApi +from pypaimon.api.token_loader import HTTPClient +from pypaimon.common import user_agent +from pypaimon.common.options import Options + +IDENTITY = "pypaimon/2.2.dev0" +REST_USER_AGENT = "{}(python-requests/{})".format(IDENTITY, requests.__version__) + + +class UserAgentIdentityTest(unittest.TestCase): + + def setUp(self): + user_agent.identity.cache_clear() + self.addCleanup(user_agent.identity.cache_clear) + + def test_identity_uses_build_version(self): + with mock.patch.object(build_info, "version", return_value="2.2.dev"): + self.assertEqual("pypaimon/2.2.dev", user_agent.identity()) + + def test_identity_without_build_version(self): + with mock.patch.object(build_info, "version", return_value=None): + self.assertEqual("pypaimon", user_agent.identity()) + + def test_build_version_from_full_version(self): + for full_version, expected in (("python-2.2.dev-abc123", "2.2.dev"), + ("python-2.2.0-UNKNOWN", "2.2.0"), + ("UNKNOWN", None)): + with mock.patch.object(build_info, "_FULL_VERSION", full_version): + self.assertEqual(expected, build_info.version()) + + +class UserAgentFormatTest(unittest.TestCase): + + def test_format(self): + self.assertEqual("m/1(t/2)", user_agent.format_user_agent("m/1", "t/2")) + self.assertEqual("m/1(t/2;a;b) ext k/v", + user_agent.format_user_agent("m/1", "t/2", ["a", "", "b"], "ext k/v")) + + def test_rest_user_agent(self): + with mock.patch.object(user_agent, "identity", return_value=IDENTITY): + self.assertEqual(REST_USER_AGENT, user_agent.rest_user_agent()) + self.assertEqual(REST_USER_AGENT[:-1] + ";Flink) vvr", user_agent.rest_user_agent( + Options({"user-agent.features": " Flink ", "user-agent.extended": "vvr"}))) + self.assertEqual("MyApp/1.0(python-requests/{})".format(requests.__version__), + user_agent.rest_user_agent(Options({"user-agent.module": "MyApp/1.0"}))) + + def test_with_feature_goes_first_once(self): + options = Options({"user-agent.features": "Flink PythonPVFS"}) + user_agent.with_feature(options, "PythonPVFS") + user_agent.with_feature(options, "PythonPVFS") + self.assertEqual("PythonPVFS Flink", options.to_map()["user-agent.features"]) + + +class RestUserAgentTest(unittest.TestCase): + + def setUp(self): + self.requests = [] + recorded = self.requests + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + recorded.append(dict(self.headers)) + body = ({"defaults": {}} if "/config" in self.path + else {"databases": [], "nextPageToken": None}) + data = json.dumps(body).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *args): + pass + + self.server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.uri = "http://127.0.0.1:{}".format(self.server.server_port) + + patcher = mock.patch.object(user_agent, "identity", return_value=IDENTITY) + patcher.start() + self.addCleanup(patcher.stop) + + def _options(self, **extra): + options = {"uri": self.uri, "warehouse": "wh", "token.provider": "bear", "token": "t"} + options.update(extra) + return options + + def _user_agents(self): + return [headers.get("User-Agent") for headers in self.requests] + + def test_default_user_agent(self): + RESTApi(Options(self._options())).list_databases() + + self.assertEqual([REST_USER_AGENT, REST_USER_AGENT], self._user_agents()) + + def test_common_options_user_agent(self): + options = self._options(**{"user-agent.features": "Flink", "user-agent.extended": "vvr"}) + RESTApi(Options(options)).list_databases() + + expected = REST_USER_AGENT[:-1] + ";Flink) vvr" + self.assertEqual([expected, expected], self._user_agents()) + + def test_user_set_user_agent_wins(self): + options = self._options(**{"header.User-Agent": "starrocks/user", "user-agent.extended": "vvr"}) + RESTApi(Options(options)).list_databases() + + self.assertEqual(["starrocks/user", "starrocks/user"], self._user_agents()) + + def test_pvfs_user_agent(self): + PaimonVirtualFileSystem(self._options(**{"user-agent.features": "Flink"})).ls("pvfs://wh/") + + pvfs_user_agent = REST_USER_AGENT[:-1] + ";PythonPVFS;Flink)" + self.assertEqual([pvfs_user_agent, pvfs_user_agent], self._user_agents()) + for headers in self.requests: + self.assertNotIn("http_user_agent", {key.lower() for key in headers}) + + def test_token_loader_user_agent(self): + HTTPClient().get(self.uri + "/v1/config") + + self.assertEqual([REST_USER_AGENT], self._user_agents()) + + +if __name__ == "__main__": + unittest.main()