[python] Unify the REST User-Agent format

paimon-python/pypaimon/api/client.py

@@ -37,6 +37,7 @@
                                          ServiceFailureException,
                                          ServiceUnavailableException)
 from pypaimon.api.typedef import RESTAuthParameter
+from pypaimon.common import user_agent
 from pypaimon.common.json_util import JSON
 
 T = TypeVar('T', bound='RESTResponse')
@@ -286,7 +287,7 @@ class HttpClient(RESTClient):
     _READ_TIMEOUT_SECONDS = 180
     _MAX_RETRIES = 5
 
-    def __init__(self, uri: str):
+    def __init__(self, uri: str, user_agent_value: Optional[str] = None):
         self.logger = logging.getLogger(self.__class__.__name__)
         self.uri = _normalize_uri(uri)
         self.error_handler = DefaultErrorHandler.get_instance()
@@ -301,10 +302,15 @@ def __init__(self, uri: str):
         # and pass it explicitly on every call (see ``_execute_request``).
         self._timeout = (self._CONNECT_TIMEOUT_SECONDS, self._READ_TIMEOUT_SECONDS)
 
+        # A per-request User-Agent, e.g. from ``header.User-Agent``, still takes precedence.
         self.session.headers.update({
-            'Accept': 'application/json'
+            'Accept': 'application/json',
+            'User-Agent': user_agent_value or user_agent.rest_user_agent(),
         })
 
+    def set_user_agent(self, value: str) -> None:
+        self.session.headers['User-Agent'] = value
+
     def set_error_handler(self, error_handler: ErrorHandler) -> None:
         self.error_handler = error_handler
 

paimon-python/pypaimon/api/rest_api.py

@@ -50,6 +50,7 @@
 from pypaimon.api.resource_paths import ResourcePaths
 from pypaimon.api.rest_util import RESTUtil
 from pypaimon.api.typedef import T
+from pypaimon.common import user_agent
 from pypaimon.common.options import Options
 from pypaimon.common.options.config import CatalogOptions
 from pypaimon.common.identifier import Identifier
@@ -85,7 +86,7 @@ def __init__(self, options: Union[Options, Dict[str, str]], config_required: boo
             raise ValueError("URI cannot be empty")
 
         self.logger = logging.getLogger(self.__class__.__name__)
-        self.client = HttpClient(uri)
+        self.client = HttpClient(uri, user_agent.rest_user_agent(options))
         auth_provider = AuthProviderFactory.create_auth_provider(options)
         base_headers = RESTUtil.extract_prefix_map(options, self.HEADER_PREFIX)
 
@@ -105,6 +106,7 @@ def __init__(self, options: Union[Options, Dict[str, str]], config_required: boo
                 RESTAuthFunction(base_headers, auth_provider),
             )
             options = config_response.merge(options)
+            self.client.set_user_agent(user_agent.rest_user_agent(options))
             base_headers.update(
                 RESTUtil.extract_prefix_map(options, self.HEADER_PREFIX)
             )

paimon-python/pypaimon/api/token_loader.py

@@ -27,6 +27,7 @@
 from requests.exceptions import RequestException
 
 from pypaimon.api.client import ExponentialRetry
+from pypaimon.common import user_agent
 from pypaimon.common.options import Options
 from pypaimon.common.options.config import CatalogOptions
 from pypaimon.common.json_util import JSON, json_field, json_ignore_field
@@ -136,6 +137,7 @@ def __init__(self, connect_timeout: int = 180, read_timeout: int = 180,
         self.connect_timeout = connect_timeout
         self.read_timeout = read_timeout
         self.session = requests.Session()
+        self.session.headers['User-Agent'] = user_agent.rest_user_agent()
 
         # Add retry adapter
         retry_interceptor = ExponentialRetry(max_retries=3)

paimon-python/pypaimon/build_info.py

@@ -84,3 +84,11 @@ def _load_full_version():
 def full_version():
     """Return ``<pypaimon-version>-<commit-id>`` for snapshot provenance."""
     return _FULL_VERSION
+
+
+def version():
+    """Return the pypaimon version embedded at build time, or None when unknown."""
+    prefix = "python-"
+    if not _FULL_VERSION.startswith(prefix):
+        return None
+    return _FULL_VERSION[len(prefix):].rsplit("-", 1)[0] or None

paimon-python/pypaimon/common/options/config.py

@@ -131,7 +131,17 @@ class CatalogOptions:
         "If not set, will be automatically selected based on endpoint host.")
     PREFIX = ConfigOptions.key("prefix").string_type().no_default_value().with_description("Prefix")
     HTTP_USER_AGENT_HEADER = ConfigOptions.key(
-        "header.HTTP_USER_AGENT").string_type().no_default_value().with_description("HTTP User Agent header")
+        "header.User-Agent").string_type().no_default_value().with_description(
+        "The User-Agent of REST catalog requests, replacing Paimon's unified one")
+    USER_AGENT_MODULE = ConfigOptions.key(
+        "user-agent.module").string_type().no_default_value().with_description(
+        "The module of Paimon's unified User-Agent, 'pypaimon/<version>' by default")
+    USER_AGENT_FEATURES = ConfigOptions.key(
+        "user-agent.features").string_type().no_default_value().with_description(
+        "Space-separated features of Paimon's unified User-Agent")
+    USER_AGENT_EXTENDED = ConfigOptions.key(
+        "user-agent.extended").string_type().no_default_value().with_description(
+        "Free text appended to Paimon's unified User-Agent")
     SYNC_ALL_PROPERTIES = ConfigOptions.key("sync-all-properties").boolean_type().default_value(True).with_description(
         "Sync all table properties to the catalog metastore")
     RESOLVING_FILE_IO_ENABLED = (

paimon-python/pypaimon/common/user_agent.py

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Paimon's unified User-Agent ``module(transport;features) extended``."""
+
+import functools
+from typing import Iterable, Optional
+
+import requests
+
+from pypaimon import build_info
+from pypaimon.common.options import Options
+from pypaimon.common.options.config import CatalogOptions
+
+NAME = "pypaimon"
+
+
+@functools.lru_cache(maxsize=None)
+def identity() -> str:
+    """``pypaimon/<version>`` with the version embedded at build time, or bare ``pypaimon``."""
+    version = build_info.version()
+    return "{}/{}".format(NAME, version) if version else NAME
+
+
+def format_user_agent(module: str, transport: str, features: Iterable[str] = (),
+                      extended: Optional[str] = None) -> str:
+    """Render ``module(transport;feature...) extended``, skipping empty parts."""
+    user_agent = "{}({})".format(module, ";".join([transport] + [f for f in features if f]))
+    return "{} {}".format(user_agent, extended) if extended else user_agent
+
+
+def rest_user_agent(options: Optional[Options] = None) -> str:
+    """The User-Agent of REST requests, built from the catalog's ``user-agent.*`` options."""
+    data = options.to_map() if options is not None else {}
+    module = _value(data, CatalogOptions.USER_AGENT_MODULE.key()) or identity()
+    features = (_value(data, CatalogOptions.USER_AGENT_FEATURES.key()) or "").split()
+    return format_user_agent(module, "python-requests/" + requests.__version__, features,
+                             _value(data, CatalogOptions.USER_AGENT_EXTENDED.key()))
+
+
+def with_feature(options: Options, feature: str) -> None:
+    """Put ``feature`` first in the ``user-agent.features`` option."""
+    key = CatalogOptions.USER_AGENT_FEATURES.key()
+    features = [f for f in (_value(options.to_map(), key) or "").split() if f != feature]
+    options.set(CatalogOptions.USER_AGENT_FEATURES, " ".join([feature] + features))
+
+
+def _value(data, key: str) -> Optional[str]:
+    value = data.get(key)
+    return str(value).strip() if value is not None and str(value).strip() else None

paimon-python/pypaimon/filesystem/pvfs.py

@@ -34,6 +34,7 @@
 from pypaimon.api.api_response import GetTableResponse, GetTableTokenResponse
 from pypaimon.api.client import AlreadyExistsException, NoSuchResourceException
 from pypaimon.api.rest_api import RESTApi
+from pypaimon.common import user_agent
 from pypaimon.common.options import Options
 from pypaimon.common.options.config import CatalogOptions, OssOptions, PVFSOptions
 from pypaimon.common.identifier import Identifier
@@ -149,7 +150,7 @@ class PaimonVirtualFileSystem(fsspec.AbstractFileSystem):
     def __init__(self, options: Union[Options, Dict[str, str]] = None, **kwargs):
         if isinstance(options, dict):
             options = Options(options)
-        options.set(CatalogOptions.HTTP_USER_AGENT_HEADER, 'PythonPVFS')
+        user_agent.with_feature(options, 'PythonPVFS')
         self.options = options
         self.warehouse = options.get(CatalogOptions.WAREHOUSE)
         cache_expired_time = (

paimon-python/pypaimon/tests/rest_user_agent_test.py

@@ -0,0 +1,152 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from unittest import mock
+
+import requests
+
+from pypaimon import PaimonVirtualFileSystem, build_info
+from pypaimon.api.rest_api import RESTApi
+from pypaimon.api.token_loader import HTTPClient
+from pypaimon.common import user_agent
+from pypaimon.common.options import Options
+
+IDENTITY = "pypaimon/2.2.dev0"
+REST_USER_AGENT = "{}(python-requests/{})".format(IDENTITY, requests.__version__)
+
+
+class UserAgentIdentityTest(unittest.TestCase):
+
+    def setUp(self):
+        user_agent.identity.cache_clear()
+        self.addCleanup(user_agent.identity.cache_clear)
+
+    def test_identity_uses_build_version(self):
+        with mock.patch.object(build_info, "version", return_value="2.2.dev"):
+            self.assertEqual("pypaimon/2.2.dev", user_agent.identity())
+
+    def test_identity_without_build_version(self):
+        with mock.patch.object(build_info, "version", return_value=None):
+            self.assertEqual("pypaimon", user_agent.identity())
+
+    def test_build_version_from_full_version(self):
+        for full_version, expected in (("python-2.2.dev-abc123", "2.2.dev"),
+                                       ("python-2.2.0-UNKNOWN", "2.2.0"),
+                                       ("UNKNOWN", None)):
+            with mock.patch.object(build_info, "_FULL_VERSION", full_version):
+                self.assertEqual(expected, build_info.version())
+
+
+class UserAgentFormatTest(unittest.TestCase):
+
+    def test_format(self):
+        self.assertEqual("m/1(t/2)", user_agent.format_user_agent("m/1", "t/2"))
+        self.assertEqual("m/1(t/2;a;b) ext k/v",
+                         user_agent.format_user_agent("m/1", "t/2", ["a", "", "b"], "ext k/v"))
+
+    def test_rest_user_agent(self):
+        with mock.patch.object(user_agent, "identity", return_value=IDENTITY):
+            self.assertEqual(REST_USER_AGENT, user_agent.rest_user_agent())
+            self.assertEqual(REST_USER_AGENT[:-1] + ";Flink) vvr", user_agent.rest_user_agent(
+                Options({"user-agent.features": " Flink ", "user-agent.extended": "vvr"})))
+            self.assertEqual("MyApp/1.0(python-requests/{})".format(requests.__version__),
+                             user_agent.rest_user_agent(Options({"user-agent.module": "MyApp/1.0"})))
+
+    def test_with_feature_goes_first_once(self):
+        options = Options({"user-agent.features": "Flink PythonPVFS"})
+        user_agent.with_feature(options, "PythonPVFS")
+        user_agent.with_feature(options, "PythonPVFS")
+        self.assertEqual("PythonPVFS Flink", options.to_map()["user-agent.features"])
+
+
+class RestUserAgentTest(unittest.TestCase):
+
+    def setUp(self):
+        self.requests = []
+        recorded = self.requests
+
+        class Handler(BaseHTTPRequestHandler):
+            def do_GET(self):
+                recorded.append(dict(self.headers))
+                body = ({"defaults": {}} if "/config" in self.path
+                        else {"databases": [], "nextPageToken": None})
+                data = json.dumps(body).encode("utf-8")
+                self.send_response(200)
+                self.send_header("Content-Type", "application/json")
+                self.send_header("Content-Length", str(len(data)))
+                self.end_headers()
+                self.wfile.write(data)
+
+            def log_message(self, *args):
+                pass
+
+        self.server = HTTPServer(("127.0.0.1", 0), Handler)
+        threading.Thread(target=self.server.serve_forever, daemon=True).start()
+        self.addCleanup(self.server.server_close)
+        self.addCleanup(self.server.shutdown)
+        self.uri = "http://127.0.0.1:{}".format(self.server.server_port)
+
+        patcher = mock.patch.object(user_agent, "identity", return_value=IDENTITY)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def _options(self, **extra):
+        options = {"uri": self.uri, "warehouse": "wh", "token.provider": "bear", "token": "t"}
+        options.update(extra)
+        return options
+
+    def _user_agents(self):
+        return [headers.get("User-Agent") for headers in self.requests]
+
+    def test_default_user_agent(self):
+        RESTApi(Options(self._options())).list_databases()
+
+        self.assertEqual([REST_USER_AGENT, REST_USER_AGENT], self._user_agents())
+
+    def test_common_options_user_agent(self):
+        options = self._options(**{"user-agent.features": "Flink", "user-agent.extended": "vvr"})
+        RESTApi(Options(options)).list_databases()
+
+        expected = REST_USER_AGENT[:-1] + ";Flink) vvr"
+        self.assertEqual([expected, expected], self._user_agents())
+
+    def test_user_set_user_agent_wins(self):
+        options = self._options(**{"header.User-Agent": "starrocks/user", "user-agent.extended": "vvr"})
+        RESTApi(Options(options)).list_databases()
+
+        self.assertEqual(["starrocks/user", "starrocks/user"], self._user_agents())
+
+    def test_pvfs_user_agent(self):
+        PaimonVirtualFileSystem(self._options(**{"user-agent.features": "Flink"})).ls("pvfs://wh/")
+
+        pvfs_user_agent = REST_USER_AGENT[:-1] + ";PythonPVFS;Flink)"
+        self.assertEqual([pvfs_user_agent, pvfs_user_agent], self._user_agents())
+        for headers in self.requests:
+            self.assertNotIn("http_user_agent", {key.lower() for key in headers})
+
+    def test_token_loader_user_agent(self):
+        HTTPClient().get(self.uri + "/v1/config")
+
+        self.assertEqual([REST_USER_AGENT], self._user_agents())
+
+
+if __name__ == "__main__":
+    unittest.main()