py/envoy.base.utils: Fix inventory docs url

py/envoy.base.utils/envoy/base/utils/abstract/project/inventory.py

@@ -1,4 +1,5 @@
 
+import os
 import pathlib
 import types
 from collections.abc import Iterator
@@ -18,7 +19,11 @@
 INVENTORY_PATH_GLOB = "docs/inventories/v*.*/objects.inv"
 INVENTORY_PATH_FMT = "docs/inventories/v{minor_version}/objects.inv"
 INVENTORY_VERSIONS_PATH = "docs/versions.yaml"
-INVENTORY_URL_FMT = (
+INVENTORY_BASE_URL_DEFAULT = (
+    "https://storage.googleapis.com/envoy-docs-archive")
+INVENTORY_BASE_URL_ENV = "ENVOY_DOCS_ARCHIVE_URL"
+INVENTORY_URL_FMT = "{base_url}/envoy/docs/v{version}/objects.inv"
+INVENTORY_LEGACY_URL_FMT = (
     "https://github.com/envoyproxy/archive/raw/main/"
     "docs/envoy/v{version}/objects.inv")
 
@@ -46,6 +51,12 @@ def inventories(self) -> typing.InventoryDict:
             for path
             in self.paths}
 
+    @property
+    def inventory_base_url(self) -> str:
+        return os.environ.get(
+            INVENTORY_BASE_URL_ENV,
+            INVENTORY_BASE_URL_DEFAULT).rstrip("/")
+
     @property
     def paths(self) -> Iterator[pathlib.Path]:
         return self.project.path.glob(INVENTORY_PATH_GLOB)
@@ -94,17 +105,22 @@ def changes_for_commit(self, change: typing.ProjectChangeDict) -> set[str]:
         return changed
 
     async def fetch(self, version: _version.Version) -> bytes | None:
-        response = await self.project.session.get(self.inventory_url(version))
-        return (
-            await response.read()
-            if response.status != 404
-            else None)
+        for url in (self.inventory_url, self.legacy_inventory_url):
+            response = await self.project.session.get(url(version))
+            if response.status != 404:
+                return await response.read()
+        return None
 
     def inventory_path(self, version: _version.Version) -> pathlib.Path:
         return self.project.path.joinpath(self.rel_inventory_path(version))
 
     def inventory_url(self, version: _version.Version) -> str:
-        return INVENTORY_URL_FMT.format(version=version.base_version)
+        return INVENTORY_URL_FMT.format(
+            base_url=self.inventory_base_url,
+            version=version.base_version)
+
+    def legacy_inventory_url(self, version: _version.Version) -> str:
+        return INVENTORY_LEGACY_URL_FMT.format(version=version.base_version)
 
     def rel_inventory_path(self, version: _version.Version) -> str:
         return INVENTORY_PATH_FMT.format(

py/envoy.base.utils/tests/test_abstract_project_inventories.py

@@ -306,38 +306,58 @@ def test_abstract_inventories_changes(patches, items, sync):
 
 
 @pytest.mark.parametrize("response", [None, 404, "OTHER"])
-async def test_abstract_inventories_fetch(patches, response):
+@pytest.mark.parametrize("legacy_response", [None, 404, "OTHER"])
+async def test_abstract_inventories_fetch(
+        patches, response, legacy_response):
     project = MagicMock()
     inventories = DummyInventories(project)
     patched = patches(
         "AInventories.inventory_url",
+        "AInventories.legacy_inventory_url",
         prefix="envoy.base.utils.abstract.project.inventory")
     version = MagicMock()
-    get = AsyncMock()
-    read = AsyncMock()
-    get.return_value.read.side_effect = read
-    get.return_value.status = response
+    primary = MagicMock()
+    primary.status = response
+    primary.read = AsyncMock()
+    legacy = MagicMock()
+    legacy.status = legacy_response
+    legacy.read = AsyncMock()
+    get = AsyncMock(side_effect=[primary, legacy])
     project.session.get.side_effect = get
 
-    with patched as (m_url, ):
-        assert (
-            await inventories.fetch(version)
-            == (read.return_value
-                if response != 404
-                else None))
+    with patched as (m_url, m_legacy):
+        result = await inventories.fetch(version)
 
-    assert (
-        get.call_args
-        == [(m_url.return_value, ), {}])
     assert (
         m_url.call_args
         == [(version, ), {}])
-    if response == 404:
-        assert not read.called
-    else:
+    if response != 404:
+        assert result == primary.read.return_value
         assert (
-            read.call_args
+            get.call_args_list
+            == [[(m_url.return_value, ), {}]])
+        assert (
+            primary.read.call_args
             == [(), {}])
+        assert not m_legacy.called
+        assert not legacy.read.called
+        return
+    assert not primary.read.called
+    assert (
+        get.call_args_list
+        == [[(m_url.return_value, ), {}],
+            [(m_legacy.return_value, ), {}]])
+    assert (
+        m_legacy.call_args
+        == [(version, ), {}])
+    if legacy_response == 404:
+        assert result is None
+        assert not legacy.read.called
+        return
+    assert result == legacy.read.return_value
+    assert (
+        legacy.read.call_args
+        == [(), {}])
 
 
 def test_abstract_inventories_inventory_path(patches):
@@ -365,13 +385,71 @@ def test_abstract_inventories_inventory_url(patches):
     inventories = DummyInventories(project)
     patched = patches(
         "INVENTORY_URL_FMT",
+        ("AInventories.inventory_base_url",
+         dict(new_callable=PropertyMock)),
         prefix="envoy.base.utils.abstract.project.inventory")
     version = MagicMock()
 
-    with patched as (m_tpl, ):
+    with patched as (m_tpl, m_base):
         assert (
             inventories.inventory_url(version)
             == m_tpl.format.return_value)
+    assert (
+        m_tpl.format.call_args
+        == [(),
+            dict(base_url=m_base.return_value,
+                 version=version.base_version)])
+
+
+@pytest.mark.parametrize(
+    "env",
+    [None,
+     "https://example.com/bucket",
+     "https://example.com/bucket/",
+     "https://example.com/bucket//"])
+def test_abstract_inventories_inventory_base_url(monkeypatch, env):
+    inventories = DummyInventories("PROJECT")
+    inventory = abstract.project.inventory
+    if env is None:
+        monkeypatch.delenv(inventory.INVENTORY_BASE_URL_ENV, raising=False)
+    else:
+        monkeypatch.setenv(inventory.INVENTORY_BASE_URL_ENV, env)
+    assert (
+        inventories.inventory_base_url
+        == (inventory.INVENTORY_BASE_URL_DEFAULT
+            if env is None
+            else "https://example.com/bucket"))
+    assert "inventory_base_url" not in inventories.__dict__
+
+
+def test_abstract_inventories_inventory_url_default(monkeypatch):
+    inventories = DummyInventories("PROJECT")
+    inventory = abstract.project.inventory
+    monkeypatch.delenv(inventory.INVENTORY_BASE_URL_ENV, raising=False)
+    version = MagicMock()
+    version.base_version = "1.36.3"
+    assert (
+        inventories.inventory_url(version)
+        == ("https://storage.googleapis.com/envoy-docs-archive/"
+            "envoy/docs/v1.36.3/objects.inv"))
+    assert (
+        inventories.legacy_inventory_url(version)
+        == ("https://github.com/envoyproxy/archive/raw/main/"
+            "docs/envoy/v1.36.3/objects.inv"))
+
+
+def test_abstract_inventories_legacy_inventory_url(patches):
+    project = MagicMock()
+    inventories = DummyInventories(project)
+    patched = patches(
+        "INVENTORY_LEGACY_URL_FMT",
+        prefix="envoy.base.utils.abstract.project.inventory")
+    version = MagicMock()
+
+    with patched as (m_tpl, ):
+        assert (
+            inventories.legacy_inventory_url(version)
+            == m_tpl.format.return_value)
     assert (
         m_tpl.format.call_args
         == [(), dict(version=version.base_version)])