From a1c5445a85dc27fd4fedaa13d57ad073f036de8e Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 22 Oct 2025 22:14:06 -0700 Subject: [PATCH 1/6] [bfops/fix-csharp-quickstart-smoketest]: C# quickstart test properly uses nuget --- .github/workflows/ci.yml | 4 +-- smoketests/tests/quickstart.py | 64 +++++++++++++++++++++++++++++----- 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fae3383572..1ad49418be5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,8 +67,8 @@ jobs: - uses: actions/setup-python@v5 with: { python-version: '3.12' } if: runner.os == 'Windows' - - name: Install psycopg2 - run: python -m pip install psycopg2-binary + - name: Install python deps + run: python -m pip install psycopg2-binary xmltodict - name: Run smoketests # Note: clear_database and replication only work in private run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication diff --git a/smoketests/tests/quickstart.py b/smoketests/tests/quickstart.py index 4b67578f559..edf34ffa1cd 100644 --- a/smoketests/tests/quickstart.py +++ b/smoketests/tests/quickstart.py @@ -3,6 +3,7 @@ import shutil from pathlib import Path import tempfile +import xmltodict import smoketests from .. import Smoketest, STDB_DIR, run_cmd, TEMPLATE_CARGO_TOML @@ -43,17 +44,60 @@ def _parse_quickstart(doc_path: Path, language: str) -> str: # So we could have a different db for each language return "\n".join(blocks).replace("quickstart-chat", f"quickstart-chat-{language}") + end - -def _dotnet_add_package(project_path: Path, package_name: str, source_path: Path): +def load_nuget_config(p: Path): + if p.exists(): + with p.open("rb") as f: + return xmltodict.parse(f.read(), force_list=["add", "packageSource", "package"]) + return {} + +def save_nuget_config(p: Path, doc: dict): + # Write back (pretty, UTF-8, no BOM) + xml = xmltodict.unparse(doc, pretty=True) + p.write_text(xml, encoding="utf-8") + +def add_source(doc: dict, *, key: str, path: str) -> None: + cfg = doc.setdefault("configuration", {}) + sources = cfg.setdefault("packageSources", {}) + source_entries = sources.setdefault("add", []) + source = next((s for s in source_entries if s.get("@key") == key), None) + if source is None: + source = {"@key": key, "@value": path} + source_entries.append(source) + +def add_mapping(doc: dict, *, key: str, pattern: str) -> None: + cfg = doc.setdefault("configuration", {}) + + psm = cfg.setdefault("packageSourceMapping", {}) + mapping_sources = psm.setdefault("packageSource", []) + + # Find or create the target + target = next((s for s in mapping_sources if s.get("@key") == key), None) + if target is None: + target = {"@key": key, "package": []} + mapping_sources.append(target) + + pkgs = target.setdefault("package", []) + + existing = {pkg.get("@pattern") for pkg in pkgs if "@pattern" in pkg} + if pattern not in existing: + pkgs.append({"@pattern": pattern}) + +def _dotnet_add_package(project_path: Path, package_name: str, source_path: Path, build_dir: str): """Add a local NuGet package to a .NET project""" sources = run_cmd("dotnet", "nuget", "list", "source", cwd=project_path, capture_stderr=True) - # Is the source already added? - if package_name in sources: - run_cmd("dotnet", "nuget", "remove", "source", package_name, cwd=project_path, capture_stderr=True) - run_cmd("dotnet", "nuget", "add", "source", source_path, "--name", package_name, cwd=project_path, - capture_stderr=True) + run_cmd("dotnet", "pack", cwd=source_path) + + p = Path(project_path) / "nuget.config" + doc = load_nuget_config(p) + add_source(doc, key=package_name, path=source_path/build_dir) + add_mapping(doc, key=package_name, pattern=package_name) + add_mapping(doc, key="nuget.org", pattern="*") + save_nuget_config(p, doc) + run_cmd("dotnet", "add", "package", package_name, cwd=project_path, capture_stderr=True) + run_cmd("dotnet", "nuget", "locals", "--clear", "all", cwd=project_path, capture_stderr=True) + run_cmd("dotnet", "restore", cwd=project_path, capture_stderr=True) class BaseQuickstart(Smoketest): AUTOPUBLISH = False @@ -246,11 +290,13 @@ def project_init(self, path: Path): run_cmd("dotnet", "new", "console", "--name", "QuickstartChatClient", "--output", path, capture_stderr=True) def sdk_setup(self, path: Path): - _dotnet_add_package(path, "SpacetimeDB.ClientSDK", (STDB_DIR / "sdks/csharp").absolute()) + _dotnet_add_package(STDB_DIR/"sdks/csharp", "SpacetimeDB.BSATN.Runtime", (STDB_DIR / "crates/bindings-csharp/BSATN.Runtime").absolute(), "bin/Release") + _dotnet_add_package(path, "SpacetimeDB.ClientSDK", (STDB_DIR / "sdks/csharp").absolute(), "bin~/Release") def server_postprocess(self, server_path: Path): _dotnet_add_package(server_path, "SpacetimeDB.Runtime", - (STDB_DIR / "crates/bindings-csharp/Runtime").absolute()) + (STDB_DIR / "crates/bindings-csharp/Runtime").absolute(), + "bin/Release") def test_quickstart(self): """Run the C# quickstart guides for server and client.""" From 78e6da92b2594b96be488213bb4c781df4f27a71 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 23 Oct 2025 09:18:14 -0700 Subject: [PATCH 2/6] [bfops/fix-csharp-quickstart-smoketest]: review --- smoketests/tests/quickstart.py | 72 +++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/smoketests/tests/quickstart.py b/smoketests/tests/quickstart.py index edf34ffa1cd..391cf85b87f 100644 --- a/smoketests/tests/quickstart.py +++ b/smoketests/tests/quickstart.py @@ -1,4 +1,5 @@ import logging +import os import re import shutil from pathlib import Path @@ -44,18 +45,17 @@ def _parse_quickstart(doc_path: Path, language: str) -> str: # So we could have a different db for each language return "\n".join(blocks).replace("quickstart-chat", f"quickstart-chat-{language}") + end -def load_nuget_config(p: Path): - if p.exists(): +def load_nuget_config(nuget_path: Path) -> dict: + if nuget_path.exists(): with p.open("rb") as f: return xmltodict.parse(f.read(), force_list=["add", "packageSource", "package"]) return {} -def save_nuget_config(p: Path, doc: dict): - # Write back (pretty, UTF-8, no BOM) +def save_nuget_config(nuget_path: Path, doc: dict) -> None: xml = xmltodict.unparse(doc, pretty=True) - p.write_text(xml, encoding="utf-8") + nuget_path.write_text(xml, encoding="utf-8") -def add_source(doc: dict, *, key: str, path: str) -> None: +def add_nuget_source(doc: dict, *, key: str, path: str) -> None: cfg = doc.setdefault("configuration", {}) sources = cfg.setdefault("packageSources", {}) source_entries = sources.setdefault("add", []) @@ -64,7 +64,7 @@ def add_source(doc: dict, *, key: str, path: str) -> None: source = {"@key": key, "@value": path} source_entries.append(source) -def add_mapping(doc: dict, *, key: str, pattern: str) -> None: +def add_nuget_mapping(doc: dict, *, key: str, pattern: str) -> None: cfg = doc.setdefault("configuration", {}) psm = cfg.setdefault("packageSourceMapping", {}) @@ -82,22 +82,9 @@ def add_mapping(doc: dict, *, key: str, pattern: str) -> None: if pattern not in existing: pkgs.append({"@pattern": pattern}) -def _dotnet_add_package(project_path: Path, package_name: str, source_path: Path, build_dir: str): - """Add a local NuGet package to a .NET project""" - sources = run_cmd("dotnet", "nuget", "list", "source", cwd=project_path, capture_stderr=True) - run_cmd("dotnet", "pack", cwd=source_path) - - p = Path(project_path) / "nuget.config" - doc = load_nuget_config(p) - add_source(doc, key=package_name, path=source_path/build_dir) - add_mapping(doc, key=package_name, pattern=package_name) - add_mapping(doc, key="nuget.org", pattern="*") - save_nuget_config(p, doc) - - run_cmd("dotnet", "add", "package", package_name, cwd=project_path, capture_stderr=True) - - run_cmd("dotnet", "nuget", "locals", "--clear", "all", cwd=project_path, capture_stderr=True) - run_cmd("dotnet", "restore", cwd=project_path, capture_stderr=True) +def override_nuget_package(nuget_config: dict, *, package_name: str, source_path: Path): + add_nuget_source(nuget_config, key=package_name, path=source_path.absolute()) + add_nuget_mapping(nuget_config, key=package_name, pattern=package_name) class BaseQuickstart(Smoketest): AUTOPUBLISH = False @@ -289,14 +276,43 @@ class CSharp(BaseQuickstart): def project_init(self, path: Path): run_cmd("dotnet", "new", "console", "--name", "QuickstartChatClient", "--output", path, capture_stderr=True) + nuget_config_path = Path(self.enterClassContext(tempfile.NamedTemporaryFile())) + print("NuGet config path:", nuget_config_path) + + nuget_config = load_nuget_config(nuget_config_path) + # Other packages should fall back to NuGet + add_nuget_mapping(nuget_config, key="nuget.org", pattern="*") + + # Clear any cached nuget dependencies + run_cmd("dotnet", "nuget", "locals", "--clear", "all", capture_stderr=True) + + run_cmd("dotnet", "pack", cwd=STDB_DIR/"crates/bindings-csharp") + override_nuget_package( + nuget_config, + package_name="SpacetimeDB.BSATN.Runtime", + source_path=(STDB_DIR / "crates/bindings-csharp/BSATN.Runtime/bin/Release"), + ) + override_nuget_package( + nuget_config, + package_name="SpacetimeDB.Runtime", + source_path=(STDB_DIR / "crates/bindings-csharp/Runtime/bin/Release") + ) + + run_cmd("dotnet", "pack", cwd=STDB_DIR/"sdks/csharp") + override_nuget_package( + nuget_config, + package_name="SpacetimeDB.ClientSDK", + source_path=(STDB_DIR / "sdks/csharp/bin~/Release") + ) + + save_nuget_config(nuget_config_path, nuget_config) + os.environ["NUGET_CONFIG_PATH"] = nuget_config_path + def sdk_setup(self, path: Path): - _dotnet_add_package(STDB_DIR/"sdks/csharp", "SpacetimeDB.BSATN.Runtime", (STDB_DIR / "crates/bindings-csharp/BSATN.Runtime").absolute(), "bin/Release") - _dotnet_add_package(path, "SpacetimeDB.ClientSDK", (STDB_DIR / "sdks/csharp").absolute(), "bin~/Release") + run_cmd("dotnet", "add", "package", "SpacetimeDB.ClientSDK", cwd=path, capture_stderr=True) def server_postprocess(self, server_path: Path): - _dotnet_add_package(server_path, "SpacetimeDB.Runtime", - (STDB_DIR / "crates/bindings-csharp/Runtime").absolute(), - "bin/Release") + pass def test_quickstart(self): """Run the C# quickstart guides for server and client.""" From dca33483d3ae251daedafee09716f982fa2db028 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 23 Oct 2025 10:43:58 -0700 Subject: [PATCH 3/6] [bfops/fix-csharp-quickstart-smoketest]: revert --- smoketests/tests/quickstart.py | 83 ++++++++++++++++------------------ 1 file changed, 39 insertions(+), 44 deletions(-) diff --git a/smoketests/tests/quickstart.py b/smoketests/tests/quickstart.py index 391cf85b87f..ddcef8f327e 100644 --- a/smoketests/tests/quickstart.py +++ b/smoketests/tests/quickstart.py @@ -1,5 +1,4 @@ import logging -import os import re import shutil from pathlib import Path @@ -45,26 +44,25 @@ def _parse_quickstart(doc_path: Path, language: str) -> str: # So we could have a different db for each language return "\n".join(blocks).replace("quickstart-chat", f"quickstart-chat-{language}") + end -def load_nuget_config(nuget_path: Path) -> dict: - if nuget_path.exists(): +def load_nuget_config(p: Path): + if p.exists(): with p.open("rb") as f: return xmltodict.parse(f.read(), force_list=["add", "packageSource", "package"]) return {} -def save_nuget_config(nuget_path: Path, doc: dict) -> None: +def save_nuget_config(p: Path, doc: dict): + # Write back (pretty, UTF-8, no BOM) xml = xmltodict.unparse(doc, pretty=True) - nuget_path.write_text(xml, encoding="utf-8") + p.write_text(xml, encoding="utf-8") -def add_nuget_source(doc: dict, *, key: str, path: str) -> None: +def add_source(doc: dict, *, key: str, path: str) -> None: cfg = doc.setdefault("configuration", {}) sources = cfg.setdefault("packageSources", {}) source_entries = sources.setdefault("add", []) - source = next((s for s in source_entries if s.get("@key") == key), None) - if source is None: - source = {"@key": key, "@value": path} - source_entries.append(source) + source = {"@key": key, "@value": path} + source_entries.append(source) -def add_nuget_mapping(doc: dict, *, key: str, pattern: str) -> None: +def add_mapping(doc: dict, *, key: str, pattern: str) -> None: cfg = doc.setdefault("configuration", {}) psm = cfg.setdefault("packageSourceMapping", {}) @@ -82,9 +80,21 @@ def add_nuget_mapping(doc: dict, *, key: str, pattern: str) -> None: if pattern not in existing: pkgs.append({"@pattern": pattern}) -def override_nuget_package(nuget_config: dict, *, package_name: str, source_path: Path): - add_nuget_source(nuget_config, key=package_name, path=source_path.absolute()) - add_nuget_mapping(nuget_config, key=package_name, pattern=package_name) +def override_nuget_package(project_path: Path, package_name: str, source_dir: Path, build_subdir: str): + """Override nuget config to use a local NuGet package on a .NET project""" + # Make sure the local package is built + run_cmd("dotnet", "pack", cwd=source_dir) + + p = Path(project_path) / "nuget.config" + doc = load_nuget_config(p) + add_source(doc, key=package_name, path=source_dir/build_subdir) + add_mapping(doc, key=package_name, pattern=package_name) + # Fallback for other packages + add_mapping(doc, key="nuget.org", pattern="*") + save_nuget_config(p, doc) + + # Clear any caches for nuget packages + run_cmd("dotnet", "nuget", "locals", "--clear", "all", capture_stderr=True) class BaseQuickstart(Smoketest): AUTOPUBLISH = False @@ -276,44 +286,29 @@ class CSharp(BaseQuickstart): def project_init(self, path: Path): run_cmd("dotnet", "new", "console", "--name", "QuickstartChatClient", "--output", path, capture_stderr=True) - nuget_config_path = Path(self.enterClassContext(tempfile.NamedTemporaryFile())) - print("NuGet config path:", nuget_config_path) - - nuget_config = load_nuget_config(nuget_config_path) - # Other packages should fall back to NuGet - add_nuget_mapping(nuget_config, key="nuget.org", pattern="*") - - # Clear any cached nuget dependencies - run_cmd("dotnet", "nuget", "locals", "--clear", "all", capture_stderr=True) - - run_cmd("dotnet", "pack", cwd=STDB_DIR/"crates/bindings-csharp") + def sdk_setup(self, path: Path): override_nuget_package( - nuget_config, - package_name="SpacetimeDB.BSATN.Runtime", - source_path=(STDB_DIR / "crates/bindings-csharp/BSATN.Runtime/bin/Release"), + project_dir=STDB_DIR/"sdks/csharp", + package="SpacetimeDB.BSATN.Runtime", + source_dir=(STDB_DIR / "crates/bindings-csharp/BSATN.Runtime").absolute(), + build_subdir="bin/Release" ) override_nuget_package( - nuget_config, - package_name="SpacetimeDB.Runtime", - source_path=(STDB_DIR / "crates/bindings-csharp/Runtime/bin/Release") + project_dir=path, + package="SpacetimeDB.ClientSDK", + source_dir=(STDB_DIR / "sdks/csharp").absolute(), + build_subdir="bin~/Release" ) + run_cmd("dotnet", "add", "package", "SpacetimeDB.ClientSDK", cwd=project_path, capture_stderr=True) - run_cmd("dotnet", "pack", cwd=STDB_DIR/"sdks/csharp") + def server_postprocess(self, server_path: Path): override_nuget_package( - nuget_config, - package_name="SpacetimeDB.ClientSDK", - source_path=(STDB_DIR / "sdks/csharp/bin~/Release") + project_dir=server_path, + package="SpacetimeDB.Runtime", + source_dir=(STDB_DIR / "crates/bindings-csharp/Runtime").absolute(), + build_subdir="bin/Release" ) - save_nuget_config(nuget_config_path, nuget_config) - os.environ["NUGET_CONFIG_PATH"] = nuget_config_path - - def sdk_setup(self, path: Path): - run_cmd("dotnet", "add", "package", "SpacetimeDB.ClientSDK", cwd=path, capture_stderr=True) - - def server_postprocess(self, server_path: Path): - pass - def test_quickstart(self): """Run the C# quickstart guides for server and client.""" if not smoketests.HAVE_DOTNET: From d560994a69381a9c10ae3a420109cfc63b7183e9 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 23 Oct 2025 10:49:06 -0700 Subject: [PATCH 4/6] [bfops/fix-csharp-quickstart-smoketest]: fix --- smoketests/tests/quickstart.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smoketests/tests/quickstart.py b/smoketests/tests/quickstart.py index ddcef8f327e..5363a494c78 100644 --- a/smoketests/tests/quickstart.py +++ b/smoketests/tests/quickstart.py @@ -80,12 +80,12 @@ def add_mapping(doc: dict, *, key: str, pattern: str) -> None: if pattern not in existing: pkgs.append({"@pattern": pattern}) -def override_nuget_package(project_path: Path, package_name: str, source_dir: Path, build_subdir: str): +def override_nuget_package(project_dir: Path, package_name: str, source_dir: Path, build_subdir: str): """Override nuget config to use a local NuGet package on a .NET project""" # Make sure the local package is built run_cmd("dotnet", "pack", cwd=source_dir) - p = Path(project_path) / "nuget.config" + p = Path(project_dir) / "nuget.config" doc = load_nuget_config(p) add_source(doc, key=package_name, path=source_dir/build_subdir) add_mapping(doc, key=package_name, pattern=package_name) From eff3cbc195944bc82c7f432fd37b47b4d962f380 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 23 Oct 2025 11:01:43 -0700 Subject: [PATCH 5/6] [bfops/fix-csharp-quickstart-smoketest]: fix --- smoketests/tests/quickstart.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/smoketests/tests/quickstart.py b/smoketests/tests/quickstart.py index 5363a494c78..d66c580ee28 100644 --- a/smoketests/tests/quickstart.py +++ b/smoketests/tests/quickstart.py @@ -80,15 +80,15 @@ def add_mapping(doc: dict, *, key: str, pattern: str) -> None: if pattern not in existing: pkgs.append({"@pattern": pattern}) -def override_nuget_package(project_dir: Path, package_name: str, source_dir: Path, build_subdir: str): +def override_nuget_package(*, project_dir: Path, package: str, source_dir: Path, build_subdir: str): """Override nuget config to use a local NuGet package on a .NET project""" # Make sure the local package is built run_cmd("dotnet", "pack", cwd=source_dir) p = Path(project_dir) / "nuget.config" doc = load_nuget_config(p) - add_source(doc, key=package_name, path=source_dir/build_subdir) - add_mapping(doc, key=package_name, pattern=package_name) + add_source(doc, key=package, path=source_dir/build_subdir) + add_mapping(doc, key=package, pattern=package) # Fallback for other packages add_mapping(doc, key="nuget.org", pattern="*") save_nuget_config(p, doc) From 98cd0671947bb790da11b5bec01a89fc5293dc61 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 23 Oct 2025 11:13:28 -0700 Subject: [PATCH 6/6] [bfops/fix-csharp-quickstart-smoketest]: fix --- smoketests/tests/quickstart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smoketests/tests/quickstart.py b/smoketests/tests/quickstart.py index d66c580ee28..5f2bf0ecff8 100644 --- a/smoketests/tests/quickstart.py +++ b/smoketests/tests/quickstart.py @@ -299,7 +299,7 @@ def sdk_setup(self, path: Path): source_dir=(STDB_DIR / "sdks/csharp").absolute(), build_subdir="bin~/Release" ) - run_cmd("dotnet", "add", "package", "SpacetimeDB.ClientSDK", cwd=project_path, capture_stderr=True) + run_cmd("dotnet", "add", "package", "SpacetimeDB.ClientSDK", cwd=path, capture_stderr=True) def server_postprocess(self, server_path: Path): override_nuget_package(