From 366e5d6e6f3b5c1af2c14b6f2d61b19843ad2dd3 Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Fri, 14 Aug 2026 15:46:34 +0200 Subject: [PATCH 1/2] Warn when a standalone pyspark dep collides with databricks-connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit *Why* databricks-connect vendors its own pyspark (it ships the pyspark/ package tree rather than depending on the standalone distribution). A project that also declares a standalone pyspark dependency resolves it into the same namespace, the two overwrite each other, and the environment then fails to start a session — surfacing to users as a cryptic Java or protobuf gencode error long after setup reported success. The merge-warning system compared user pins against env constraints only, and the env deliberately does not pin pyspark, so this collision was never flagged. *What* - Add warning code W_STANDALONE_PYSPARK_CONFLICT. - isPysparkDep: PEP 503 name match for the standalone pyspark requirement. - standalonePysparkWarnings: report the collision once, independent of the pinned version (it is a coexistence conflict, not a version one), emitted only when the env manages databricks-connect. - Wire it into detectMergeWarnings between the databricks-connect warnings and the constraint conflicts. *Verification* - go test ./libs/localenv/ (unit + regenerated acceptance goldens) passes. - go vet ./libs/localenv/ clean. Co-authored-by: Isaac --- .../localenv/merge-warnings-json/output.txt | 6 ++- .../localenv/merge-warnings-json/script | 5 +- acceptance/localenv/merge-warnings/output.txt | 1 + acceptance/localenv/merge-warnings/script | 2 +- libs/localenv/constraints.go | 12 +++++ libs/localenv/result.go | 9 ++++ libs/localenv/warnings.go | 29 ++++++++++- libs/localenv/warnings_test.go | 52 +++++++++++++++++++ 8 files changed, 110 insertions(+), 6 deletions(-) diff --git a/acceptance/localenv/merge-warnings-json/output.txt b/acceptance/localenv/merge-warnings-json/output.txt index 2f11282d105..c5d7333eb4e 100644 --- a/acceptance/localenv/merge-warnings-json/output.txt +++ b/acceptance/localenv/merge-warnings-json/output.txt @@ -21,7 +21,7 @@ "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,20 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n-spark = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n+spark = []\n qa = [\"pandas==4.0.0\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,20 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\", \"pyspark\u003e=3.5.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n-spark = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n+spark = []\n qa = [\"pandas==4.0.0\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { @@ -62,6 +62,10 @@ "code": "W_DBCONNECT_CONSOLIDATED", "message": "databricks-connect \"databricks-connect==15.0.0\" in [dependency-groups].spark conflicts with the environment's \"databricks-connect~=17.2.0\" and is removed; it is managed in \"dev\"" }, + { + "code": "W_STANDALONE_PYSPARK_CONFLICT", + "message": "dependency \"pyspark\u003e=3.5.0\" collides with the pyspark bundled in databricks-connect; the two cannot share one environment — remove it, or install it in a separate environment for local Spark" + }, { "code": "W_USER_CONSTRAINT_CONFLICT", "message": "dependency \"pyarrow==21.0.0\" conflicts with the environment constraint \"pyarrow\u003c19\"" diff --git a/acceptance/localenv/merge-warnings-json/script b/acceptance/localenv/merge-warnings-json/script index 303d347527c..f6e4cbc2393 100644 --- a/acceptance/localenv/merge-warnings-json/script +++ b/acceptance/localenv/merge-warnings-json/script @@ -1,11 +1,12 @@ # An existing project whose pins the merge overrides, plus a dependency the env's -# constraints exclude and a pin in a non-dev group uv still locks. All four warning +# constraints exclude, a pin in a non-dev group uv still locks, and a standalone +# pyspark that collides with the pyspark databricks-connect vendors. All five warning # conditions in one run, so warnings[] is exercised end to end. cat > pyproject.toml <<'PY' [project] name = "demo" requires-python = ">=3.10" -dependencies = ["pyarrow==21.0.0"] +dependencies = ["pyarrow==21.0.0", "pyspark>=3.5.0"] [dependency-groups] dev = ["databricks-connect~=16.0.0", {include-group = "spark"}] diff --git a/acceptance/localenv/merge-warnings/output.txt b/acceptance/localenv/merge-warnings/output.txt index 67372978cb8..45879bd6f62 100644 --- a/acceptance/localenv/merge-warnings/output.txt +++ b/acceptance/localenv/merge-warnings/output.txt @@ -3,6 +3,7 @@ warning: requires-python ">=3.10" is replaced by the environment's ">=3.12" warning: databricks-connect "databricks-connect~=16.0.0" is replaced by the environment's "databricks-connect~=17.2.0" warning: databricks-connect "databricks-connect==15.0.0" in [dependency-groups].spark conflicts with the environment's "databricks-connect~=17.2.0" and is removed; it is managed in "dev" +warning: dependency "pyspark>=3.5.0" collides with the pyspark bundled in databricks-connect; the two cannot share one environment — remove it, or install it in a separate environment for local Spark warning: dependency "pyarrow==21.0.0" conflicts with the environment constraint "pyarrow<19" warning: dependency "pandas==4.0.0" conflicts with the environment constraint "pandas<3" Plan: [TEST_TMP_DIR]/pyproject.toml diff --git a/acceptance/localenv/merge-warnings/script b/acceptance/localenv/merge-warnings/script index e1f60b96709..51a86a67d48 100644 --- a/acceptance/localenv/merge-warnings/script +++ b/acceptance/localenv/merge-warnings/script @@ -4,7 +4,7 @@ cat > pyproject.toml <<'PY' [project] name = "demo" requires-python = ">=3.10" -dependencies = ["pyarrow==21.0.0"] +dependencies = ["pyarrow==21.0.0", "pyspark>=3.5.0"] [dependency-groups] dev = ["databricks-connect~=16.0.0", {include-group = "spark"}] diff --git a/libs/localenv/constraints.go b/libs/localenv/constraints.go index bee8ba723e0..33dcb788b66 100644 --- a/libs/localenv/constraints.go +++ b/libs/localenv/constraints.go @@ -306,6 +306,18 @@ func isDatabricksConnectDep(entry string) bool { return normalizePackageName(name) == "databricks-connect" } +// isPysparkDep reports whether a dependency-group entry is the standalone pyspark +// requirement, extracting the leading package name and comparing it under the same +// PEP 503 normalization as isDatabricksConnectDep. It matches "pyspark" exactly and +// not a distinct package such as "pyspark-stubs". +func isPysparkDep(entry string) bool { + name := strings.TrimSpace(entry) + if i := depNameSepRe.FindStringIndex(name); i != nil { + name = name[:i[0]] + } + return normalizePackageName(name) == "pyspark" +} + // pep503SepRe matches runs of "-", "_", or "." for PEP 503 name normalization. var pep503SepRe = regexp.MustCompile(`[-_.]+`) diff --git a/libs/localenv/result.go b/libs/localenv/result.go index 99266128abc..7c36113332a 100644 --- a/libs/localenv/result.go +++ b/libs/localenv/result.go @@ -281,6 +281,15 @@ const ( // project is no longer set up for — worth surfacing because VS Code and serverless // Jobs read that section as a source of truth. WarnStaleEnvironmentVersion = "W_STALE_ENVIRONMENT_VERSION" + // WarnStandalonePysparkConflict: the user declares a standalone pyspark dependency + // while the environment installs databricks-connect. databricks-connect vendors its + // own pyspark (it ships the pyspark/ package tree rather than depending on the + // standalone distribution), so a separately declared pyspark resolves into the same + // namespace and the two overwrite each other — the environment then fails to start a + // session. This is a coexistence conflict, not a version one, so it is reported + // independent of the pyspark version pinned. Only in default mode, where the env + // manages databricks-connect and thus owns the vendored pyspark. + WarnStandalonePysparkConflict = "W_STANDALONE_PYSPARK_CONFLICT" ) // Result is the full outcome of a sync run and the root of the --json object diff --git a/libs/localenv/warnings.go b/libs/localenv/warnings.go index 5e1c15f2203..642a8146688 100644 --- a/libs/localenv/warnings.go +++ b/libs/localenv/warnings.go @@ -170,8 +170,9 @@ func resolutionRequirements(p userPyprojectTOML) []string { // pyprojectTOML, so no shape a real pyproject.toml may legitimately carry can // suppress the checks that do not depend on it. Greenfield projects (no pre-existing // content) produce nothing — there is nothing of the user's to override. Warnings are -// deterministic and ordered (requires-python, then databricks-connect, then -// constraint conflicts in the order uv would encounter them) so goldens are stable. +// deterministic and ordered (requires-python, then databricks-connect, then the +// standalone-pyspark collision, then constraint conflicts in the order uv would +// encounter them) so goldens are stable. func detectMergeWarnings(userPyproject []byte, c Constraints, plan dbconnectPlan) []Warning { if len(userPyproject) == 0 { return nil @@ -206,6 +207,10 @@ func detectMergeWarnings(userPyproject []byte, c Constraints, plan dbconnectPlan // databricks-connect is left untouched). if c.DatabricksConnect != "" { warnings = append(warnings, dbconnectWarnings(plan, survivors, c.DatabricksConnect)...) + // Gated on the env managing databricks-connect: only then does it install the + // vendored pyspark a standalone one would collide with. In constraints-only mode + // the env installs no databricks-connect, so a standalone pyspark is harmless. + warnings = append(warnings, standalonePysparkWarnings(survivors)...) } warnings = append(warnings, constraintConflicts(survivors, c.ConstraintDeps)...) @@ -323,6 +328,26 @@ func dbconnectPins(entries []string) []string { return out } +// standalonePysparkWarnings flags a standalone pyspark requirement among reqs. It is a +// coexistence conflict, not a version one: databricks-connect vendors its own pyspark, +// so any separately declared pyspark overwrites it in a shared environment regardless +// of the version pinned. It is therefore reported without inspecting the specifier, and +// once however many groups declare it — one collision to fix, and repeating it would +// inflate the code histogram consumers build from warnings[]. The caller gates this on +// the env managing databricks-connect. +func standalonePysparkWarnings(reqs []string) []Warning { + for _, r := range reqs { + if isPysparkDep(r) { + return []Warning{{ + Code: WarnStandalonePysparkConflict, + Message: fmt.Sprintf("dependency %q collides with the pyspark bundled in databricks-connect; the two cannot share one environment — remove it, or install it in a separate environment for local Spark", + strings.TrimSpace(r)), + }} + } + } + return nil +} + // constraintConflicts flags each user dependency pin that the env's // constraint-dependencies also constrains to a provably non-overlapping version. // It is deliberately conservative — it only fires when the two ranges are provably diff --git a/libs/localenv/warnings_test.go b/libs/localenv/warnings_test.go index d2f4c07cfb7..7566bd201b5 100644 --- a/libs/localenv/warnings_test.go +++ b/libs/localenv/warnings_test.go @@ -140,6 +140,58 @@ dependencies = ["pyarrow==21.0.0", "requests>=2.0"] assert.Equal(t, []string{WarnUserConstraintConflict}, codes(detectWarnings(user, tilde))) } +func TestDetectMergeWarningsStandalonePysparkConflict(t *testing.T) { + // databricks-connect vendors its own pyspark, so a standalone pyspark the user + // declares collides with it in a shared environment. The warning fires whenever + // the env manages databricks-connect, independent of the pyspark version pinned, + // and sits after the databricks-connect warnings but before constraint conflicts. + user := []byte(`[project] +requires-python = "==3.12.*" +dependencies = ["pyspark>=3.5.0", "pyarrow==21.0.0"] + +[dependency-groups] +dev = ["databricks-connect~=16.1.0"] +`) + c := Constraints{ + RequiresPython: "==3.12.*", + DatabricksConnect: "databricks-connect~=18.0.0", + ConstraintDeps: []string{"pyarrow<19"}, + } + got := detectWarnings(user, c) + assert.Equal(t, []string{ + WarnDBConnectPinOverridden, + WarnStandalonePysparkConflict, + WarnUserConstraintConflict, + }, codes(got)) + assert.Contains(t, got[1].Message, "pyspark") + assert.Contains(t, got[1].Message, "databricks-connect") +} + +func TestStandalonePysparkIgnoredInConstraintsOnly(t *testing.T) { + // In constraints-only mode the env does not install databricks-connect, so there + // is no vendored pyspark to collide with — a standalone pyspark is fine. + user := []byte(`[project] +requires-python = "==3.12.*" +dependencies = ["pyspark>=3.5.0"] +`) + c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: ""} + assert.Empty(t, detectWarnings(user, c)) +} + +func TestStandalonePysparkReportedOnce(t *testing.T) { + // pyspark declared in both [project].dependencies and a dependency group is one + // collision to fix, reported once so it does not inflate the code histogram. + user := []byte(`[project] +requires-python = "==3.12.*" +dependencies = ["PySpark == 4.2.0"] + +[dependency-groups] +dev = ["pyspark", "databricks-connect~=18.0.0"] +`) + c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: "databricks-connect~=18.0.0"} + assert.Equal(t, []string{WarnStandalonePysparkConflict}, codes(detectWarnings(user, c))) +} + func TestDetectMergeWarningsNoConflictWhenCompatible(t *testing.T) { user := []byte(`[project] requires-python = "==3.12.*" From 34667bebae47a9bedbfdb03b7b90939bfba6fca2 Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Fri, 14 Aug 2026 18:37:32 +0200 Subject: [PATCH 2/2] Fire the pyspark warning whenever databricks-connect is present MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit *Why* Review feedback: the warning gated on the mode (c.DatabricksConnect != ""), but the validate hard-fail gates on the installed venv. That disagreement is reachable — in constraints-only mode a project that pins databricks-connect itself keeps that pin (mergeDatabricksConnect is a no-op on an empty managed value, merge.go), so uv installs it, yet the warning stayed silent while validate would hard-fail. The user got no warning at merge and then an unexplained failure at validate. Key the warning on databricks-connect being present by any route so the two agree. *What* - Gate standalonePysparkWarnings on `c.DatabricksConnect != "" || len(dbconnectPins(survivors)) > 0` instead of the mode alone. - Dedupe isDatabricksConnectDep / isPysparkDep into one isDepNamed(entry, name) with both as thin wrappers, removing the duplicated PEP 508/503 name logic. - Update the warning-code and function docs to describe the by-either-route gate. - Add a constraints-only + user-pinned-databricks-connect + pyspark test. *Verification* - go test ./libs/localenv/ and the merge-warnings acceptance goldens pass. - go vet ./libs/localenv/ clean. Co-authored-by: Isaac --- libs/localenv/constraints.go | 34 ++++++++++++++++------------------ libs/localenv/result.go | 6 ++++-- libs/localenv/warnings.go | 14 ++++++++++---- libs/localenv/warnings_test.go | 14 ++++++++++++++ 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/libs/localenv/constraints.go b/libs/localenv/constraints.go index 33dcb788b66..36357facddb 100644 --- a/libs/localenv/constraints.go +++ b/libs/localenv/constraints.go @@ -292,31 +292,29 @@ func parseConstraints(data []byte) (requiresPython, dbconnect string, deps []str // package name: a version specifier, extra, marker, url, or list separator. var depNameSepRe = regexp.MustCompile(`[<>=!~;,@\[( \t]`) -// isDatabricksConnectDep reports whether a dependency-group entry is the -// databricks-connect requirement. It extracts the leading package name (up to -// the first PEP 508 delimiter) and compares it under PEP 503 normalization, so -// case, and runs of "-", "_", or "." are all treated as equivalent: -// "Databricks-Connect", "databricks_connect", and "databricks.connect" all match, -// while a distinct package like "databricks-connectors" does not. -func isDatabricksConnectDep(entry string) bool { +// isDepNamed reports whether a dependency-group entry names the package normalizedName +// (which the caller passes already PEP 503-normalized). It extracts the leading package +// name (up to the first PEP 508 delimiter) and compares it under PEP 503 normalization, +// so case and runs of "-", "_", or "." are all treated as equivalent: for +// "databricks-connect", "Databricks-Connect", "databricks_connect", and +// "databricks.connect" all match, while a distinct package like "databricks-connectors" +// does not. +func isDepNamed(entry, normalizedName string) bool { name := strings.TrimSpace(entry) if i := depNameSepRe.FindStringIndex(name); i != nil { name = name[:i[0]] } - return normalizePackageName(name) == "databricks-connect" + return normalizePackageName(name) == normalizedName } +// isDatabricksConnectDep reports whether a dependency-group entry is the +// databricks-connect requirement. +func isDatabricksConnectDep(entry string) bool { return isDepNamed(entry, "databricks-connect") } + // isPysparkDep reports whether a dependency-group entry is the standalone pyspark -// requirement, extracting the leading package name and comparing it under the same -// PEP 503 normalization as isDatabricksConnectDep. It matches "pyspark" exactly and -// not a distinct package such as "pyspark-stubs". -func isPysparkDep(entry string) bool { - name := strings.TrimSpace(entry) - if i := depNameSepRe.FindStringIndex(name); i != nil { - name = name[:i[0]] - } - return normalizePackageName(name) == "pyspark" -} +// requirement. It matches "pyspark" exactly and not a distinct package such as +// "pyspark-stubs". +func isPysparkDep(entry string) bool { return isDepNamed(entry, "pyspark") } // pep503SepRe matches runs of "-", "_", or "." for PEP 503 name normalization. var pep503SepRe = regexp.MustCompile(`[-_.]+`) diff --git a/libs/localenv/result.go b/libs/localenv/result.go index 7c36113332a..236d845d685 100644 --- a/libs/localenv/result.go +++ b/libs/localenv/result.go @@ -287,8 +287,10 @@ const ( // standalone distribution), so a separately declared pyspark resolves into the same // namespace and the two overwrite each other — the environment then fails to start a // session. This is a coexistence conflict, not a version one, so it is reported - // independent of the pyspark version pinned. Only in default mode, where the env - // manages databricks-connect and thus owns the vendored pyspark. + // independent of the pyspark version pinned. Emitted whenever databricks-connect + // ends up in the resolved environment — whether the env manages it (default mode) or + // the user's own pyproject pins it (constraints-only mode) — so it agrees with the + // validate hard-fail, which keys on the installed venv rather than the mode. WarnStandalonePysparkConflict = "W_STANDALONE_PYSPARK_CONFLICT" ) diff --git a/libs/localenv/warnings.go b/libs/localenv/warnings.go index 642a8146688..1767efb3242 100644 --- a/libs/localenv/warnings.go +++ b/libs/localenv/warnings.go @@ -207,9 +207,15 @@ func detectMergeWarnings(userPyproject []byte, c Constraints, plan dbconnectPlan // databricks-connect is left untouched). if c.DatabricksConnect != "" { warnings = append(warnings, dbconnectWarnings(plan, survivors, c.DatabricksConnect)...) - // Gated on the env managing databricks-connect: only then does it install the - // vendored pyspark a standalone one would collide with. In constraints-only mode - // the env installs no databricks-connect, so a standalone pyspark is harmless. + } + + // A standalone pyspark collides with databricks-connect's vendored pyspark whenever + // databricks-connect ends up in the environment — whether the env manages it (default + // mode, c.DatabricksConnect set) or the user's own pyproject pins it (kept as-is in + // constraints-only mode, since mergeDatabricksConnect is a no-op on an empty managed + // value). Gate on its presence by either route, not on the mode, so this agrees with + // the validate hard-fail, which keys on the installed venv rather than the mode. + if c.DatabricksConnect != "" || len(dbconnectPins(survivors)) > 0 { warnings = append(warnings, standalonePysparkWarnings(survivors)...) } @@ -334,7 +340,7 @@ func dbconnectPins(entries []string) []string { // of the version pinned. It is therefore reported without inspecting the specifier, and // once however many groups declare it — one collision to fix, and repeating it would // inflate the code histogram consumers build from warnings[]. The caller gates this on -// the env managing databricks-connect. +// databricks-connect being present in the resolved environment (by either route). func standalonePysparkWarnings(reqs []string) []Warning { for _, r := range reqs { if isPysparkDep(r) { diff --git a/libs/localenv/warnings_test.go b/libs/localenv/warnings_test.go index 7566bd201b5..eaa948f01b2 100644 --- a/libs/localenv/warnings_test.go +++ b/libs/localenv/warnings_test.go @@ -192,6 +192,20 @@ dev = ["pyspark", "databricks-connect~=18.0.0"] assert.Equal(t, []string{WarnStandalonePysparkConflict}, codes(detectWarnings(user, c))) } +func TestStandalonePysparkFiresInConstraintsOnlyWhenUserPinsDBConnect(t *testing.T) { + // Constraints-only mode does not manage databricks-connect, but if the user's own + // pyproject pins it the merge leaves that pin in place (mergeDatabricksConnect is a + // no-op on an empty managed value), so uv still installs databricks-connect and a + // standalone pyspark still collides. The warning must fire — matching the validate + // hard-fail, which keys on the installed venv rather than the mode. + user := []byte(`[project] +requires-python = "==3.12.*" +dependencies = ["pyspark>=3.5.0", "databricks-connect~=18.0.0"] +`) + c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: ""} + assert.Equal(t, []string{WarnStandalonePysparkConflict}, codes(detectWarnings(user, c))) +} + func TestDetectMergeWarningsNoConflictWhenCompatible(t *testing.T) { user := []byte(`[project] requires-python = "==3.12.*"