Warn when a standalone pyspark dependency collides with databricks-connect

acceptance/localenv/merge-warnings-json/output.txt

@@ -21,7 +21,7 @@
     "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml",
     "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak",
     "wouldInstallPython": "3.12",
-    "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,20 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n-spark = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n+spark = []\n qa = [\"pandas==4.0.0\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+    \"pyarrow\u003c19\",\n+    \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n"
+    "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,20 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\", \"pyspark\u003e=3.5.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n-spark = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n+spark = []\n qa = [\"pandas==4.0.0\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+    \"pyarrow\u003c19\",\n+    \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n"
   },
   "phases": [
     {
@@ -62,6 +62,10 @@
       "code": "W_DBCONNECT_CONSOLIDATED",
       "message": "databricks-connect \"databricks-connect==15.0.0\" in [dependency-groups].spark conflicts with the environment's \"databricks-connect~=17.2.0\" and is removed; it is managed in \"dev\""
     },
+    {
+      "code": "W_STANDALONE_PYSPARK_CONFLICT",
+      "message": "dependency \"pyspark\u003e=3.5.0\" collides with the pyspark bundled in databricks-connect; the two cannot share one environment — remove it, or install it in a separate environment for local Spark"
+    },
     {
       "code": "W_USER_CONSTRAINT_CONFLICT",
       "message": "dependency \"pyarrow==21.0.0\" conflicts with the environment constraint \"pyarrow\u003c19\""

acceptance/localenv/merge-warnings-json/script

@@ -1,11 +1,12 @@
 # An existing project whose pins the merge overrides, plus a dependency the env's
-# constraints exclude and a pin in a non-dev group uv still locks. All four warning
+# constraints exclude, a pin in a non-dev group uv still locks, and a standalone
+# pyspark that collides with the pyspark databricks-connect vendors. All five warning
 # conditions in one run, so warnings[] is exercised end to end.
 cat > pyproject.toml <<'PY'
 [project]
 name = "demo"
 requires-python = ">=3.10"
-dependencies = ["pyarrow==21.0.0"]
+dependencies = ["pyarrow==21.0.0", "pyspark>=3.5.0"]
 
 [dependency-groups]
 dev = ["databricks-connect~=16.0.0", {include-group = "spark"}]

acceptance/localenv/merge-warnings/output.txt

@@ -3,6 +3,7 @@
 warning: requires-python ">=3.10" is replaced by the environment's ">=3.12"
 warning: databricks-connect "databricks-connect~=16.0.0" is replaced by the environment's "databricks-connect~=17.2.0"
 warning: databricks-connect "databricks-connect==15.0.0" in [dependency-groups].spark conflicts with the environment's "databricks-connect~=17.2.0" and is removed; it is managed in "dev"
+warning: dependency "pyspark>=3.5.0" collides with the pyspark bundled in databricks-connect; the two cannot share one environment — remove it, or install it in a separate environment for local Spark
 warning: dependency "pyarrow==21.0.0" conflicts with the environment constraint "pyarrow<19"
 warning: dependency "pandas==4.0.0" conflicts with the environment constraint "pandas<3"
 Plan: [TEST_TMP_DIR]/pyproject.toml

acceptance/localenv/merge-warnings/script

@@ -4,7 +4,7 @@ cat > pyproject.toml <<'PY'
 [project]
 name = "demo"
 requires-python = ">=3.10"
-dependencies = ["pyarrow==21.0.0"]
+dependencies = ["pyarrow==21.0.0", "pyspark>=3.5.0"]
 
 [dependency-groups]
 dev = ["databricks-connect~=16.0.0", {include-group = "spark"}]

libs/localenv/constraints.go

@@ -292,20 +292,30 @@ func parseConstraints(data []byte) (requiresPython, dbconnect string, deps []str
 // package name: a version specifier, extra, marker, url, or list separator.
 var depNameSepRe = regexp.MustCompile(`[<>=!~;,@\[( \t]`)
 
-// isDatabricksConnectDep reports whether a dependency-group entry is the
-// databricks-connect requirement. It extracts the leading package name (up to
-// the first PEP 508 delimiter) and compares it under PEP 503 normalization, so
-// case, and runs of "-", "_", or "." are all treated as equivalent:
-// "Databricks-Connect", "databricks_connect", and "databricks.connect" all match,
-// while a distinct package like "databricks-connectors" does not.
-func isDatabricksConnectDep(entry string) bool {
+// isDepNamed reports whether a dependency-group entry names the package normalizedName
+// (which the caller passes already PEP 503-normalized). It extracts the leading package
+// name (up to the first PEP 508 delimiter) and compares it under PEP 503 normalization,
+// so case and runs of "-", "_", or "." are all treated as equivalent: for
+// "databricks-connect", "Databricks-Connect", "databricks_connect", and
+// "databricks.connect" all match, while a distinct package like "databricks-connectors"
+// does not.
+func isDepNamed(entry, normalizedName string) bool {
 	name := strings.TrimSpace(entry)
 	if i := depNameSepRe.FindStringIndex(name); i != nil {
 		name = name[:i[0]]
 	}
-	return normalizePackageName(name) == "databricks-connect"
+	return normalizePackageName(name) == normalizedName
 }
 
+// isDatabricksConnectDep reports whether a dependency-group entry is the
+// databricks-connect requirement.
+func isDatabricksConnectDep(entry string) bool { return isDepNamed(entry, "databricks-connect") }
+
+// isPysparkDep reports whether a dependency-group entry is the standalone pyspark
+// requirement. It matches "pyspark" exactly and not a distinct package such as
+// "pyspark-stubs".
+func isPysparkDep(entry string) bool { return isDepNamed(entry, "pyspark") }
+
 // pep503SepRe matches runs of "-", "_", or "." for PEP 503 name normalization.
 var pep503SepRe = regexp.MustCompile(`[-_.]+`)
 

libs/localenv/result.go

@@ -281,6 +281,17 @@ const (
 	// project is no longer set up for — worth surfacing because VS Code and serverless
 	// Jobs read that section as a source of truth.
 	WarnStaleEnvironmentVersion = "W_STALE_ENVIRONMENT_VERSION"
+	// WarnStandalonePysparkConflict: the user declares a standalone pyspark dependency
+	// while the environment installs databricks-connect. databricks-connect vendors its
+	// own pyspark (it ships the pyspark/ package tree rather than depending on the
+	// standalone distribution), so a separately declared pyspark resolves into the same
+	// namespace and the two overwrite each other — the environment then fails to start a
+	// session. This is a coexistence conflict, not a version one, so it is reported
+	// independent of the pyspark version pinned. Emitted whenever databricks-connect
+	// ends up in the resolved environment — whether the env manages it (default mode) or
+	// the user's own pyproject pins it (constraints-only mode) — so it agrees with the
+	// validate hard-fail, which keys on the installed venv rather than the mode.
+	WarnStandalonePysparkConflict = "W_STANDALONE_PYSPARK_CONFLICT"
 )
 
 // Result is the full outcome of a sync run and the root of the --json object

libs/localenv/warnings.go

@@ -170,8 +170,9 @@ func resolutionRequirements(p userPyprojectTOML) []string {
 // pyprojectTOML, so no shape a real pyproject.toml may legitimately carry can
 // suppress the checks that do not depend on it. Greenfield projects (no pre-existing
 // content) produce nothing — there is nothing of the user's to override. Warnings are
-// deterministic and ordered (requires-python, then databricks-connect, then
-// constraint conflicts in the order uv would encounter them) so goldens are stable.
+// deterministic and ordered (requires-python, then databricks-connect, then the
+// standalone-pyspark collision, then constraint conflicts in the order uv would
+// encounter them) so goldens are stable.
 func detectMergeWarnings(userPyproject []byte, c Constraints, plan dbconnectPlan) []Warning {
 	if len(userPyproject) == 0 {
 		return nil
@@ -208,6 +209,16 @@ func detectMergeWarnings(userPyproject []byte, c Constraints, plan dbconnectPlan
 		warnings = append(warnings, dbconnectWarnings(plan, survivors, c.DatabricksConnect)...)
 	}
 
+	// A standalone pyspark collides with databricks-connect's vendored pyspark whenever
+	// databricks-connect ends up in the environment — whether the env manages it (default
+	// mode, c.DatabricksConnect set) or the user's own pyproject pins it (kept as-is in
+	// constraints-only mode, since mergeDatabricksConnect is a no-op on an empty managed
+	// value). Gate on its presence by either route, not on the mode, so this agrees with
+	// the validate hard-fail, which keys on the installed venv rather than the mode.
+	if c.DatabricksConnect != "" || len(dbconnectPins(survivors)) > 0 {
+		warnings = append(warnings, standalonePysparkWarnings(survivors)...)
+	}
+
 	warnings = append(warnings, constraintConflicts(survivors, c.ConstraintDeps)...)
 
 	// A cluster target leaves c.EnvironmentVersion empty and does not manage the
@@ -323,6 +334,26 @@ func dbconnectPins(entries []string) []string {
 	return out
 }
 
+// standalonePysparkWarnings flags a standalone pyspark requirement among reqs. It is a
+// coexistence conflict, not a version one: databricks-connect vendors its own pyspark,
+// so any separately declared pyspark overwrites it in a shared environment regardless
+// of the version pinned. It is therefore reported without inspecting the specifier, and
+// once however many groups declare it — one collision to fix, and repeating it would
+// inflate the code histogram consumers build from warnings[]. The caller gates this on
+// databricks-connect being present in the resolved environment (by either route).
+func standalonePysparkWarnings(reqs []string) []Warning {
+	for _, r := range reqs {
+		if isPysparkDep(r) {
+			return []Warning{{
+				Code: WarnStandalonePysparkConflict,
+				Message: fmt.Sprintf("dependency %q collides with the pyspark bundled in databricks-connect; the two cannot share one environment — remove it, or install it in a separate environment for local Spark",
+					strings.TrimSpace(r)),
+			}}
+		}
+	}
+	return nil
+}
+
 // constraintConflicts flags each user dependency pin that the env's
 // constraint-dependencies also constrains to a provably non-overlapping version.
 // It is deliberately conservative — it only fires when the two ranges are provably

libs/localenv/warnings_test.go

@@ -140,6 +140,72 @@ dependencies = ["pyarrow==21.0.0", "requests>=2.0"]
 	assert.Equal(t, []string{WarnUserConstraintConflict}, codes(detectWarnings(user, tilde)))
 }
 
+func TestDetectMergeWarningsStandalonePysparkConflict(t *testing.T) {
+	// databricks-connect vendors its own pyspark, so a standalone pyspark the user
+	// declares collides with it in a shared environment. The warning fires whenever
+	// the env manages databricks-connect, independent of the pyspark version pinned,
+	// and sits after the databricks-connect warnings but before constraint conflicts.
+	user := []byte(`[project]
+requires-python = "==3.12.*"
+dependencies = ["pyspark>=3.5.0", "pyarrow==21.0.0"]
+
+[dependency-groups]
+dev = ["databricks-connect~=16.1.0"]
+`)
+	c := Constraints{
+		RequiresPython:    "==3.12.*",
+		DatabricksConnect: "databricks-connect~=18.0.0",
+		ConstraintDeps:    []string{"pyarrow<19"},
+	}
+	got := detectWarnings(user, c)
+	assert.Equal(t, []string{
+		WarnDBConnectPinOverridden,
+		WarnStandalonePysparkConflict,
+		WarnUserConstraintConflict,
+	}, codes(got))
+	assert.Contains(t, got[1].Message, "pyspark")
+	assert.Contains(t, got[1].Message, "databricks-connect")
+}
+
+func TestStandalonePysparkIgnoredInConstraintsOnly(t *testing.T) {
+	// In constraints-only mode the env does not install databricks-connect, so there
+	// is no vendored pyspark to collide with — a standalone pyspark is fine.
+	user := []byte(`[project]
+requires-python = "==3.12.*"
+dependencies = ["pyspark>=3.5.0"]
+`)
+	c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: ""}
+	assert.Empty(t, detectWarnings(user, c))
+}
+
+func TestStandalonePysparkReportedOnce(t *testing.T) {
+	// pyspark declared in both [project].dependencies and a dependency group is one
+	// collision to fix, reported once so it does not inflate the code histogram.
+	user := []byte(`[project]
+requires-python = "==3.12.*"
+dependencies = ["PySpark == 4.2.0"]
+
+[dependency-groups]
+dev = ["pyspark", "databricks-connect~=18.0.0"]
+`)
+	c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: "databricks-connect~=18.0.0"}
+	assert.Equal(t, []string{WarnStandalonePysparkConflict}, codes(detectWarnings(user, c)))
+}
+
+func TestStandalonePysparkFiresInConstraintsOnlyWhenUserPinsDBConnect(t *testing.T) {
+	// Constraints-only mode does not manage databricks-connect, but if the user's own
+	// pyproject pins it the merge leaves that pin in place (mergeDatabricksConnect is a
+	// no-op on an empty managed value), so uv still installs databricks-connect and a
+	// standalone pyspark still collides. The warning must fire — matching the validate
+	// hard-fail, which keys on the installed venv rather than the mode.
+	user := []byte(`[project]
+requires-python = "==3.12.*"
+dependencies = ["pyspark>=3.5.0", "databricks-connect~=18.0.0"]
+`)
+	c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: ""}
+	assert.Equal(t, []string{WarnStandalonePysparkConflict}, codes(detectWarnings(user, c)))
+}
+
 func TestDetectMergeWarningsNoConflictWhenCompatible(t *testing.T) {
 	user := []byte(`[project]
 requires-python = "==3.12.*"