From b44de12dcc220085d7f3f7cce6b96482d931c1cf Mon Sep 17 00:00:00 2001 From: Art Chen Date: Wed, 10 Jun 2026 18:06:25 -0700 Subject: [PATCH 1/2] Preserve `.flow.ipynb` suffix when translating notebook task paths --- .../bundle/paths/flow_notebook/databricks.yml | 14 +++ .../bundle/paths/flow_notebook/output.txt | 10 ++ acceptance/bundle/paths/flow_notebook/script | 1 + .../paths/flow_notebook/src/regular.ipynb | 10 ++ .../paths/flow_notebook/src/test.flow.ipynb | 10 ++ bundle/config/mutator/translate_paths_test.go | 92 +++++++++++++++++++ libs/notebook/ext.go | 10 +- libs/notebook/ext_test.go | 4 + 8 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/paths/flow_notebook/databricks.yml create mode 100644 acceptance/bundle/paths/flow_notebook/output.txt create mode 100644 acceptance/bundle/paths/flow_notebook/script create mode 100644 acceptance/bundle/paths/flow_notebook/src/regular.ipynb create mode 100644 acceptance/bundle/paths/flow_notebook/src/test.flow.ipynb diff --git a/acceptance/bundle/paths/flow_notebook/databricks.yml b/acceptance/bundle/paths/flow_notebook/databricks.yml new file mode 100644 index 00000000000..673c834a4fd --- /dev/null +++ b/acceptance/bundle/paths/flow_notebook/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: flow_paths + +resources: + jobs: + flow_job: + name: flow_job + tasks: + - task_key: flow_task + notebook_task: + notebook_path: ./src/test.flow.ipynb + - task_key: regular_task + notebook_task: + notebook_path: ./src/regular.ipynb diff --git a/acceptance/bundle/paths/flow_notebook/output.txt b/acceptance/bundle/paths/flow_notebook/output.txt new file mode 100644 index 00000000000..933ea06ed68 --- /dev/null +++ b/acceptance/bundle/paths/flow_notebook/output.txt @@ -0,0 +1,10 @@ +[ + { + "task_key": "flow_task", + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/flow_paths/default/files/src/test.flow.ipynb" + }, + { + "task_key": "regular_task", + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/flow_paths/default/files/src/regular" + } +] diff --git a/acceptance/bundle/paths/flow_notebook/script b/acceptance/bundle/paths/flow_notebook/script new file mode 100644 index 00000000000..8384a3822b7 --- /dev/null +++ b/acceptance/bundle/paths/flow_notebook/script @@ -0,0 +1 @@ +$CLI bundle validate -o json | jq '.resources.jobs.flow_job.tasks | map({task_key, notebook_path: .notebook_task.notebook_path})' diff --git a/acceptance/bundle/paths/flow_notebook/src/regular.ipynb b/acceptance/bundle/paths/flow_notebook/src/regular.ipynb new file mode 100644 index 00000000000..1ec2d26c287 --- /dev/null +++ b/acceptance/bundle/paths/flow_notebook/src/regular.ipynb @@ -0,0 +1,10 @@ +{ + "cells": [], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "language": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/acceptance/bundle/paths/flow_notebook/src/test.flow.ipynb b/acceptance/bundle/paths/flow_notebook/src/test.flow.ipynb new file mode 100644 index 00000000000..1ec2d26c287 --- /dev/null +++ b/acceptance/bundle/paths/flow_notebook/src/test.flow.ipynb @@ -0,0 +1,10 @@ +{ + "cells": [], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "language": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 226a848b723..3c9aeeb83fc 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -1240,3 +1240,95 @@ func TestTranslatePathsDesignerNotebookSkipLocalFileValidation(t *testing.T) { "/bundle/src/regular", b.Config.Resources.Jobs["job"].Tasks[1].NotebookTask.NotebookPath) } + +// TestTranslatePathsFlowNotebook verifies .flow.ipynb suffix is preserved. +func TestTranslatePathsFlowNotebook(t *testing.T) { + dir := t.TempDir() + touchDesignerFile(t, filepath.Join(dir, "src", "test.flow.ipynb")) + touchNotebookFile(t, filepath.Join(dir, "src", "regular.py")) + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/bundle", + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: jobs.JobSettings{ + Tasks: []jobs.Task{ + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "./src/test.flow.ipynb", + }, + }, + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "./src/regular.py", + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}}) + + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + require.NoError(t, diags.Error()) + + assert.Equal(t, "/bundle/src/test.flow.ipynb", b.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath) + assert.Equal(t, "/bundle/src/regular", b.Config.Resources.Jobs["job"].Tasks[1].NotebookTask.NotebookPath) +} + +// TestTranslatePathsFlowNotebookSkipLocalFileValidation verifies .flow.ipynb +// suffix is preserved when SkipLocalFileValidation is set. +func TestTranslatePathsFlowNotebookSkipLocalFileValidation(t *testing.T) { + dir := t.TempDir() + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + SkipLocalFileValidation: true, + Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/bundle", + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: jobs.JobSettings{ + Tasks: []jobs.Task{ + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "./src/test.flow.ipynb", + }, + }, + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "./src/regular.ipynb", + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}}) + + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + require.NoError(t, diags.Error()) + + assert.Equal(t, "/bundle/src/test.flow.ipynb", b.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath) + assert.Equal(t, "/bundle/src/regular", b.Config.Resources.Jobs["job"].Tasks[1].NotebookTask.NotebookPath) +} diff --git a/libs/notebook/ext.go b/libs/notebook/ext.go index 49155aaa8eb..1382b7a1f87 100644 --- a/libs/notebook/ext.go +++ b/libs/notebook/ext.go @@ -14,17 +14,15 @@ const ( ExtensionScala string = ".scala" ExtensionSql string = ".sql" ExtensionJupyter string = ".ipynb" - // ExtensionDesigner is the compound suffix for Lakeflow Designer files. - // Unlike other notebook types, designer files keep this full suffix when - // imported into the workspace. + // Designer and flow files keep their full compound suffix on workspace import. ExtensionDesigner string = ".designer.ipynb" + ExtensionFlow string = ".flow.ipynb" ) // StripExtension returns the workspace path for a local notebook file. -// Designer files keep their full ".designer.ipynb" suffix in the workspace; -// other notebook types lose their extension on import. +// Designer and flow files keep their compound suffix; other types lose their extension. func StripExtension(name string) string { - if strings.HasSuffix(name, ExtensionDesigner) { + if strings.HasSuffix(name, ExtensionDesigner) || strings.HasSuffix(name, ExtensionFlow) { return name } return strings.TrimSuffix(name, path.Ext(name)) diff --git a/libs/notebook/ext_test.go b/libs/notebook/ext_test.go index c1eb47ccd20..58e5f8d4823 100644 --- a/libs/notebook/ext_test.go +++ b/libs/notebook/ext_test.go @@ -22,6 +22,10 @@ func TestStripExtension(t *testing.T) { {"foo.designer.ipynb", "foo.designer.ipynb"}, {"a/b/c.designer.ipynb", "a/b/c.designer.ipynb"}, + // Flow files keep their full ".flow.ipynb" suffix. + {"foo.flow.ipynb", "foo.flow.ipynb"}, + {"a/b/c.flow.ipynb", "a/b/c.flow.ipynb"}, + // Files without a known extension are passed through path.Ext; // the last-segment extension is removed. {"foo", "foo"}, From 853297f7c72b87b04b229bf733d0a038cfd4cb2b Mon Sep 17 00:00:00 2001 From: Art Chen Date: Thu, 11 Jun 2026 00:04:43 -0700 Subject: [PATCH 2/2] Address PR review: consolidate flow test and fix capitalization - Merge flow_notebook acceptance test into designer_notebook by adding flow_task to the existing designer_job (avoids a redundant test dir) - Capitalize "Lakeflow" in the ext.go compound-suffix comment Co-authored-by: Art Chen --- .../bundle/paths/designer_notebook/databricks.yml | 3 +++ .../bundle/paths/designer_notebook/output.txt | 4 ++++ .../src/test.flow.ipynb | 0 .../bundle/paths/flow_notebook/databricks.yml | 14 -------------- acceptance/bundle/paths/flow_notebook/output.txt | 10 ---------- acceptance/bundle/paths/flow_notebook/script | 1 - .../bundle/paths/flow_notebook/src/regular.ipynb | 10 ---------- libs/notebook/ext.go | 2 +- 8 files changed, 8 insertions(+), 36 deletions(-) rename acceptance/bundle/paths/{flow_notebook => designer_notebook}/src/test.flow.ipynb (100%) delete mode 100644 acceptance/bundle/paths/flow_notebook/databricks.yml delete mode 100644 acceptance/bundle/paths/flow_notebook/output.txt delete mode 100644 acceptance/bundle/paths/flow_notebook/script delete mode 100644 acceptance/bundle/paths/flow_notebook/src/regular.ipynb diff --git a/acceptance/bundle/paths/designer_notebook/databricks.yml b/acceptance/bundle/paths/designer_notebook/databricks.yml index 291e63df797..e187d37be1d 100644 --- a/acceptance/bundle/paths/designer_notebook/databricks.yml +++ b/acceptance/bundle/paths/designer_notebook/databricks.yml @@ -9,6 +9,9 @@ resources: - task_key: designer_task notebook_task: notebook_path: ./src/test.designer.ipynb + - task_key: flow_task + notebook_task: + notebook_path: ./src/test.flow.ipynb - task_key: regular_task notebook_task: notebook_path: ./src/regular.ipynb diff --git a/acceptance/bundle/paths/designer_notebook/output.txt b/acceptance/bundle/paths/designer_notebook/output.txt index b473fac5f8d..300ca9f3972 100644 --- a/acceptance/bundle/paths/designer_notebook/output.txt +++ b/acceptance/bundle/paths/designer_notebook/output.txt @@ -3,6 +3,10 @@ "task_key": "designer_task", "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/designer_paths/default/files/src/test.designer.ipynb" }, + { + "task_key": "flow_task", + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/designer_paths/default/files/src/test.flow.ipynb" + }, { "task_key": "regular_task", "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/designer_paths/default/files/src/regular" diff --git a/acceptance/bundle/paths/flow_notebook/src/test.flow.ipynb b/acceptance/bundle/paths/designer_notebook/src/test.flow.ipynb similarity index 100% rename from acceptance/bundle/paths/flow_notebook/src/test.flow.ipynb rename to acceptance/bundle/paths/designer_notebook/src/test.flow.ipynb diff --git a/acceptance/bundle/paths/flow_notebook/databricks.yml b/acceptance/bundle/paths/flow_notebook/databricks.yml deleted file mode 100644 index 673c834a4fd..00000000000 --- a/acceptance/bundle/paths/flow_notebook/databricks.yml +++ /dev/null @@ -1,14 +0,0 @@ -bundle: - name: flow_paths - -resources: - jobs: - flow_job: - name: flow_job - tasks: - - task_key: flow_task - notebook_task: - notebook_path: ./src/test.flow.ipynb - - task_key: regular_task - notebook_task: - notebook_path: ./src/regular.ipynb diff --git a/acceptance/bundle/paths/flow_notebook/output.txt b/acceptance/bundle/paths/flow_notebook/output.txt deleted file mode 100644 index 933ea06ed68..00000000000 --- a/acceptance/bundle/paths/flow_notebook/output.txt +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "task_key": "flow_task", - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/flow_paths/default/files/src/test.flow.ipynb" - }, - { - "task_key": "regular_task", - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/flow_paths/default/files/src/regular" - } -] diff --git a/acceptance/bundle/paths/flow_notebook/script b/acceptance/bundle/paths/flow_notebook/script deleted file mode 100644 index 8384a3822b7..00000000000 --- a/acceptance/bundle/paths/flow_notebook/script +++ /dev/null @@ -1 +0,0 @@ -$CLI bundle validate -o json | jq '.resources.jobs.flow_job.tasks | map({task_key, notebook_path: .notebook_task.notebook_path})' diff --git a/acceptance/bundle/paths/flow_notebook/src/regular.ipynb b/acceptance/bundle/paths/flow_notebook/src/regular.ipynb deleted file mode 100644 index 1ec2d26c287..00000000000 --- a/acceptance/bundle/paths/flow_notebook/src/regular.ipynb +++ /dev/null @@ -1,10 +0,0 @@ -{ - "cells": [], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "language": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/libs/notebook/ext.go b/libs/notebook/ext.go index 1382b7a1f87..c9e33ab7150 100644 --- a/libs/notebook/ext.go +++ b/libs/notebook/ext.go @@ -14,7 +14,7 @@ const ( ExtensionScala string = ".scala" ExtensionSql string = ".sql" ExtensionJupyter string = ".ipynb" - // Designer and flow files keep their full compound suffix on workspace import. + // Designer and Flow files keep their full compound suffix on workspace import. ExtensionDesigner string = ".designer.ipynb" ExtensionFlow string = ".flow.ipynb" )