diff --git a/.execs/scripts/test-python.py b/.execs/scripts/test-python.py new file mode 100644 index 00000000..c2c3d0dc --- /dev/null +++ b/.execs/scripts/test-python.py @@ -0,0 +1,42 @@ +"""Integration check for flow's python interpreter support. + +Run by the `test python-script` executable in CI on Linux, macOS, and Windows. +It asserts the things flow itself is responsible for - that a real interpreter +ran, that flow-resolved parameters arrived in the environment, and that the +python env defaults were applied - so a regression in interpreter resolution +fails CI rather than silently degrading to shell execution. +""" + +import os +import platform +import sys + +print("Running python file execution test...") +print(f"OS: {platform.system()}") +print(f"Python: {sys.version.splitlines()[0]}") +print(f"Interpreter: {sys.executable}") + +failures = [] + +# A shell interpreter could never have reached this file at all, but assert the +# version explicitly so a python2 interpreter is caught rather than tolerated. +if sys.version_info[0] != 3: + failures.append(f"expected python 3, got {sys.version_info[0]}") + +# flow injects these so output streams live and workspaces stay free of +# __pycache__; see internal/services/run/python.go pythonEnv. +for key in ("PYTHONUNBUFFERED", "PYTHONDONTWRITEBYTECODE"): + if os.environ.get(key) != "1": + failures.append(f"{key} not set by flow (got {os.environ.get(key)!r})") + +# Set by env.DefaultEnv for every run; interpreter resolution relies on it to +# find a workspace's .venv. +if not os.environ.get("FLOW_WORKSPACE_PATH"): + failures.append("FLOW_WORKSPACE_PATH missing from the run environment") + +if failures: + for failure in failures: + print(f"FAIL: {failure}", file=sys.stderr) + sys.exit(1) + +print("Python file execution works correctly.") diff --git a/.execs/test.flow b/.execs/test.flow index bdf035d0..d2a81059 100644 --- a/.execs/test.flow +++ b/.execs/test.flow @@ -43,6 +43,14 @@ executables: exit 1 fi + - verb: test + name: python-script + description: Test .py file execution via the resolved python interpreter + tags: [python] + exec: + dir: // + file: .execs/scripts/test-python.py + - verb: test name: unit description: Run unit tests with coverage diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5196425f..1e566e18 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -48,6 +48,10 @@ jobs: uses: actions/setup-go@v7 with: go-version: "1.25.x" + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" # Build the flow under review and run every task with it, so CI exercises # the code as it would land on main rather than the released CLI. - name: Build flow @@ -75,6 +79,10 @@ jobs: uses: actions/setup-go@v7 with: go-version: "1.25.x" + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" # Build the flow under review and run every task with it, so CI exercises # the code as it would land on main rather than the released CLI. - name: Build flow @@ -181,6 +189,32 @@ jobs: flow-version: 'main' timeout: '5m' + python-scripts: + name: "Test / python script execution" + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: "1.25.x" + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" + # Run this one directly rather than through the action: it exercises flow's + # own interpreter resolution, so the binary under test must be the one that + # runs, with no install path in between. + - name: Build flow + run: go build -o ./bin/flow . + - name: Run .py file execution test + run: ./bin/flow test python-script + env: + DISABLE_FLOW_INTERACTIVE: "true" + security: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml index 888efd70..5c303830 100644 --- a/.github/workflows/windows-ci.yml +++ b/.github/workflows/windows-ci.yml @@ -26,6 +26,10 @@ jobs: with: go-version: "1.25.x" cache: true + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" # Build the flow under review and run every task with it, so CI exercises # the code as it would land on main rather than the released CLI. - name: Build flow @@ -60,6 +64,10 @@ jobs: with: go-version: "1.25.x" cache: true + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" # Build the flow under review and run every task with it, so CI exercises # the code as it would land on main rather than the released CLI. - name: Build flow @@ -153,6 +161,10 @@ jobs: with: go-version: "1.25.x" cache: true + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" # Build the flow under review and run every task with it, so CI exercises # the code as it would land on main rather than the released CLI. - name: Build flow @@ -166,6 +178,13 @@ jobs: flow-binary: ./bin/flow.exe # Fallback for action releases predating flow-binary; drop once v1 includes it. flow-version: "main" + # Run this one directly rather than through the action: it exercises flow's + # own interpreter resolution, so the binary under test must be the one that + # runs, with no install path in between. + - name: Run .py file execution test + run: ./bin/flow.exe test python-script + env: + DISABLE_FLOW_INTERACTIVE: "true" windows-validation-complete: name: "Windows validation complete" diff --git a/docs/guides/executables.md b/docs/guides/executables.md index c8c6ae28..9173ac00 100644 --- a/docs/guides/executables.md +++ b/docs/guides/executables.md @@ -231,9 +231,60 @@ executables: **Options:** - `cmd`: Inline command to run - `file`: Script file to execute +- `interpreter`: Which interpreter runs `cmd` — `sh` (default) or `python` (see below) - `logMode`: How to format command output - `container`: Run the command or file inside a container image (see below) +#### Running Python + +Set `exec.interpreter` to `python` to run `cmd` as a Python script instead of a shell command: + +```yaml +executables: + - verb: run + name: report + exec: + interpreter: python + cmd: | + import json, sys + print(json.dumps({"python": sys.version_info[:2]})) +``` + +Parameters, arguments, and secrets reach the script through the environment exactly as they do for +a shell command, so `os.environ` is how you read them. + +A `.py` file needs no `interpreter` at all — the extension implies it: + +```yaml +exec: + file: scripts/analyze.py +``` + +Setting `interpreter` explicitly overrides whatever the extension would have implied. + +**How flow finds Python.** A project's virtualenv wins over bare system Python, so a script gets +the dependencies its repository installed: + +| Order | Source | +|-------|--------| +| 1 | `FLOW_PYTHON_BIN`, if set | +| 2 | `$VIRTUAL_ENV` — an activated virtualenv | +| 3 | `/.venv` | +| 4 | `python3` on the `PATH` | +| 5 | `python` on the `PATH` | + +`FLOW_PYTHON_BIN` is an environment variable rather than a field, so you can pin an interpreter for +a whole workspace in its `.env` file, or for one executable via `params`. If it is set but does not +resolve, the run fails rather than silently falling back to a different interpreter. + +On Windows the virtualenv path is `Scripts\python.exe`, and `python` is preferred over `python3` — +`python3.exe` there is usually the Microsoft Store alias stub rather than a real interpreter. + +flow runs `cmd` from a temporary file rather than `python -c`, which keeps your code out of the +process table and means tracebacks carry real line numbers. It also sets `PYTHONUNBUFFERED=1` so +output streams as it is produced, and `PYTHONDONTWRITEBYTECODE=1` to keep `__pycache__` out of your +workspace. Set either variable yourself to override. + #### Running in a container Set `exec.container` to run the command inside a container instead of on the host. This gives you a diff --git a/docs/public/schemas/flowfile_schema.json b/docs/public/schemas/flowfile_schema.json index 40f24e8b..c06720e1 100644 --- a/docs/public/schemas/flowfile_schema.json +++ b/docs/public/schemas/flowfile_schema.json @@ -242,10 +242,14 @@ "default": "" }, "file": { - "description": "The file to execute (`.sh`, `.bat`, `.cmd`, `.ps1`).\nOnly one of `cmd` or `file` must be set.\n", + "description": "The file to execute (`.sh`, `.bat`, `.cmd`, `.ps1`, `.py`).\nOnly one of `cmd` or `file` must be set.\n", "type": "string", "default": "" }, + "interpreter": { + "$ref": "#/definitions/ExecutableExecInterpreter", + "description": "The interpreter used to run `cmd`. Defaults to `sh`, flow's built-in POSIX\nshell interpreter; `python` runs `cmd` as a Python script.\n\nWhen set, this also overrides the interpreter inferred from a `file`\nextension. When unset, `file` is run by the interpreter matching its\nextension, so `file: script.py` runs under Python without setting this.\n" + }, "logMode": { "description": "The log mode to use when running the executable.\nThis can either be `hidden`, `json`, `logfmt` or `text`\n", "type": "string", @@ -256,6 +260,14 @@ } } }, + "ExecutableExecInterpreter": { + "description": "The interpreter used to run a command.\n`sh` uses flow's built-in POSIX shell interpreter; `python` runs the command\nas a Python script.\n", + "type": "string", + "enum": [ + "sh", + "python" + ] + }, "ExecutableLaunchExecutableType": { "description": "Launches an application or opens a URI.", "type": "object", diff --git a/docs/types/flowfile.md b/docs/types/flowfile.md index 27dfb983..c78d0c55 100644 --- a/docs/types/flowfile.md +++ b/docs/types/flowfile.md @@ -209,10 +209,26 @@ Standard executable type. Runs a command/file in a subprocess. | `cmd` | The command to execute. Only one of `cmd` or `file` must be set. | `string` | | | | `container` | | [ExecutableExecContainer](#executableexeccontainer) | | | | `dir` | | [ExecutableDirectory](#executabledirectory) | | | -| `file` | The file to execute (`.sh`, `.bat`, `.cmd`, `.ps1`). Only one of `cmd` or `file` must be set. | `string` | | | +| `file` | The file to execute (`.sh`, `.bat`, `.cmd`, `.ps1`, `.py`). Only one of `cmd` or `file` must be set. | `string` | | | +| `interpreter` | The interpreter used to run `cmd`. Defaults to `sh`, flow's built-in POSIX shell interpreter; `python` runs `cmd` as a Python script. When set, this also overrides the interpreter inferred from a `file` extension. When unset, `file` is run by the interpreter matching its extension, so `file: script.py` runs under Python without setting this. | [ExecutableExecInterpreter](#executableexecinterpreter) | | | | `logMode` | The log mode to use when running the executable. This can either be `hidden`, `json`, `logfmt` or `text` | `string` | logfmt | | | `params` | | [ExecutableParameterList](#executableparameterlist) | | | +### ExecutableExecInterpreter + +The interpreter used to run a command. +`sh` uses flow's built-in POSIX shell interpreter; `python` runs the command +as a Python script. + + +**Type:** `string` + +**Valid values:** +- `sh` +- `python` + + + ### ExecutableLaunchExecutableType Launches an application or opens a URI. diff --git a/internal/io/executable/detail.go b/internal/io/executable/detail.go index 95d9325e..94da5c69 100644 --- a/internal/io/executable/detail.go +++ b/internal/io/executable/detail.go @@ -28,6 +28,9 @@ func executableDetailOpts(exec *executable.Executable) views.DetailContentOpts { func execTypeName(exec *executable.Executable) string { switch { case exec.Exec != nil: + if exec.Exec.InterpreterForFile() == executable.InterpreterPython { + return "Python Executable" + } return "Shell Executable" case exec.Launch != nil: return "Launch Executable" @@ -143,12 +146,23 @@ func shellExecConfig(e *executable.ExecutableEnvironment, s *executable.ExecExec if s == nil { return "" } - md := "## Shell Configuration\n" + lang, label := "sh", "Shell" + if s.ResolveInterpreter() == executable.InterpreterPython { + lang = "python" + } + if s.InterpreterForFile() == executable.InterpreterPython { + label = "Python" + } + + md := fmt.Sprintf("## %s Configuration\n", label) if s.LogMode != "" { md += fmt.Sprintf("**Log Mode:** %s\n\n", s.LogMode) } + if s.InterpreterIsSet() { + md += fmt.Sprintf("**Interpreter:** %s\n\n", s.ResolveInterpreter()) + } if s.Cmd != "" { - md += fmt.Sprintf("**Command**\n```sh\n%s\n```\n", s.Cmd) + md += fmt.Sprintf("**Command**\n```%s\n%s\n```\n", lang, s.Cmd) } else if s.File != "" { md += fmt.Sprintf("**File:** `%s`\n\n", s.File) } diff --git a/internal/io/executable/detail_interpreter_test.go b/internal/io/executable/detail_interpreter_test.go new file mode 100644 index 00000000..4468089f --- /dev/null +++ b/internal/io/executable/detail_interpreter_test.go @@ -0,0 +1,77 @@ +package executable_test + +import ( + "strings" + "testing" + + io "github.com/flowexec/flow/v2/internal/io/executable" + "github.com/flowexec/flow/v2/types/executable" +) + +func interp(v executable.ExecInterpreter) *executable.ExecInterpreter { return &v } + +// The browse views are how an interpreter is discovered without opening the +// flowfile, so each surface is pinned: the library subtitle, the detail heading, +// the interpreter line, and the fence language on the command block. +func TestBrowseSurfacesInterpreter(t *testing.T) { + tests := []struct { + name string + exec *executable.ExecExecutableType + wantSubtitle string + wantContains []string + wantMissing []string + }{ + { + name: "shell command", + exec: &executable.ExecExecutableType{Cmd: "echo hi"}, + wantSubtitle: "Shell Executable", + wantContains: []string{"## Shell Configuration", "```sh"}, + wantMissing: []string{"**Interpreter:**", "```python"}, + }, + { + name: "python command", + exec: &executable.ExecExecutableType{Cmd: "print('hi')", Interpreter: interp(executable.InterpreterPython)}, + wantSubtitle: "Python Executable", + wantContains: []string{"## Python Configuration", "**Interpreter:** python", "```python"}, + wantMissing: []string{"```sh"}, + }, + { + // The extension already says python, so the heading follows it while + // the interpreter line stays out - nothing was configured to report. + name: "py file without an explicit interpreter", + exec: &executable.ExecExecutableType{File: "report.py"}, + wantSubtitle: "Python Executable", + wantContains: []string{"## Python Configuration", "report.py"}, + wantMissing: []string{"**Interpreter:**"}, + }, + { + name: "explicit sh overrides a py extension", + exec: &executable.ExecExecutableType{File: "report.py", Interpreter: interp(executable.InterpreterSh)}, + wantSubtitle: "Shell Executable", + wantContains: []string{"## Shell Configuration"}, + wantMissing: []string{"## Python Configuration"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &executable.Executable{Verb: "run", Name: "probe", Exec: tt.exec} + + if got := io.ExecTypeNameForTest(e); got != tt.wantSubtitle { + t.Errorf("execTypeName() = %q, want %q", got, tt.wantSubtitle) + } + + body := io.ExecBodyMarkdownForTest(e) + for _, want := range tt.wantContains { + if !strings.Contains(body, want) { + t.Errorf("detail body missing %q:\n%s", want, body) + } + } + for _, missing := range tt.wantMissing { + if strings.Contains(body, missing) { + t.Errorf("detail body unexpectedly contains %q:\n%s", missing, body) + } + } + }) + } +} diff --git a/internal/io/executable/export_test.go b/internal/io/executable/export_test.go new file mode 100644 index 00000000..8825f6a7 --- /dev/null +++ b/internal/io/executable/export_test.go @@ -0,0 +1,16 @@ +package executable + +import "github.com/flowexec/flow/v2/types/executable" + +// Test seams for the browse rendering helpers, so the external test package can +// assert what the library and detail views show without exporting them. + +// ExecTypeNameForTest exposes execTypeName. +func ExecTypeNameForTest(e *executable.Executable) string { + return execTypeName(e) +} + +// ExecBodyMarkdownForTest exposes execBodyMarkdown. +func ExecBodyMarkdownForTest(e *executable.Executable) string { + return execBodyMarkdown(e) +} diff --git a/internal/runner/exec/exec.go b/internal/runner/exec/exec.go index 57971e33..0a6a0670 100644 --- a/internal/runner/exec/exec.go +++ b/internal/runner/exec/exec.go @@ -1,6 +1,8 @@ package exec import ( + "path/filepath" + "github.com/pkg/errors" "github.com/flowexec/flow/v2/internal/runner" @@ -16,6 +18,8 @@ import ( var ( runCmdFn = run.RunCmd runFileFn = run.RunFile + runPythonFn = run.RunPythonCmd + runPythonFileFn = run.RunPythonFile runContainerFn = run.RunContainer resolveRuntimeFn = run.ResolveRuntime ) @@ -102,8 +106,17 @@ func (r *execRunner) Exec( } switch { + case execSpec.Cmd != "" && execSpec.ResolveInterpreter() == executable.InterpreterPython: + return runPythonFn(execSpec.Cmd, targetDir, envList, logMode, logger.Log(), ctx.StdIn(), logFields, ctx.CurrentTask) case execSpec.Cmd != "": return runCmdFn(execSpec.Cmd, targetDir, envList, logMode, logger.Log(), ctx.StdIn(), logFields, ctx.CurrentTask) + case execSpec.File != "" && execSpec.InterpreterForFile() == executable.InterpreterPython: + // An explicit interpreter overrides the extension, so route here rather + // than letting RunFile dispatch on the suffix alone. + return runPythonFileFn( + filepath.Join(targetDir, execSpec.File), targetDir, + envList, logMode, logger.Log(), ctx.StdIn(), logFields, ctx.CurrentTask, + ) case execSpec.File != "": return runFileFn(execSpec.File, targetDir, envList, logMode, logger.Log(), ctx.StdIn(), logFields, ctx.CurrentTask) default: diff --git a/internal/runner/exec/exec_test.go b/internal/runner/exec/exec_test.go index c4ababe2..73019ef4 100644 --- a/internal/runner/exec/exec_test.go +++ b/internal/runner/exec/exec_test.go @@ -38,16 +38,20 @@ var _ = Describe("Exec Runner", func() { ctx *testUtils.ContextWithMocks mockEngine *mocks.MockEngine - cmdCalls []runCall - fileCalls []runCall - containerSpecs []run.ContainerSpec - cmdErr error - fileErr error - containerErr error - - restoreCmd func() - restoreFile func() - restoreContainer func() + cmdCalls []runCall + fileCalls []runCall + pythonCalls []runCall + pythonFileCalls []runCall + containerSpecs []run.ContainerSpec + cmdErr error + fileErr error + containerErr error + + restoreCmd func() + restoreFile func() + restorePython func() + restorePythonFile func() + restoreContainer func() ) BeforeEach(func() { @@ -58,6 +62,8 @@ var _ = Describe("Exec Runner", func() { cmdCalls = nil fileCalls = nil + pythonCalls = nil + pythonFileCalls = nil containerSpecs = nil cmdErr = nil fileErr = nil @@ -77,6 +83,20 @@ var _ = Describe("Exec Runner", func() { fileCalls = append(fileCalls, runCall{target: s, dir: dir, envList: envList, mode: logMode}) return fileErr }) + restorePython = exec.SetRunPythonFnForTest(func( + s, dir string, envList []string, logMode tuikitIO.LogMode, + _ tuikitIO.Logger, _ *os.File, _ map[string]any, _ *tuikitIO.TaskContext, + ) error { + pythonCalls = append(pythonCalls, runCall{target: s, dir: dir, envList: envList, mode: logMode}) + return cmdErr + }) + restorePythonFile = exec.SetRunPythonFileFnForTest(func( + s, dir string, envList []string, logMode tuikitIO.LogMode, + _ tuikitIO.Logger, _ *os.File, _ map[string]any, _ *tuikitIO.TaskContext, + ) error { + pythonFileCalls = append(pythonFileCalls, runCall{target: s, dir: dir, envList: envList, mode: logMode}) + return fileErr + }) restoreContainer = exec.SetRunContainerFnForTest(func( _ stdCtx.Context, spec run.ContainerSpec, _ tuikitIO.LogMode, _ tuikitIO.Logger, _ *os.File, _ map[string]any, _ *tuikitIO.TaskContext, @@ -89,6 +109,8 @@ var _ = Describe("Exec Runner", func() { AfterEach(func() { restoreCmd() restoreFile() + restorePython() + restorePythonFile() restoreContainer() }) @@ -141,6 +163,54 @@ var _ = Describe("Exec Runner", func() { Expect(fileCalls).To(BeEmpty()) }) + It("routes a python cmd through runPython instead of the shell", func() { + interpreter := executable.InterpreterPython + e := &executable.Executable{Exec: &executable.ExecExecutableType{ + Cmd: "print('hello')", Interpreter: &interpreter, + }} + e.SetContext(ctx.Ctx.CurrentWorkspace.AssignedName(), ctx.Ctx.CurrentWorkspace.Location(), "", "") + + Expect(execRnr.Exec(ctx.Ctx, e, mockEngine, map[string]string{}, nil)).To(Succeed()) + Expect(pythonCalls).To(HaveLen(1)) + Expect(pythonCalls[0].target).To(Equal("print('hello')")) + Expect(cmdCalls).To(BeEmpty()) + }) + + It("routes an explicit sh cmd through runCmd", func() { + interpreter := executable.InterpreterSh + e := &executable.Executable{Exec: &executable.ExecExecutableType{ + Cmd: "echo hello", Interpreter: &interpreter, + }} + e.SetContext(ctx.Ctx.CurrentWorkspace.AssignedName(), ctx.Ctx.CurrentWorkspace.Location(), "", "") + + Expect(execRnr.Exec(ctx.Ctx, e, mockEngine, map[string]string{}, nil)).To(Succeed()) + Expect(cmdCalls).To(HaveLen(1)) + Expect(pythonCalls).To(BeEmpty()) + }) + + It("routes a .py file through runPythonFile without an explicit interpreter", func() { + e := &executable.Executable{Exec: &executable.ExecExecutableType{File: "script.py"}} + e.SetContext(ctx.Ctx.CurrentWorkspace.AssignedName(), ctx.Ctx.CurrentWorkspace.Location(), "", "") + + Expect(execRnr.Exec(ctx.Ctx, e, mockEngine, map[string]string{}, nil)).To(Succeed()) + Expect(pythonFileCalls).To(HaveLen(1)) + // The python file seam receives a full path, unlike runFile. + Expect(pythonFileCalls[0].target).To(HaveSuffix("script.py")) + Expect(fileCalls).To(BeEmpty()) + }) + + It("lets an explicit interpreter override the file extension", func() { + interpreter := executable.InterpreterPython + e := &executable.Executable{Exec: &executable.ExecExecutableType{ + File: "script.sh", Interpreter: &interpreter, + }} + e.SetContext(ctx.Ctx.CurrentWorkspace.AssignedName(), ctx.Ctx.CurrentWorkspace.Location(), "", "") + + Expect(execRnr.Exec(ctx.Ctx, e, mockEngine, map[string]string{}, nil)).To(Succeed()) + Expect(pythonFileCalls).To(HaveLen(1)) + Expect(fileCalls).To(BeEmpty()) + }) + It("routes a file through runFile", func() { e := &executable.Executable{Exec: &executable.ExecExecutableType{File: "script.sh"}} e.SetContext(ctx.Ctx.CurrentWorkspace.AssignedName(), ctx.Ctx.CurrentWorkspace.Location(), "", "") diff --git a/internal/runner/exec/export_test.go b/internal/runner/exec/export_test.go index cf53d7c4..ffaa4d2a 100644 --- a/internal/runner/exec/export_test.go +++ b/internal/runner/exec/export_test.go @@ -35,6 +35,20 @@ func SetRunFileFnForTest(fn RunFunc) func() { return func() { runFileFn = prev } } +// SetRunPythonFnForTest swaps the python command-runner seam and returns a restore func. +func SetRunPythonFnForTest(fn RunFunc) func() { + prev := runPythonFn + runPythonFn = fn + return func() { runPythonFn = prev } +} + +// SetRunPythonFileFnForTest swaps the python file-runner seam and returns a restore func. +func SetRunPythonFileFnForTest(fn RunFunc) func() { + prev := runPythonFileFn + runPythonFileFn = fn + return func() { runPythonFileFn = prev } +} + // ContainerRunFunc matches the signature of run.RunContainer. type ContainerRunFunc = func( ctx stdctx.Context, diff --git a/internal/services/run/export_test.go b/internal/services/run/export_test.go index 82315e20..646a0375 100644 --- a/internal/services/run/export_test.go +++ b/internal/services/run/export_test.go @@ -33,3 +33,23 @@ func ResetRuntimeCacheForTest() { defer autoRuntimeMu.Unlock() autoRuntime = "" } + +// EnvValueForTest exposes envValue. +func EnvValueForTest(envList []string, key string) string { + return envValue(envList, key) +} + +// PythonEnvForTest exposes pythonEnv. +func PythonEnvForTest(envList []string) []string { + return pythonEnv(envList) +} + +// VenvPythonForTest exposes venvPython. +func VenvPythonForTest(venvDir string) string { + return venvPython(venvDir) +} + +// PathCandidatesForTest exposes pathCandidates. +func PathCandidatesForTest() []string { + return pathCandidates() +} diff --git a/internal/services/run/python.go b/internal/services/run/python.go new file mode 100644 index 00000000..983544c0 --- /dev/null +++ b/internal/services/run/python.go @@ -0,0 +1,228 @@ +package run + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/flowexec/tuikit/io" + + flowErrors "github.com/flowexec/flow/v2/pkg/errors" +) + +const ( + // PythonBinEnv overrides interpreter discovery entirely. It is read from the + // flow-resolved environment before the process environment, so it can be set + // per-workspace in a .env file or per-executable via params. + PythonBinEnv = "FLOW_PYTHON_BIN" + + // virtualEnvEnv is the variable an activated virtualenv exports. + virtualEnvEnv = "VIRTUAL_ENV" + + // workspacePathEnv is set by env.DefaultEnv for every run; it is how the + // workspace root reaches this package without widening the run signatures. + workspacePathEnv = "FLOW_WORKSPACE_PATH" + + // conventionalVenvDir is the in-repo virtualenv location flow looks for when + // none is active. + conventionalVenvDir = ".venv" +) + +// ResolvePython locates the python interpreter to run with, preferring a project's +// virtualenv over bare system python so that an agent running python inside a repo +// gets that repo's dependencies. +// +// Order: FLOW_PYTHON_BIN, the active VIRTUAL_ENV, the workspace's .venv, then +// python3 and python on the PATH. +func ResolvePython(envList []string) (string, error) { + if override := envValue(envList, PythonBinEnv); override != "" { + // An override may be a bare name or a path; resolve the former on PATH and + // trust the latter, so a deliberately-chosen interpreter never silently + // falls through to a different one. + if strings.ContainsRune(override, os.PathSeparator) || strings.ContainsRune(override, '/') { + if isExecutableFile(override) { + return override, nil + } + } else if path, err := lookPath(override); err == nil { + return path, nil + } + return "", flowErrors.NewInterpreterNotFoundError("python", []string{PythonBinEnv + "=" + override}) + } + + searched := []string{PythonBinEnv} + + if venv := envValue(envList, virtualEnvEnv); venv != "" { + candidate := venvPython(venv) + searched = append(searched, candidate) + if isExecutableFile(candidate) { + return candidate, nil + } + } + + if wsRoot := envValue(envList, workspacePathEnv); wsRoot != "" { + candidate := venvPython(filepath.Join(wsRoot, conventionalVenvDir)) + searched = append(searched, candidate) + if isExecutableFile(candidate) { + return candidate, nil + } + } + + for _, name := range pathCandidates() { + searched = append(searched, name) + if path, err := lookPath(name); err == nil { + return path, nil + } + } + + return "", flowErrors.NewInterpreterNotFoundError("python", searched) +} + +// pathCandidates returns the bare interpreter names to try on the PATH, in order. +// +// Windows deliberately tries "python" first: "python3.exe" there is usually the +// Microsoft Store App Execution Alias, which is on the PATH even when Python is +// not installed and merely prints a store advertisement before exiting non-zero. +// Preferring "python" finds a real installation and only falls back to the alias. +func pathCandidates() []string { + if runtime.GOOS == "windows" { + return []string{"python", "python3"} + } + return []string{"python3", "python"} +} + +// venvPython returns the interpreter path inside a virtualenv directory. +func venvPython(venvDir string) string { + if runtime.GOOS == "windows" { + return filepath.Join(venvDir, "Scripts", "python.exe") + } + return filepath.Join(venvDir, "bin", "python") +} + +func isExecutableFile(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} + +// envValue reads a key from a KEY=VALUE list, falling back to the process +// environment. The list is scanned in reverse because later entries win, matching +// how RunCmd layers the flow environment over os.Environ. +func envValue(envList []string, key string) string { + prefix := key + "=" + for i := len(envList) - 1; i >= 0; i-- { + if strings.HasPrefix(envList[i], prefix) { + return strings.TrimPrefix(envList[i], prefix) + } + } + return os.Getenv(key) +} + +// RunPythonCmd executes python source in a specific directory. +// +// The code is written to a 0600 temp file rather than passed with `python -c`: +// that keeps user code (which may interpolate secrets) out of the process table, +// produces tracebacks with real line numbers, and sidesteps shell quoting for +// multi-line scripts. +func RunPythonCmd( + code, dir string, + envList []string, + logMode io.LogMode, + logger io.Logger, + stdIn *os.File, + logFields map[string]interface{}, + task *io.TaskContext, +) error { + pythonBin, err := ResolvePython(envList) + if err != nil { + return err + } + + scriptPath, cleanup, err := WritePythonScript(code) + if err != nil { + return err + } + defer cleanup() + + logger.Debugf("running python (%s) in dir (%s)", pythonBin, dir) + return runNativeFile( + pythonBin, []string{scriptPath}, dir, + pythonEnv(envList), logMode, logger, stdIn, logFields, task, + ) +} + +// RunPythonFile executes an existing .py file with the resolved interpreter. +func RunPythonFile( + fullPath, dir string, + envList []string, + logMode io.LogMode, + logger io.Logger, + stdIn *os.File, + logFields map[string]interface{}, + task *io.TaskContext, +) error { + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + return fmt.Errorf("file does not exist - %s", fullPath) + } + + pythonBin, err := ResolvePython(envList) + if err != nil { + return err + } + + logger.Debugf("executing python file (%s) with %s", fullPath, pythonBin) + return runNativeFile( + pythonBin, []string{fullPath}, dir, + pythonEnv(envList), logMode, logger, stdIn, logFields, task, + ) +} + +// WritePythonScript writes python source to a 0600 temp file and returns its path +// alongside a cleanup func. Exported so callers that need the script to outlive +// the call - notably container execution, which bind-mounts it - can manage it. +func WritePythonScript(code string) (path string, cleanup func(), err error) { + file, err := os.CreateTemp("", "flow-*.py") + if err != nil { + return "", func() {}, fmt.Errorf("unable to create python script file - %w", err) + } + name := file.Name() + remove := func() { _ = os.Remove(name) } + + if _, err := file.WriteString(code); err != nil { + _ = file.Close() + remove() + return "", func() {}, fmt.Errorf("unable to write python script file - %w", err) + } + if err := file.Close(); err != nil { + remove() + return "", func() {}, fmt.Errorf("unable to close python script file - %w", err) + } + if err := os.Chmod(name, 0600); err != nil { + remove() + return "", func() {}, fmt.Errorf("unable to set python script permissions - %w", err) + } + return name, remove, nil +} + +// pythonEnv layers python-specific defaults over the resolved environment. +// +// PYTHONUNBUFFERED: flow pipes stdout to a log writer rather than a tty, and +// CPython block-buffers to a pipe - without this a long run emits nothing until +// it exits, so it looks hung to anyone (or any agent) watching the output. +// +// PYTHONDONTWRITEBYTECODE: `file:` execution runs from the workspace, and flow +// should not litter a user's repository with __pycache__ directories. +// +// Both stay overridable: an explicit value in the run's environment wins. +func pythonEnv(envList []string) []string { + out := append([]string{}, envList...) + for _, kv := range [][2]string{ + {"PYTHONUNBUFFERED", "1"}, + {"PYTHONDONTWRITEBYTECODE", "1"}, + } { + if envValue(envList, kv[0]) == "" { + out = append(out, kv[0]+"="+kv[1]) + } + } + return out +} diff --git a/internal/services/run/python_test.go b/internal/services/run/python_test.go new file mode 100644 index 00000000..39ce44a4 --- /dev/null +++ b/internal/services/run/python_test.go @@ -0,0 +1,164 @@ +//go:build unit + +package run_test + +import ( + "os" + "path/filepath" + "runtime" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flowexec/flow/v2/internal/services/run" + flowErrors "github.com/flowexec/flow/v2/pkg/errors" +) + +// writeFakePython creates an executable stub at the conventional interpreter path +// inside venvDir and returns that path. +func writeFakePython(venvDir string) string { + path := run.VenvPythonForTest(venvDir) + Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed()) + Expect(os.WriteFile(path, []byte("#!/bin/sh\nexit 0\n"), 0700)).To(Succeed()) + return path +} + +var _ = Describe("Python", func() { + Describe("envValue", func() { + It("prefers the last matching entry, since later entries win at exec time", func() { + list := []string{"FOO=first", "BAR=x", "FOO=second"} + Expect(run.EnvValueForTest(list, "FOO")).To(Equal("second")) + }) + + It("falls back to the process environment", func() { + GinkgoTB().Setenv("FLOW_PY_TEST_ONLY", "from-process") + Expect(run.EnvValueForTest(nil, "FLOW_PY_TEST_ONLY")).To(Equal("from-process")) + }) + + It("returns empty for an unset key", func() { + Expect(run.EnvValueForTest([]string{"A=1"}, "MISSING_ENTIRELY")).To(BeEmpty()) + }) + }) + + Describe("pythonEnv", func() { + It("adds unbuffered and no-bytecode defaults", func() { + out := strings.Join(run.PythonEnvForTest([]string{"A=1"}), "\n") + Expect(out).To(ContainSubstring("PYTHONUNBUFFERED=1")) + Expect(out).To(ContainSubstring("PYTHONDONTWRITEBYTECODE=1")) + }) + + It("does not override an explicit value from the run environment", func() { + out := run.PythonEnvForTest([]string{"PYTHONUNBUFFERED=0"}) + Expect(out).NotTo(ContainElement("PYTHONUNBUFFERED=1")) + Expect(out).To(ContainElement("PYTHONUNBUFFERED=0")) + }) + + It("does not mutate the caller's slice", func() { + in := []string{"A=1"} + run.PythonEnvForTest(in) + Expect(in).To(HaveLen(1)) + }) + }) + + Describe("pathCandidates", func() { + It("prefers python over python3 on Windows to dodge the Store alias stub", func() { + if runtime.GOOS == "windows" { + Expect(run.PathCandidatesForTest()).To(Equal([]string{"python", "python3"})) + } else { + Expect(run.PathCandidatesForTest()).To(Equal([]string{"python3", "python"})) + } + }) + }) + + Describe("ResolvePython", func() { + It("honours FLOW_PYTHON_BIN when it points at a real file", func() { + dir := GinkgoTB().TempDir() + bin := writeFakePython(filepath.Join(dir, "venv")) + + got, err := run.ResolvePython([]string{run.PythonBinEnv + "=" + bin}) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(bin)) + }) + + It("fails loudly rather than falling through when FLOW_PYTHON_BIN is wrong", func() { + // A deliberately-chosen interpreter must never silently resolve to a + // different one. + _, err := run.ResolvePython([]string{run.PythonBinEnv + "=/nonexistent/python"}) + Expect(err).To(HaveOccurred()) + var notFound flowErrors.InterpreterNotFoundError + Expect(err).To(BeAssignableToTypeOf(notFound)) + Expect(err.Error()).To(ContainSubstring("/nonexistent/python")) + }) + + It("uses an active VIRTUAL_ENV", func() { + venv := filepath.Join(GinkgoTB().TempDir(), "venv") + bin := writeFakePython(venv) + + got, err := run.ResolvePython([]string{"VIRTUAL_ENV=" + venv}) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(bin)) + }) + + It("falls back to the workspace .venv when no virtualenv is active", func() { + ws := GinkgoTB().TempDir() + bin := writeFakePython(filepath.Join(ws, ".venv")) + + got, err := run.ResolvePython([]string{"FLOW_WORKSPACE_PATH=" + ws}) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(bin)) + }) + + It("prefers an active VIRTUAL_ENV over the workspace .venv", func() { + ws := GinkgoTB().TempDir() + writeFakePython(filepath.Join(ws, ".venv")) + active := filepath.Join(GinkgoTB().TempDir(), "active") + activeBin := writeFakePython(active) + + got, err := run.ResolvePython([]string{ + "FLOW_WORKSPACE_PATH=" + ws, + "VIRTUAL_ENV=" + active, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(activeBin)) + }) + + It("skips a virtualenv path that does not exist and keeps searching", func() { + // A stale VIRTUAL_ENV should not be fatal while a real interpreter exists. + got, err := run.ResolvePython([]string{"VIRTUAL_ENV=/nonexistent/venv"}) + if err != nil { + Skip("no python on PATH to fall back to") + } + Expect(got).NotTo(ContainSubstring("/nonexistent/venv")) + }) + }) + + Describe("WritePythonScript", func() { + It("writes the code to a .py file and cleans up", func() { + path, cleanup, err := run.WritePythonScript("print('hi')\n") + Expect(err).NotTo(HaveOccurred()) + Expect(filepath.Ext(path)).To(Equal(".py")) + + content, err := os.ReadFile(path) //nolint:gosec + Expect(err).NotTo(HaveOccurred()) + Expect(string(content)).To(Equal("print('hi')\n")) + + cleanup() + _, statErr := os.Stat(path) + Expect(os.IsNotExist(statErr)).To(BeTrue()) + }) + + It("restricts permissions so script contents are not world-readable", func() { + if runtime.GOOS == "windows" { + Skip("unix file modes are ACL-mapped on Windows") + } + path, cleanup, err := run.WritePythonScript("print(1)\n") + Expect(err).NotTo(HaveOccurred()) + defer cleanup() + + info, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600))) + }) + }) +}) diff --git a/internal/services/run/run.go b/internal/services/run/run.go index de0c5b17..6f46f80d 100644 --- a/internal/services/run/run.go +++ b/internal/services/run/run.go @@ -77,6 +77,7 @@ func RunCmd( // RunFile executes a file in a specific directory. // Shell scripts (.sh) are interpreted via the built-in POSIX shell interpreter. // Batch files (.bat, .cmd) are executed via cmd.exe and PowerShell scripts (.ps1) via pwsh/powershell. +// Python scripts (.py) are executed via the interpreter resolved by ResolvePython. func RunFile( filename, dir string, envList []string, @@ -101,6 +102,8 @@ func RunFile( shell := findPowerShell() return runNativeFile(shell, []string{"-NoProfile", "-ExecutionPolicy", "Bypass", "-File", fullPath}, dir, envList, logMode, logger, stdIn, logFields, task) + case ".py": + return RunPythonFile(fullPath, dir, envList, logMode, logger, stdIn, logFields, task) default: return runShellFile(fullPath, envList, logMode, logger, stdIn, logFields, task) } diff --git a/internal/validation/flowfile_schema.json b/internal/validation/flowfile_schema.json index 40f24e8b..c06720e1 100644 --- a/internal/validation/flowfile_schema.json +++ b/internal/validation/flowfile_schema.json @@ -242,10 +242,14 @@ "default": "" }, "file": { - "description": "The file to execute (`.sh`, `.bat`, `.cmd`, `.ps1`).\nOnly one of `cmd` or `file` must be set.\n", + "description": "The file to execute (`.sh`, `.bat`, `.cmd`, `.ps1`, `.py`).\nOnly one of `cmd` or `file` must be set.\n", "type": "string", "default": "" }, + "interpreter": { + "$ref": "#/definitions/ExecutableExecInterpreter", + "description": "The interpreter used to run `cmd`. Defaults to `sh`, flow's built-in POSIX\nshell interpreter; `python` runs `cmd` as a Python script.\n\nWhen set, this also overrides the interpreter inferred from a `file`\nextension. When unset, `file` is run by the interpreter matching its\nextension, so `file: script.py` runs under Python without setting this.\n" + }, "logMode": { "description": "The log mode to use when running the executable.\nThis can either be `hidden`, `json`, `logfmt` or `text`\n", "type": "string", @@ -256,6 +260,14 @@ } } }, + "ExecutableExecInterpreter": { + "description": "The interpreter used to run a command.\n`sh` uses flow's built-in POSIX shell interpreter; `python` runs the command\nas a Python script.\n", + "type": "string", + "enum": [ + "sh", + "python" + ] + }, "ExecutableLaunchExecutableType": { "description": "Launches an application or opens a URI.", "type": "object", diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 7478eeeb..d38dbc6e 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -2,6 +2,7 @@ package errors import ( "fmt" + "strings" ) type ExecutableNotFoundError struct { @@ -98,6 +99,31 @@ func NewContainerRuntimeError(runtime string, err error) ContainerRuntimeError { return ContainerRuntimeError{Runtime: runtime, Err: err} } +// InterpreterNotFoundError indicates flow could not locate the interpreter +// binary required to run an executable, e.g. no python on the PATH. +type InterpreterNotFoundError struct { + Interpreter string + // Searched lists the candidates flow tried, in order, so the message can + // tell the user where to point FLOW_PYTHON_BIN. + Searched []string +} + +func (e InterpreterNotFoundError) Error() string { + if len(e.Searched) == 0 { + return fmt.Sprintf("no %s interpreter found", e.Interpreter) + } + return fmt.Sprintf( + "no %s interpreter found (tried %s)", + e.Interpreter, strings.Join(e.Searched, ", "), + ) +} + +func (e InterpreterNotFoundError) Code() string { return ErrCodeExecutionFailed } + +func NewInterpreterNotFoundError(interpreter string, searched []string) InterpreterNotFoundError { + return InterpreterNotFoundError{Interpreter: interpreter, Searched: searched} +} + // ValidationError indicates a value failed semantic or schema validation. type ValidationError struct { Msg string diff --git a/tests/python_exec_e2e_test.go b/tests/python_exec_e2e_test.go new file mode 100644 index 00000000..4d1eddd1 --- /dev/null +++ b/tests/python_exec_e2e_test.go @@ -0,0 +1,136 @@ +//go:build e2e + +package tests_test + +import ( + stdCtx "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "runtime" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flowexec/flow/v2/tests/utils" +) + +// canRunPython reports whether a python interpreter is reachable, mirroring the +// runner's own PATH precedence (see internal/services/run/python.go +// pathCandidates): Windows prefers "python" because "python3" there is usually +// the Microsoft Store alias stub rather than a real interpreter. +// +// CI pins an interpreter with actions/setup-python, so this guard should never +// skip there; it only spares a developer machine without python installed. +func canRunPython() bool { + candidates := []string{"python3", "python"} + if runtime.GOOS == "windows" { + candidates = []string{"python", "python3"} + } + for _, name := range candidates { + if _, err := exec.LookPath(name); err == nil { + return true + } + } + return false +} + +// pySpec builds a --spec argument for a transient python executable. +func pySpec(name string, e map[string]any) string { + spec := map[string]any{"verb": "run", "name": name, "exec": e} + out, err := json.Marshal(spec) + Expect(err).NotTo(HaveOccurred()) + return string(out) +} + +var _ = Describe("python exec e2e", func() { + var ctx *utils.Context + + BeforeEach(func() { + ctx = utils.NewContext(stdCtx.Background(), GinkgoTB()) + if !canRunPython() { + Skip("no python interpreter available on PATH") + } + }) + + AfterEach(func() { + ctx.Finalize() + }) + + When("an executable sets interpreter: python", func() { + It("runs the command as python rather than shell", func() { + runner := utils.NewE2ECommandRunner() + stdOut := ctx.StdOut() + Expect(runner.Run(ctx.Context, "exec", "examples:with-python")).To(Succeed()) + out, _ := readFileContent(stdOut) + Expect(out).To(ContainSubstring("hello from with-python")) + // Proves a real interpreter ran: flow's shell interpreter would not + // evaluate sys.version_info. + Expect(out).To(ContainSubstring("py-major=3")) + }) + + It("reports a traceback with the failing line when the script raises", func() { + runner := utils.NewE2ECommandRunner() + stdOut := ctx.StdOut() + ctx.ExpectFailure() + err := runner.Run(ctx.Context, "exec", "--spec", pySpec("py-raise", map[string]any{ + "interpreter": "python", + "cmd": "x = 1\ny = 2\nraise ValueError('boom')\n", + })) + Expect(err).To(HaveOccurred()) + out, _ := readFileContent(stdOut) + Expect(out).To(ContainSubstring("ValueError: boom")) + // Running from a temp file preserves line numbers; `python -c` would not. + Expect(out).To(ContainSubstring("line 3")) + }) + + It("passes flow-resolved parameters through the environment", func() { + runner := utils.NewE2ECommandRunner() + stdOut := ctx.StdOut() + Expect(runner.Run(ctx.Context, "exec", "--spec", pySpec("py-params", map[string]any{ + "interpreter": "python", + "params": []map[string]any{{"envKey": "GREETING", "text": "from-params"}}, + "cmd": "import os\nprint('greeting=' + os.environ['GREETING'])\n", + }))).To(Succeed()) + out, _ := readFileContent(stdOut) + Expect(out).To(ContainSubstring("greeting=from-params")) + }) + }) + + When("an executable runs a .py file", func() { + It("infers the python interpreter from the extension", func() { + dir := ctx.WorkspaceDir() + Expect(os.WriteFile( + filepath.Join(dir, "e2e_script.py"), + []byte("print('hello from py file')\n"), 0600, + )).To(Succeed()) + + runner := utils.NewE2ECommandRunner() + stdOut := ctx.StdOut() + Expect(runner.Run(ctx.Context, "exec", "--spec", pySpec("py-file", map[string]any{ + "file": "e2e_script.py", + "dir": filepath.ToSlash(dir), + }))).To(Succeed()) + out, _ := readFileContent(stdOut) + Expect(out).To(ContainSubstring("hello from py file")) + }) + }) + + When("the configured interpreter cannot be found", func() { + It("fails with an actionable error naming what was tried", func() { + GinkgoTB().Setenv("FLOW_PYTHON_BIN", filepath.Join(ctx.WorkspaceDir(), "not-python")) + + runner := utils.NewE2ECommandRunner() + stdOut := ctx.StdOut() + ctx.ExpectFailure() + err := runner.Run(ctx.Context, "exec", "--spec", pySpec("py-missing", map[string]any{ + "interpreter": "python", + "cmd": "print(1)\n", + })) + Expect(err).To(HaveOccurred()) + out, _ := readFileContent(stdOut) + Expect(out).To(ContainSubstring("no python interpreter found")) + }) + }) +}) diff --git a/tests/utils/builder/exec.go b/tests/utils/builder/exec.go index defab8d4..99f985a9 100644 --- a/tests/utils/builder/exec.go +++ b/tests/utils/builder/exec.go @@ -258,6 +258,28 @@ func ExecWithContainer(opts ...Option) *executable.Executable { return e } +func ExecWithPython(opts ...Option) *executable.Executable { + name := "with-python" + interpreter := executable.InterpreterPython + e := &executable.Executable{ + Verb: "run", + Name: name, + Visibility: privateExecVisibility(), + Exec: &executable.ExecExecutableType{ + Interpreter: &interpreter, + Cmd: fmt.Sprintf( + "import sys\nprint('hello from %s')\nprint('py-major=%%d' %% sys.version_info[0])\n", + name, + ), + }, + } + if len(opts) > 0 { + vals := NewOptionValues(opts...) + e.SetContext(vals.WorkspaceName, vals.WorkspacePath, vals.NamespaceName, vals.FlowFilePath) + } + return e +} + func ExecWithWorkspaceEnv(opts ...Option) *executable.Executable { name := "with-workspace-env" e := &executable.Executable{ diff --git a/tests/utils/builder/flowfile.go b/tests/utils/builder/flowfile.go index db5fa639..12519d42 100644 --- a/tests/utils/builder/flowfile.go +++ b/tests/utils/builder/flowfile.go @@ -36,6 +36,7 @@ func ExamplesExecFlowFile(opts ...Option) *executable.FlowFile { ExecWithEnvOutputFiles(opts...), ExecWithWorkspaceEnv(opts...), ExecWithContainer(opts...), + ExecWithPython(opts...), }, } if len(opts) > 0 { diff --git a/types/executable/executable.gen.go b/types/executable/executable.gen.go index 88fd8c8e..a879684a 100644 --- a/types/executable/executable.gen.go +++ b/types/executable/executable.gen.go @@ -143,11 +143,20 @@ type ExecExecutableType struct { // Dir corresponds to the JSON schema field "dir". Dir Directory `json:"dir,omitempty" yaml:"dir,omitempty" mapstructure:"dir,omitempty"` - // The file to execute (`.sh`, `.bat`, `.cmd`, `.ps1`). + // The file to execute (`.sh`, `.bat`, `.cmd`, `.ps1`, `.py`). // Only one of `cmd` or `file` must be set. // File string `json:"file,omitempty" yaml:"file,omitempty" mapstructure:"file,omitempty"` + // The interpreter used to run `cmd`. Defaults to `sh`, flow's built-in POSIX + // shell interpreter; `python` runs `cmd` as a Python script. + // + // When set, this also overrides the interpreter inferred from a `file` + // extension. When unset, `file` is run by the interpreter matching its + // extension, so `file: script.py` runs under Python without setting this. + // + Interpreter *ExecInterpreter `json:"interpreter,omitempty" yaml:"interpreter,omitempty" mapstructure:"interpreter,omitempty"` + // logFields corresponds to the JSON schema field "logFields". logFields map[string]interface{} `json:"logFields,omitempty" yaml:"logFields,omitempty" mapstructure:"logFields,omitempty"` @@ -160,6 +169,11 @@ type ExecExecutableType struct { Params ParameterList `json:"params,omitempty" yaml:"params,omitempty" mapstructure:"params,omitempty"` } +type ExecInterpreter string + +const ExecInterpreterPython ExecInterpreter = "python" +const ExecInterpreterSh ExecInterpreter = "sh" + // The executable schema defines the structure of an executable in the Flow CLI. // Executables are the building blocks of workflows and are used to define the // actions that can be performed in a workspace. diff --git a/types/executable/executable.go b/types/executable/executable.go index f5540cf1..6959a4e1 100644 --- a/types/executable/executable.go +++ b/types/executable/executable.go @@ -274,9 +274,14 @@ func (e *Executable) Validate() error { return err } - if e.Exec != nil && e.Exec.Container != nil { - if err := e.Exec.Container.Validate(); err != nil { - return fmt.Errorf("container validation failed - %w", err) + if e.Exec != nil { + if err := e.Exec.Validate(); err != nil { + return fmt.Errorf("exec validation failed - %w", err) + } + if e.Exec.Container != nil { + if err := e.Exec.Container.Validate(); err != nil { + return fmt.Errorf("container validation failed - %w", err) + } } } diff --git a/types/executable/executable_md.go b/types/executable/executable_md.go index 74391cd2..a116f4c2 100644 --- a/types/executable/executable_md.go +++ b/types/executable/executable_md.go @@ -90,15 +90,18 @@ func shellExecMarkdown(e *ExecutableEnvironment, s *ExecExecutableType) string { if s == nil { return "" } - mkdwn := "## Shell Configuration\n" + mkdwn := fmt.Sprintf("## %s Configuration\n", interpreterLabel(s)) if s.Dir != "" { mkdwn += fmt.Sprintf("**Executed from:** `%s`\n", s.Dir) } if s.LogMode != "" { mkdwn += fmt.Sprintf("**Log Mode:** %s\n", s.LogMode) } + if s.InterpreterIsSet() { + mkdwn += fmt.Sprintf("**Interpreter:** %s\n", s.ResolveInterpreter()) + } if s.Cmd != "" { - mkdwn += fmt.Sprintf("**Command**\n```sh\n%s\n```\n", s.Cmd) + mkdwn += fmt.Sprintf("**Command**\n```%s\n%s\n```\n", codeFence(s), s.Cmd) } else if s.File != "" { mkdwn += fmt.Sprintf("**File:** `%s`\n", s.File) } @@ -389,3 +392,20 @@ func addPrefx(s, prefix string) string { } return final } + +// interpreterLabel names the exec section after whatever actually runs the +// command, so a python executable does not present itself as a shell one. +func interpreterLabel(s *ExecExecutableType) string { + if s.InterpreterForFile() == InterpreterPython { + return "Python" + } + return "Shell" +} + +// codeFence returns the markdown language hint for the command block. +func codeFence(s *ExecExecutableType) string { + if s.ResolveInterpreter() == InterpreterPython { + return "python" + } + return "sh" +} diff --git a/types/executable/executable_schema.yaml b/types/executable/executable_schema.yaml index e820784b..0f0d5f08 100644 --- a/types/executable/executable_schema.yaml +++ b/types/executable/executable_schema.yaml @@ -302,6 +302,14 @@ definitions: The host path may be absolute, `~/`-prefixed, or `//`-prefixed (workspace-relative). The container path must be absolute. + ExecInterpreter: + type: string + enum: [sh, python] + description: | + The interpreter used to run a command. + `sh` uses flow's built-in POSIX shell interpreter; `python` runs the command + as a Python script. + ExecContainer: type: object required: [image] @@ -381,9 +389,18 @@ definitions: file: type: string description: | - The file to execute (`.sh`, `.bat`, `.cmd`, `.ps1`). + The file to execute (`.sh`, `.bat`, `.cmd`, `.ps1`, `.py`). Only one of `cmd` or `file` must be set. default: "" + interpreter: + $ref: '#/definitions/ExecInterpreter' + description: | + The interpreter used to run `cmd`. Defaults to `sh`, flow's built-in POSIX + shell interpreter; `python` runs `cmd` as a Python script. + + When set, this also overrides the interpreter inferred from a `file` + extension. When unset, `file` is run by the interpreter matching its + extension, so `file: script.py` runs under Python without setting this. container: $ref: '#/definitions/ExecContainer' logMode: diff --git a/types/executable/interpreter.go b/types/executable/interpreter.go new file mode 100644 index 00000000..1309fbb5 --- /dev/null +++ b/types/executable/interpreter.go @@ -0,0 +1,67 @@ +package executable + +import ( + "fmt" + "path/filepath" + "strings" +) + +const ( + InterpreterSh = ExecInterpreterSh + InterpreterPython = ExecInterpreterPython + + // DefaultInterpreter is used when interpreter is unset. + DefaultInterpreter = InterpreterSh +) + +// ResolveInterpreter returns the interpreter that should run cmd. +// +// Interpreter is a pointer so that unset (nil) stays distinguishable from an +// explicit `sh`: an unset interpreter lets a file extension decide, while an +// explicit one overrides it. It is deliberately left nil rather than filled in by +// SetDefaults so `flow browse` and `flow get` do not print "sh" on every shell +// executable. +func (e *ExecExecutableType) ResolveInterpreter() ExecInterpreter { + if e == nil || e.Interpreter == nil || *e.Interpreter == "" { + return DefaultInterpreter + } + return *e.Interpreter +} + +// InterpreterForFile returns the interpreter that should run the configured file. +// An explicit interpreter wins; otherwise it is inferred from the extension, and +// an unrecognised extension falls back to sh (flow's built-in POSIX interpreter), +// matching how RunFile has always treated unknown script types. +func (e *ExecExecutableType) InterpreterForFile() ExecInterpreter { + if e == nil { + return DefaultInterpreter + } + if e.Interpreter != nil && *e.Interpreter != "" { + return *e.Interpreter + } + if strings.EqualFold(filepath.Ext(e.File), ".py") { + return InterpreterPython + } + return DefaultInterpreter +} + +// InterpreterIsSet reports whether an interpreter was explicitly configured. +func (e *ExecExecutableType) InterpreterIsSet() bool { + return e != nil && e.Interpreter != nil && *e.Interpreter != "" +} + +// Validate performs semantic validation that the generated models do not enforce. +// go-jsonschema runs with --only-models, so the schema's enum is applied to +// flowfiles by the JSON schema validator but never by Go's unmarshalling; an +// invalid value would otherwise reach the runner and fail late. +func (e *ExecExecutableType) Validate() error { + if e == nil || e.Interpreter == nil { + return nil + } + switch *e.Interpreter { + case "", InterpreterSh, InterpreterPython: + default: + return fmt.Errorf("invalid interpreter %q (must be sh or python)", *e.Interpreter) + } + return nil +} diff --git a/types/executable/interpreter_test.go b/types/executable/interpreter_test.go new file mode 100644 index 00000000..3f0c4ad0 --- /dev/null +++ b/types/executable/interpreter_test.go @@ -0,0 +1,113 @@ +package executable_test + +import ( + "strings" + "testing" + + "github.com/flowexec/flow/v2/types/executable" +) + +func interpPtr(i executable.ExecInterpreter) *executable.ExecInterpreter { return &i } + +func TestResolveInterpreterDefaultsToSh(t *testing.T) { + // Interpreter is a pointer precisely so that unset stays distinguishable from + // an explicit value; the generated models apply no schema default in Go. + cases := map[string]*executable.ExecExecutableType{ + "nil spec": nil, + "unset": {Cmd: "echo hi"}, + "explicitly empty": {Cmd: "echo hi", Interpreter: interpPtr("")}, + } + for name, spec := range cases { + if got := spec.ResolveInterpreter(); got != executable.InterpreterSh { + t.Errorf("%s: ResolveInterpreter() = %q, want sh", name, got) + } + } +} + +func TestResolveInterpreterHonoursExplicitPython(t *testing.T) { + spec := &executable.ExecExecutableType{ + Cmd: "print(1)", + Interpreter: interpPtr(executable.InterpreterPython), + } + if got := spec.ResolveInterpreter(); got != executable.InterpreterPython { + t.Errorf("ResolveInterpreter() = %q, want python", got) + } +} + +func TestInterpreterForFile(t *testing.T) { + tests := []struct { + name string + file string + interpreter *executable.ExecInterpreter + want executable.ExecInterpreter + }{ + {"py extension infers python", "script.py", nil, executable.InterpreterPython}, + {"uppercase extension still infers", "SCRIPT.PY", nil, executable.InterpreterPython}, + {"sh extension stays sh", "script.sh", nil, executable.InterpreterSh}, + {"unknown extension falls back to sh", "script.unknown", nil, executable.InterpreterSh}, + { + "explicit interpreter overrides extension", + "script.sh", + interpPtr(executable.InterpreterPython), + executable.InterpreterPython, + }, + { + "explicit sh overrides a py extension", + "script.py", + interpPtr(executable.InterpreterSh), + executable.InterpreterSh, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := &executable.ExecExecutableType{File: tt.file, Interpreter: tt.interpreter} + if got := spec.InterpreterForFile(); got != tt.want { + t.Errorf("InterpreterForFile() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestInterpreterIsSet(t *testing.T) { + if (&executable.ExecExecutableType{}).InterpreterIsSet() { + t.Error("unset interpreter reported as set") + } + if (&executable.ExecExecutableType{Interpreter: interpPtr("")}).InterpreterIsSet() { + t.Error("empty interpreter reported as set") + } + spec := &executable.ExecExecutableType{Interpreter: interpPtr(executable.InterpreterPython)} + if !spec.InterpreterIsSet() { + t.Error("explicit interpreter reported as unset") + } +} + +func TestExecValidateRejectsUnknownInterpreter(t *testing.T) { + // The generated models enforce no enum, so Go-side validation is the only + // thing standing between a bad value and a late runtime failure. + spec := &executable.ExecExecutableType{Cmd: "x", Interpreter: interpPtr("ruby")} + err := spec.Validate() + if err == nil { + t.Fatal("Validate() = nil, want an error for an unknown interpreter") + } + if !strings.Contains(err.Error(), "ruby") { + t.Errorf("Validate() error = %q, want it to name the invalid value", err) + } +} + +func TestExecValidateAcceptsKnownInterpreters(t *testing.T) { + for _, i := range []*executable.ExecInterpreter{ + nil, + interpPtr(""), + interpPtr(executable.InterpreterSh), + interpPtr(executable.InterpreterPython), + } { + spec := &executable.ExecExecutableType{Cmd: "x", Interpreter: i} + if err := spec.Validate(); err != nil { + t.Errorf("Validate() = %v, want nil", err) + } + } + var nilSpec *executable.ExecExecutableType + if err := nilSpec.Validate(); err != nil { + t.Errorf("nil spec Validate() = %v, want nil", err) + } +}