From 4c793679c8e1fd27e0207c2b653d59152f9bb2b0 Mon Sep 17 00:00:00 2001 From: Kasper Junge Date: Sun, 11 Oct 2026 19:56:39 +0200 Subject: [PATCH 1/3] Fix connector editor creation and gate DMGs on a native UI journey --- .github/workflows/desktop-build.yml | 1 + desktop/Cargo.lock | 2 +- desktop/Cargo.toml | 2 +- desktop/README.md | 4 +- desktop/VERSION | 2 +- desktop/docs/RELEASING.md | 2 +- .../native/Sources/Umbod/ConnectorsView.swift | 6 +- desktop/native/Sources/Umbod/UmbodApp.swift | 48 ++++++-------- .../Sources/UmbodCore/ConnectorEditor.swift | 45 +++++++++++++ .../UmbodTests/ConnectorEditorTests.swift | 64 +++++++++++++++++++ .../Tests/UmbodTests/ContractTests.swift | 9 ++- desktop/native/UITests/ConnectorJourney.swift | 62 ++++++++++++++++++ .../project.pbxproj | 22 +++++++ .../xcschemes/ConnectorJourney.xcscheme | 5 ++ desktop/scripts/accept-macos-artifact.sh | 2 + desktop/scripts/fixture_agent.py | 21 ++++++ desktop/scripts/prepare-macos-acceptance.sh | 9 ++- desktop/scripts/run-macos-ui-tests.py | 43 +++++++++++++ 18 files changed, 310 insertions(+), 39 deletions(-) create mode 100644 desktop/native/Sources/UmbodCore/ConnectorEditor.swift create mode 100644 desktop/native/Tests/UmbodTests/ConnectorEditorTests.swift create mode 100644 desktop/native/UITests/ConnectorJourney.swift create mode 100644 desktop/native/UITests/ConnectorJourney.xcodeproj/project.pbxproj create mode 100644 desktop/native/UITests/ConnectorJourney.xcodeproj/xcshareddata/xcschemes/ConnectorJourney.xcscheme create mode 100644 desktop/scripts/fixture_agent.py create mode 100644 desktop/scripts/run-macos-ui-tests.py diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 7f03dcb..4c803be 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -113,6 +113,7 @@ jobs: path: | acceptance.log artifact-sha256.txt + ui-evidence/ if-no-files-found: error windows: runs-on: windows-latest diff --git a/desktop/Cargo.lock b/desktop/Cargo.lock index 90f2823..16a8c82 100644 --- a/desktop/Cargo.lock +++ b/desktop/Cargo.lock @@ -2356,7 +2356,7 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "umbod-gateway" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "axum", diff --git a/desktop/Cargo.toml b/desktop/Cargo.toml index 0141d56..27314e3 100644 --- a/desktop/Cargo.toml +++ b/desktop/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "umbod-gateway" -version = "0.2.1" +version = "0.2.2" edition = "2024" [lib] diff --git a/desktop/README.md b/desktop/README.md index 13cb736..779a6ff 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -53,7 +53,7 @@ OAuth supports native public clients with loopback callbacks, discovery, PKCE an - [`python/`](python/README.md): macOS host, transport adapters, Keychain, migration and tests; reuses `umbod/api` and `packages/umbod-sdk`. - `backend/src/`: Windows Rust gateway, profiles, platform adapters, client setup and OAuth; `backend/tests/`: integration tests. -- `native/`: SwiftUI app and Swift contract/lifecycle tests; `UmbodCore/DesktopUpdate.swift` validates update metadata and `Umbod/UpdatesView.swift` owns update preferences and the install guidance. +- `native/`: SwiftUI app and Swift contract/lifecycle tests; `UmbodCore/ConnectorEditor.swift` owns the actual editor submission and retry state; `UITests/` contains the XCUITest project for packaged-app user journeys; `UmbodCore/DesktopUpdate.swift` validates update metadata and `Umbod/UpdatesView.swift` owns update preferences and the install guidance. - `windows/`: shared .NET Core contracts, Core tests, and WPF app; `Umbod.Core/DesktopUpdate.cs` validates update metadata and `Umbod.Windows/UpdatesView.cs` provides update checks and install guidance. - `scripts/`: local builds, packaging, isolated fixtures, and release policy/metadata. - `VERSION`: independent desktop version; keep Cargo.toml and the gateway entry in Cargo.lock synchronized. Bundle, package and .NET versions derive from it. @@ -62,3 +62,5 @@ OAuth supports native public clients with loopback callbacks, discovery, PKCE an The [upstream Windows run](https://github.com/kasperjunge/umbod-desktop/actions/runs/37907078176) is historical source evidence supplied for this import. It is **not** a CI result for this target repository or these integration changes. Release packaging collects dependency inventories and notices through `scripts/dependency-notices.py`; pinned supplemental sources are documented in `licenses/README.md`. `scripts/verify-python-bundle.py` rejects editable installs and imports outside the bundled interpreter before signing and again after installation. `scripts/audit-macos-bundle.py` verifies ARM64 slices, the macOS 14 deployment floor, required resources, executable permissions, internal symlinks and relocatable native dependencies. `Resources/build-metadata.json` records source and toolchain inputs. `scripts/prepare-macos-acceptance.sh` packages a precompiled test harness and fixtures; the separate `macos-acceptance` job runs `scripts/accept-macos-artifact.sh` on current macOS ARM64 with no checkout, using the DMG’s interpreter and the unchanged release identity. This required job also runs in the manual release workflow before publication. Local UI tests retain disposable bundle identities to protect user preferences. `scripts/install-smoke.py` installs the Mac DMG into disposable storage and verifies the packaged connector/agent flow, native window lifecycle, and signature integrity before and after execution. It never uses the login Keychain. + +The acceptance kit also includes a precompiled XCUITest runner. `scripts/run-macos-ui-tests.py` runs it with `xcodebuild test-without-building` on the disposable hosted Mac. It clicks Add connector, saves a local fixture, discovers and enables its echo tool, calls it through the packaged agent bridge, edits the connector, and verifies persistence after relaunch. No API shortcut creates the connector. The required gate rejects failed or skipped journeys and preserves `.xcresult` diagnostics and screenshots. The runner uses temporary data and fixture credentials, never the login Keychain. It deliberately refuses local release-identity execution; routine local checks remain isolated. The app is still built with SwiftPM, and the separate test project does not build or modify it. diff --git a/desktop/VERSION b/desktop/VERSION index 0c62199..ee1372d 100644 --- a/desktop/VERSION +++ b/desktop/VERSION @@ -1 +1 @@ -0.2.1 +0.2.2 diff --git a/desktop/docs/RELEASING.md b/desktop/docs/RELEASING.md index 9a80133..0108d87 100644 --- a/desktop/docs/RELEASING.md +++ b/desktop/docs/RELEASING.md @@ -6,7 +6,7 @@ Desktop is versioned independently of chart, images and SDK. Update `desktop/VER 1. Resolve the source-license and dependency-notice blockers in [provenance](PROVENANCE.md), review the changes, and merge through the normal process. Do not copy historical personal docs or evidence. 2. Configure the GitHub `desktop-release` environment with required reviewers and a deployment branch rule permitting only `main`. The workflow also validates repository, ref, event, version, rights attestation, and exact confirmation. No signing secret is required for unsigned builds. No credentials belong in website code or manifests. -3. Run native desktop CI on the reviewed source. PR/push CI only uploads artifacts and has read-only permissions. Mac uses an Apple Silicon runner and Swift 6; Windows builds native x64 Rust and WPF, including its packaged smoke test. The Mac artifact-only acceptance job must also pass: it uses the exact DMG, no checkout or compiler, and an unchanged release app launched through LaunchServices on a fresh hosted runner. Both platform builds must pass; a local cross-build does not substitute for Windows tests. +3. Run native desktop CI on the reviewed source. PR/push CI only uploads artifacts and has read-only permissions. Mac uses an Apple Silicon runner and Swift 6; Windows builds native x64 Rust and WPF, including its packaged smoke test. The Mac artifact-only acceptance job must also pass: it uses the exact DMG, no checkout or compiler, and an unchanged release app launched through LaunchServices on a fresh hosted runner. Its precompiled XCUITest journey must pass without skipped tests, including UI connector creation, tool enablement and a real agent call; inspect the uploaded `.xcresult` on failure. Both platform builds must pass; a local cross-build does not substitute for Windows tests. 4. From `main`, explicitly dispatch **Desktop manual prerelease** with version `X.Y.Z`, mode `unsigned-prerelease`, rights confirmation enabled, and confirmation text `release desktop-vX.Y.Z unsigned-prerelease`. The workflow rebuilds/tests the dispatch SHA. It creates a new desktop tag, an explicitly UNSIGNED GitHub prerelease, versioned assets, release notes, SHA256SUMS and desktop-downloads.json. It never promotes repository-wide latest or overwrites existing tags/releases/assets. 5. Download the `website-manifest-update` artifact and follow [APPLY-MANIFEST.md](APPLY-MANIFEST.md). This is a PR-ready file tree; publication does not automatically edit or deploy the website. diff --git a/desktop/native/Sources/Umbod/ConnectorsView.swift b/desktop/native/Sources/Umbod/ConnectorsView.swift index 74f68de..76852f0 100644 --- a/desktop/native/Sources/Umbod/ConnectorsView.swift +++ b/desktop/native/Sources/Umbod/ConnectorsView.swift @@ -16,7 +16,7 @@ struct ServersView: View { Button { selectedConnector = nil } label: { Label("All connectors", systemImage: "chevron.left") } ConnectorToolsView(connector: connector, manage: { sheet = .manage(id) }).id(id) } else { - Button { sheet = .add } label: { Label("Add connector", systemImage: "plus") }.buttonStyle(BrandButtonStyle(primary: true)) + Button { sheet = .add } label: { Label("Add connector", systemImage: "plus") }.buttonStyle(BrandButtonStyle(primary: true)).accessibilityIdentifier("connectors.add") if model.servers.isEmpty { ContentUnavailableView("No connectors yet", systemImage: "externaldrive.badge.plus", description: Text("Add a connector to make its tools available to your apps.")) } else { @@ -39,7 +39,7 @@ struct ServersView: View { Text("\(tools.filter { model.allowedTools.contains($0["name"] as? String ?? "") }.count) / \(tools.count) tools enabled").font(.caption).foregroundStyle(Brand.muted) Image(systemName: "chevron.right").foregroundStyle(Brand.muted) }.padding(.vertical, 6).contentShape(Rectangle()) - }.buttonStyle(.plain) + }.buttonStyle(.plain).accessibilityIdentifier("connector.row.\(server["name"] as? String ?? "")") Menu { Button("Connection settings") { sheet = .manage(id) } Button("Connect / rediscover") { model.act(["action":"connect", "id":id]) }.disabled(model.busy) @@ -191,6 +191,6 @@ struct ConnectorToolsView: View { } private func permissionToggle(_ name: String) -> some View { Toggle("Allow \(String(name.dropFirst(id.count + 1)))", isOn: Binding(get: { model.allowedTools.contains(name) }, set: { model.act(["action":"grant", "tool":name, "allow":$0]) })) - .labelsHidden().toggleStyle(.switch).controlSize(.small).disabled(model.busy) + .accessibilityIdentifier("tool.allow.\(String(name.dropFirst(id.count + 1)))").labelsHidden().toggleStyle(.switch).controlSize(.small).disabled(model.busy) } } diff --git a/desktop/native/Sources/Umbod/UmbodApp.swift b/desktop/native/Sources/Umbod/UmbodApp.swift index 1b412c8..ec8fbad 100644 --- a/desktop/native/Sources/Umbod/UmbodApp.swift +++ b/desktop/native/Sources/Umbod/UmbodApp.swift @@ -224,48 +224,40 @@ struct Panel: View { struct ServerEditor: View { @EnvironmentObject var model: AppModel @Environment(\.dismiss) var dismiss - @State var draft: ServerDraft - @State private var credentialName = "Authorization" - @State private var credential = "" + @StateObject private var editor: ConnectorEditor + init(draft: ServerDraft) { _editor = StateObject(wrappedValue: ConnectorEditor(draft: draft)) } @State private var localError = "" - @State private var saving = false var body: some View { VStack(alignment: .leading, spacing: 18) { - Text(draft.id.isEmpty ? "Add connector" : "Edit connector").font(.title2.bold()) + Text(editor.draft.id.isEmpty ? "Add connector" : "Edit connector").font(.title2.bold()) ScrollView { VStack(alignment: .leading, spacing: 16) { - TextField("Name", text: $draft.name) - Picker("Connection", selection: $draft.transport) { Text("Local process · stdio").tag("stdio"); Text("Remote · Streamable HTTP").tag("http") } - if draft.transport == "stdio" { - TextField("Executable (absolute path)", text: $draft.command) + TextField("Name", text: $editor.draft.name).accessibilityIdentifier("connector.name") + Picker("Connection", selection: $editor.draft.transport) { Text("Local process · stdio").tag("stdio"); Text("Remote · Streamable HTTP").tag("http") } + if editor.draft.transport == "stdio" { + TextField("Executable (absolute path)", text: $editor.draft.command).accessibilityIdentifier("connector.command") Text("Arguments · one per line, no shell quoting").font(.caption).foregroundStyle(Brand.muted) - TextEditor(text: $draft.arguments).font(.system(.body, design: .monospaced)).frame(height: 65).padding(8).background(Brand.canvas).overlay(RoundedRectangle(cornerRadius: 8).stroke(Brand.border)) - } else { TextField("Endpoint URL", text: $draft.url) } + TextEditor(text: $editor.draft.arguments).accessibilityIdentifier("connector.arguments").font(.system(.body, design: .monospaced)).frame(height: 65).padding(8).background(Brand.canvas).overlay(RoundedRectangle(cornerRadius: 8).stroke(Brand.border)) + } else { TextField("Endpoint URL", text: $editor.draft.url) } VStack(alignment: .leading, spacing: 12) { Text("Optional credential · stored only in Keychain").font(.system(size: 14, weight: .semibold)).padding(.top, 8) - TextField(draft.transport == "stdio" ? "Environment variable name" : "Header name (Authorization, X-API-Key…)", text: $credentialName) - SecureField(draft.transport == "stdio" ? "Secret environment value" : "Header value (include Bearer prefix for a bearer token)", text: $credential) + TextField(editor.draft.transport == "stdio" ? "Environment variable name" : "Header name (Authorization, X-API-Key…)", text: $editor.credentialName) + SecureField(editor.draft.transport == "stdio" ? "Secret environment value" : "Header value (include Bearer prefix for a bearer token)", text: $editor.credential) Text("Leave blank for no change. Use browser sign-in after saving for OAuth.").font(.caption).foregroundStyle(Brand.muted) } - DisclosureGroup("Advanced credential references") { Text("JSON maps names to upstream: Keychain references. Never put secret values here.").font(.caption); TextField("Headers", text: $draft.headerReferences); TextField("Environment", text: $draft.environmentReferences) } - }.padding(2) }.textFieldStyle(BrandTextFieldStyle()) - if !draft.id.isEmpty { Text("Saving edits disconnects this connector and resets its shared tool permissions. Reconnect and enable the tools you want.").font(.caption).foregroundStyle(Brand.muted) } + DisclosureGroup("Advanced credential references") { Text("JSON maps names to upstream: Keychain references. Never put secret values here.").font(.caption); TextField("Headers", text: $editor.draft.headerReferences); TextField("Environment", text: $editor.draft.environmentReferences) } + }.padding(2) }.textFieldStyle(BrandTextFieldStyle()).disabled(editor.saving) + if !editor.draft.id.isEmpty { Text("Saving edits disconnects this connector and resets its shared tool permissions. Reconnect and enable the tools you want.").font(.caption).foregroundStyle(Brand.muted) } if !localError.isEmpty { Text(localError).foregroundStyle(Brand.danger) } - HStack { Spacer(); Button("Cancel") { dismiss() }; Button(saving ? "Saving…" : "Save connector") { save() }.buttonStyle(BrandButtonStyle(primary: true)).disabled(saving || draft.name.isEmpty) } + HStack { Spacer(); Button("Cancel") { dismiss() }; Button(editor.saving ? "Saving…" : "Save connector") { save() }.buttonStyle(BrandButtonStyle(primary: true)).disabled(editor.saving || editor.draft.name.isEmpty).accessibilityIdentifier("connector.save") } }.padding(24).frame(width: 620, height: 660).modifier(BrandAppearance()) } func save() { - Task { saving = true; defer { saving = false } + Task { do { - if draft.id.isEmpty { draft.id = UUID().uuidString.replacingOccurrences(of: "-", with: "").lowercased() } - var payload = try draft.payload() - if !credential.isEmpty { - guard !credentialName.isEmpty else { throw ContractError.message("Enter a credential header or environment name") } - let reference = "upstream:\(draft.id):\(credentialName)" - _ = try await model.backend.request(["action": "secret_set", "reference": reference, "value": credential]); credential = "" - let key = draft.transport == "stdio" ? "env" : "headers" - var references = payload[key] as? [String: String] ?? [:]; references[credentialName] = reference; payload[key] = references - } - _ = try await model.backend.request(["action": "server_save", "server": payload]); try await model.refresh(); dismiss() + localError = "" + try await editor.save { try await model.backend.request($0) } + try await model.refresh() + dismiss() } catch { localError = error.localizedDescription } } } diff --git a/desktop/native/Sources/UmbodCore/ConnectorEditor.swift b/desktop/native/Sources/UmbodCore/ConnectorEditor.swift new file mode 100644 index 0000000..5926481 --- /dev/null +++ b/desktop/native/Sources/UmbodCore/ConnectorEditor.swift @@ -0,0 +1,45 @@ +import Combine +import Foundation + +/// Owns the submission used by the native editor. Only the backend assigns connector IDs. +@MainActor public final class ConnectorEditor: ObservableObject { + @Published public var draft: ServerDraft + @Published public var credentialName = "Authorization" + @Published public var credential = "" + @Published public private(set) var saving = false + private let credentialNamespace: String + + public init(draft: ServerDraft) { + self.draft = draft + credentialNamespace = draft.id.isEmpty ? UUID().uuidString.lowercased() : draft.id + } + + public func save(request: @MainActor ([String: Any]) async throws -> [String: Any]) async throws { + guard !saving else { throw ContractError.message("A connector save is already in progress") } + saving = true + defer { saving = false } + var payload = try draft.payload() + if !credential.isEmpty { + guard !credentialName.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw ContractError.message("Enter a credential header or environment name") + } + let reference = "upstream:\(credentialNamespace):\(credentialName)" + let key = draft.transport == "stdio" ? "env" : "headers" + var references = payload[key] as? [String: String] ?? [:] + references[credentialName] = reference + let encoded = try JSONText.encode(references) + _ = try await request(["action": "secret_set", "reference": reference, "value": credential]) + // Retain the stored reference if saving the connector fails and the user retries. + if key == "env" { draft.environmentReferences = encoded } + else { draft.headerReferences = encoded } + credential = "" + payload[key] = references + } + let response = try await request(["action": "server_save", "server": payload]) + guard let server = response["server"] as? [String: Any], let id = server["id"] as? String, !id.isEmpty else { + throw ContractError.message("The backend did not return a connector identity. Refresh the connector list before retrying.") + } + // A refresh failure after a successful save must not turn a retry into another create. + draft.id = id + } +} diff --git a/desktop/native/Tests/UmbodTests/ConnectorEditorTests.swift b/desktop/native/Tests/UmbodTests/ConnectorEditorTests.swift new file mode 100644 index 0000000..b217acf --- /dev/null +++ b/desktop/native/Tests/UmbodTests/ConnectorEditorTests.swift @@ -0,0 +1,64 @@ +import Foundation +import UmbodCore + +@MainActor func connectorEditorTests() async throws { + var draft = ServerDraft() + draft.name = "New connector" + let editor = ConnectorEditor(draft: draft) + try await editor.save { request in + precondition(request["action"] as? String == "server_save") + precondition((request["server"] as! [String: Any])["id"] as? String == "", "New connectors must use create, never a locally generated update ID") + return ["server": ["id": "canonical"]] + } + precondition(editor.draft.id == "canonical") + try await editor.save { request in + precondition((request["server"] as! [String: Any])["id"] as? String == "canonical", "Edits and retries after refresh failures must preserve the backend identity") + return ["server": ["id": "canonical"]] + } + for transport in ["stdio", "http"] { + draft.transport = transport + let editor = ConnectorEditor(draft: draft) + editor.credentialName = "FIXTURE" + editor.credential = "fixture-only" + var stored = "" + var calls = 0 + do { + try await editor.save { request in + calls += 1 + if request["action"] as? String == "secret_set" { + stored = request["reference"] as! String + return [:] + } + let payload = request["server"] as! [String: Any] + precondition(payload["id"] as? String == "") + precondition((payload[transport == "stdio" ? "env" : "headers"] as! [String: String])["FIXTURE"] == stored) + throw ContractError.message("Fixture save failure") + } + preconditionFailure("Expected failure") + } catch { precondition(error.localizedDescription == "Fixture save failure") } + precondition(calls == 2 && !stored.isEmpty && editor.credential.isEmpty && editor.draft.id.isEmpty && !editor.saving) + try await editor.save { request in + precondition(request["action"] as? String == "server_save", "Retry must reuse the stored credential") + let payload = request["server"] as! [String: Any] + precondition(payload["id"] as? String == "") + precondition((payload[transport == "stdio" ? "env" : "headers"] as! [String: String])["FIXTURE"] == stored) + return ["server": ["id": "pending:fixture"]] + } + precondition(editor.draft.id == "pending:fixture") + } + let invalid = ConnectorEditor(draft: draft) + invalid.credential = "fixture-only" + invalid.draft.headerReferences = "[]" + do { + try await invalid.save { _ in preconditionFailure("Validation must precede credential writes") } + preconditionFailure("Expected validation failure") + } catch {} + precondition(invalid.credential == "fixture-only" && !invalid.saving) + invalid.draft.headerReferences = "{}" + do { + try await invalid.save { _ in throw ContractError.message("Fixture vault failure") } + preconditionFailure("Expected vault failure") + } catch {} + precondition(invalid.credential == "fixture-only" && invalid.draft.headerReferences == "{}") + ContractTests.log("PASS: editor create/edit identities, credential failure/retry, pending identity and validation") +} diff --git a/desktop/native/Tests/UmbodTests/ContractTests.swift b/desktop/native/Tests/UmbodTests/ContractTests.swift index 198e056..5e7417e 100644 --- a/desktop/native/Tests/UmbodTests/ContractTests.swift +++ b/desktop/native/Tests/UmbodTests/ContractTests.swift @@ -4,6 +4,7 @@ import UmbodCore static func log(_ value: String) { FileHandle.standardOutput.write(Data((value + "\n").utf8)) } @MainActor static func main() async throws { try updateTests() + try await connectorEditorTests() if CommandLine.arguments.contains("--updates-only") { return } precondition(ActivationState([:]).stage == .choose) let journey: [String: Any] = ["client":"c", "server":"s", "configured":false, "activated":false] @@ -52,8 +53,12 @@ import UmbodCore log("PASS: native startup exposes actionable saved-port conflict") server.command = executable.deletingLastPathComponent().deletingLastPathComponent().appendingPathComponent("Resources/python/bin/python3").path server.arguments = "-B\n" + desktop.appendingPathComponent("scripts/fixture_stdio.py").path - let saved = try await backend.request(["action": "server_save", "server": try server.payload()]) - let savedServer = saved["server"] as! [String: Any] + let editor = ConnectorEditor(draft: server) + editor.credentialName = "UMBOD_FIXTURE_SECRET" + editor.credential = "fixture-only" + editor.draft.arguments += "\n--require-env" + try await editor.save { try await backend.request($0) } + let savedServer: [String: Any] = ["id": editor.draft.id] _ = try await backend.request(["action": "connect", "id": savedServer["id"]!]) let connected = try await backend.request(["action": "status"]) precondition((connected["tools"] as? [Any])?.count == 2) diff --git a/desktop/native/UITests/ConnectorJourney.swift b/desktop/native/UITests/ConnectorJourney.swift new file mode 100644 index 0000000..3d6a59f --- /dev/null +++ b/desktop/native/UITests/ConnectorJourney.swift @@ -0,0 +1,62 @@ +import XCTest + +/// Runs against the installed DMG app, never an app rebuilt by this Xcode project. +final class ConnectorJourney: XCTestCase { + @MainActor func testAddEditAndReopenConnector() throws { + continueAfterFailure = false + let env = ProcessInfo.processInfo.environment + let appPath = try XCTUnwrap(env["UMBOD_UI_APP"]) + let fixture = try XCTUnwrap(env["UMBOD_UI_FIXTURE"]) + let directory = FileManager.default.temporaryDirectory.appendingPathComponent("umbod-test-ui-" + UUID().uuidString) + let app = XCUIApplication(url: URL(fileURLWithPath: appPath)) + app.launchArguments = ["--data-dir", directory.path, "--test-credentials"] + defer { + let screenshot = XCTAttachment(screenshot: app.screenshot()) + screenshot.lifetime = .keepAlways + add(screenshot) + app.terminate() + try? FileManager.default.removeItem(at: directory) + } + app.launch() + XCTAssertTrue(app.staticTexts["Gateway running"].waitForExistence(timeout: 60)) + app.buttons["Connectors"].click() + app.buttons["connectors.add"].click() + let name = app.textFields["connector.name"] + XCTAssertTrue(name.waitForExistence(timeout: 5)) + name.click(); name.typeText("UI fixture") + let command = app.textFields["connector.command"] + command.click(); command.typeText(appPath + "/Contents/Resources/python/bin/python3") + let arguments = app.textViews["connector.arguments"] + arguments.click(); arguments.typeText("-B\n" + fixture) + app.buttons["connector.save"].click() + let row = app.buttons["connector.row.UI fixture"] + XCTAssertTrue(row.waitForExistence(timeout: 60), "Saving a new connector must close the editor and show the created connector") + row.click() + app.buttons["Connection settings"].click() + app.buttons["Connect / rediscover"].click() + XCTAssertTrue(app.staticTexts["Connected"].waitForExistence(timeout: 60)) + app.buttons["Done"].click() + let toggle = app.descendants(matching: .any).matching(identifier: "tool.allow.echo").firstMatch + XCTAssertTrue(toggle.waitForExistence(timeout: 15)) + toggle.click() + let agent = Process() + agent.executableURL = URL(fileURLWithPath: appPath + "/Contents/Resources/python/bin/python3") + let profile = try String(contentsOfFile: appPath + "/Contents/Resources/umbod-profile", encoding: .utf8).trimmingCharacters(in: .whitespacesAndNewlines) + agent.arguments = ["-I", "-B", try XCTUnwrap(env["UMBOD_UI_AGENT"]), appPath + "/Contents/MacOS/umbod-desktop", directory.path, profile] + try agent.run() + agent.waitUntilExit() + XCTAssertEqual(agent.terminationStatus, 0, "An agent must be able to call the tool enabled through the UI") + app.buttons["Connection settings"].click() + app.buttons["Edit connection"].click() + XCTAssertTrue(name.waitForExistence(timeout: 5)) + name.click(); name.typeKey("a", modifierFlags: .command); name.typeText("UI fixture edited") + app.buttons["connector.save"].click() + XCTAssertTrue(app.buttons["connector.save"].waitForNonExistence(timeout: 60)) + app.terminate() + app.launch() + XCTAssertTrue(app.staticTexts["Gateway running"].waitForExistence(timeout: 60)) + app.buttons["Connectors"].click() + XCTAssertTrue(app.buttons["connector.row.UI fixture edited"].waitForExistence(timeout: 15)) + XCTAssertFalse(app.buttons["connector.row.UI fixture"].exists, "Editing must not create a duplicate") + } +} diff --git a/desktop/native/UITests/ConnectorJourney.xcodeproj/project.pbxproj b/desktop/native/UITests/ConnectorJourney.xcodeproj/project.pbxproj new file mode 100644 index 0000000..a68386c --- /dev/null +++ b/desktop/native/UITests/ConnectorJourney.xcodeproj/project.pbxproj @@ -0,0 +1,22 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = {}; + objectVersion = 56; + objects = { + 100000000000000000000001 = {isa = PBXProject; attributes = {LastUpgradeCheck = 1600;}; buildConfigurationList = 100000000000000000000010; compatibilityVersion = "Xcode 14.0"; developmentRegion = en; knownRegions = (en, Base); mainGroup = 100000000000000000000002; productRefGroup = 100000000000000000000003; projectDirPath = ""; projectRoot = ""; targets = (100000000000000000000004);}; + 100000000000000000000002 = {isa = PBXGroup; children = (100000000000000000000005, 100000000000000000000003); sourceTree = "";}; + 100000000000000000000003 = {isa = PBXGroup; children = (100000000000000000000006); name = Products; sourceTree = "";}; + 100000000000000000000004 = {isa = PBXNativeTarget; buildConfigurationList = 100000000000000000000011; buildPhases = (100000000000000000000007, 100000000000000000000008); buildRules = (); dependencies = (); name = ConnectorJourney; productName = ConnectorJourney; productReference = 100000000000000000000006; productType = "com.apple.product-type.bundle.ui-testing";}; + 100000000000000000000005 = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectorJourney.swift; sourceTree = "";}; + 100000000000000000000006 = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = ConnectorJourney.xctest; sourceTree = BUILT_PRODUCTS_DIR;}; + 100000000000000000000007 = {isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = (100000000000000000000009); runOnlyForDeploymentPostprocessing = 0;}; + 100000000000000000000008 = {isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = (); runOnlyForDeploymentPostprocessing = 0;}; + 100000000000000000000009 = {isa = PBXBuildFile; fileRef = 100000000000000000000005;}; + 100000000000000000000010 = {isa = XCConfigurationList; buildConfigurations = (100000000000000000000012); defaultConfigurationIsVisible = 0; defaultConfigurationName = Debug;}; + 100000000000000000000011 = {isa = XCConfigurationList; buildConfigurations = (100000000000000000000013); defaultConfigurationIsVisible = 0; defaultConfigurationName = Debug;}; + 100000000000000000000012 = {isa = XCBuildConfiguration; name = Debug; buildSettings = {SDKROOT = macosx; MACOSX_DEPLOYMENT_TARGET = 14.0; SWIFT_VERSION = 6.0; ARCHS = arm64; ONLY_ACTIVE_ARCH = YES;};}; + 100000000000000000000013 = {isa = XCBuildConfiguration; name = Debug; buildSettings = {PRODUCT_BUNDLE_IDENTIFIER = app.umbod.tests.ConnectorJourney; PRODUCT_NAME = "$(TARGET_NAME)"; GENERATE_INFOPLIST_FILE = YES; CODE_SIGN_IDENTITY = "-"; CODE_SIGN_STYLE = Manual; ENABLE_HARDENED_RUNTIME = NO;};}; + }; + rootObject = 100000000000000000000001; +} diff --git a/desktop/native/UITests/ConnectorJourney.xcodeproj/xcshareddata/xcschemes/ConnectorJourney.xcscheme b/desktop/native/UITests/ConnectorJourney.xcodeproj/xcshareddata/xcschemes/ConnectorJourney.xcscheme new file mode 100644 index 0000000..5468bbd --- /dev/null +++ b/desktop/native/UITests/ConnectorJourney.xcodeproj/xcshareddata/xcschemes/ConnectorJourney.xcscheme @@ -0,0 +1,5 @@ + + + + + diff --git a/desktop/scripts/accept-macos-artifact.sh b/desktop/scripts/accept-macos-artifact.sh index 80d204d..b2282eb 100755 --- a/desktop/scripts/accept-macos-artifact.sh +++ b/desktop/scripts/accept-macos-artifact.sh @@ -2,6 +2,7 @@ set -euo pipefail # No checkout, compiler or externally installed Python is used here. MODE="${2:-release}" +EVIDENCE="${UMBOD_UI_EVIDENCE:-$PWD/ui-evidence}" case "$MODE" in release|local-fixture) ;; *) exit 2 ;; esac KIT="$(cd "$(dirname "$0")/.." && pwd)" DMG="${1:?DMG required}" @@ -31,6 +32,7 @@ UMBOD_TEST_APP="$APP" UMBOD_TEST_RESOURCES="$KIT" "$KIT/UmbodTests" # This mode refuses to run on a developer account. No bundle ID changes or re-signing. if [[ "$MODE" == release ]]; then "$PYTHON" -I -B "$KIT/scripts/native-smoke.py" "$APP" --release-identity + "$PYTHON" -I -B "$KIT/scripts/run-macos-ui-tests.py" "$APP" "$KIT" "$EVIDENCE" else "$PYTHON" -I -B "$KIT/scripts/native-smoke.py" "$APP" fi diff --git a/desktop/scripts/fixture_agent.py b/desktop/scripts/fixture_agent.py new file mode 100644 index 0000000..493c213 --- /dev/null +++ b/desktop/scripts/fixture_agent.py @@ -0,0 +1,21 @@ +"""Call the UI-enabled echo tool through the packaged agent bridge.""" +import asyncio +import sys +from fastmcp import Client +from fastmcp.client.transports import StdioTransport + + +async def main(): + async with Client(StdioTransport(command=sys.argv[1], args=[ + "bridge", "--profile", sys.argv[3], "--data-dir", sys.argv[2], + "--test-credentials", + ]), mode="legacy") as client: + async with asyncio.timeout(30): + while not (tools := await client.list_tools()): + await asyncio.sleep(.1) + assert len(tools) == 1, "Only the UI-enabled tool should be published" + result = await client.call_tool(tools[0].name, {"text": "ui-journey-fixture"}) + assert "ui-journey-fixture" in str(result) + + +asyncio.run(asyncio.wait_for(main(), timeout=45)) diff --git a/desktop/scripts/prepare-macos-acceptance.sh b/desktop/scripts/prepare-macos-acceptance.sh index 6334311..a789689 100755 --- a/desktop/scripts/prepare-macos-acceptance.sh +++ b/desktop/scripts/prepare-macos-acceptance.sh @@ -5,5 +5,12 @@ DEST="${1:?test kit output directory required}" mkdir -p "$DEST/scripts" "$DEST/tests/fixtures" swift build --package-path native --product UmbodTests cp native/.build/debug/UmbodTests "$DEST/UmbodTests" -cp scripts/{accept-macos-artifact.sh,audit-macos-bundle.py,verify-python-bundle.py,native-smoke.py,fixture_stdio.py} "$DEST/scripts/" +cp scripts/{accept-macos-artifact.sh,audit-macos-bundle.py,verify-python-bundle.py,native-smoke.py,fixture_stdio.py,fixture_agent.py,run-macos-ui-tests.py} "$DEST/scripts/" cp tests/fixtures/update-manifest.json "$DEST/tests/fixtures/" + +# Only the test runner is compiled here. XCUITest launches the separately packaged app. +xcodebuild build-for-testing -project native/UITests/ConnectorJourney.xcodeproj \ + -scheme ConnectorJourney -destination 'platform=macOS,arch=arm64' \ + -derivedDataPath "$DEST/ui-build" +mv "$DEST/ui-build/Build/Products" "$DEST/ui-tests" +rm -rf "$DEST/ui-build" diff --git a/desktop/scripts/run-macos-ui-tests.py b/desktop/scripts/run-macos-ui-tests.py new file mode 100644 index 0000000..d5593cf --- /dev/null +++ b/desktop/scripts/run-macos-ui-tests.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Run precompiled XCUITests against the exact installed artifact on disposable CI.""" +import json +import os +from pathlib import Path +import plistlib +import subprocess +import sys + +if os.environ.get("GITHUB_ACTIONS") != "true" or os.environ.get("RUNNER_ENVIRONMENT") != "github-hosted": + raise SystemExit("Release-identity UI tests require a disposable GitHub-hosted Mac") +app, kit, evidence = map(lambda value: Path(value).resolve(), sys.argv[1:]) +evidence.mkdir(parents=True, exist_ok=True) +products = kit / "ui-tests" +plans = list(products.glob("*.xctestrun")) +if len(plans) != 1: + raise SystemExit("Expected exactly one precompiled UI test plan") +plan = plistlib.loads(plans[0].read_bytes()) +target = plan["ConnectorJourney"] +target["EnvironmentVariables"].update({ + "UMBOD_UI_APP": str(app), + "UMBOD_UI_FIXTURE": str(kit / "scripts/fixture_stdio.py"), + "UMBOD_UI_AGENT": str(kit / "scripts/fixture_agent.py"), +}) +target["TestTimeoutsEnabled"] = True +target["MaximumTestExecutionTimeAllowance"] = 180 +target["UserAttachmentLifetime"] = "keepAlways" +configured = products / "configured.xctestrun" +configured.write_bytes(plistlib.dumps(plan)) +result = evidence / "connector-journey.xcresult" +subprocess.run([ + "/usr/bin/xcodebuild", "test-without-building", "-xctestrun", str(configured), + "-destination", "platform=macOS,arch=arm64", "-parallel-testing-enabled", "NO", + "-resultBundlePath", str(result), +], check=True) +summary = subprocess.check_output([ + "/usr/bin/xcrun", "xcresulttool", "get", "test-results", "summary", "--path", str(result), +]) +(evidence / "summary.json").write_bytes(summary) +report = json.loads(summary) +if report.get("passedTests") != 1 or report.get("failedTests") != 0 or report.get("skippedTests") != 0: + raise SystemExit("Expected one passing UI journey with no failures or skipped tests") +print("PASS: packaged UI add → discover → enable → agent call → edit → relaunch") From 6d9f46cc16bcf9b105cedcd9ede3cf56d4b3d9ed Mon Sep 17 00:00:00 2001 From: Kasper Junge Date: Sun, 11 Oct 2026 20:03:37 +0200 Subject: [PATCH 2/3] Support isolated local UI runs and use a shared temporary fixture root --- desktop/README.md | 11 ++++- desktop/native/UITests/ConnectorJourney.swift | 10 ++--- desktop/scripts/run-macos-ui-tests.py | 41 ++++++++++++++++--- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/desktop/README.md b/desktop/README.md index 779a6ff..e5d1182 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -63,4 +63,13 @@ The [upstream Windows run](https://github.com/kasperjunge/umbod-desktop/actions/ Release packaging collects dependency inventories and notices through `scripts/dependency-notices.py`; pinned supplemental sources are documented in `licenses/README.md`. `scripts/verify-python-bundle.py` rejects editable installs and imports outside the bundled interpreter before signing and again after installation. `scripts/audit-macos-bundle.py` verifies ARM64 slices, the macOS 14 deployment floor, required resources, executable permissions, internal symlinks and relocatable native dependencies. `Resources/build-metadata.json` records source and toolchain inputs. `scripts/prepare-macos-acceptance.sh` packages a precompiled test harness and fixtures; the separate `macos-acceptance` job runs `scripts/accept-macos-artifact.sh` on current macOS ARM64 with no checkout, using the DMG’s interpreter and the unchanged release identity. This required job also runs in the manual release workflow before publication. Local UI tests retain disposable bundle identities to protect user preferences. `scripts/install-smoke.py` installs the Mac DMG into disposable storage and verifies the packaged connector/agent flow, native window lifecycle, and signature integrity before and after execution. It never uses the login Keychain. -The acceptance kit also includes a precompiled XCUITest runner. `scripts/run-macos-ui-tests.py` runs it with `xcodebuild test-without-building` on the disposable hosted Mac. It clicks Add connector, saves a local fixture, discovers and enables its echo tool, calls it through the packaged agent bridge, edits the connector, and verifies persistence after relaunch. No API shortcut creates the connector. The required gate rejects failed or skipped journeys and preserves `.xcresult` diagnostics and screenshots. The runner uses temporary data and fixture credentials, never the login Keychain. It deliberately refuses local release-identity execution; routine local checks remain isolated. The app is still built with SwiftPM, and the separate test project does not build or modify it. +The acceptance kit also includes a precompiled XCUITest runner. `scripts/run-macos-ui-tests.py` runs it with `xcodebuild test-without-building` on the disposable hosted Mac. It clicks Add connector, saves a local fixture, discovers and enables its echo tool, calls it through the packaged agent bridge, edits the connector, and verifies persistence after relaunch. No API shortcut creates the connector. The required gate rejects failed or skipped journeys and preserves `.xcresult` diagnostics and screenshots. The runner uses temporary data and fixture credentials, never the login Keychain. Local UI testing uses `--local-fixture` to copy and re-sign the app with a unique bundle identity; release-identity execution remains restricted to disposable CI. The app is still built with SwiftPM, and the separate test project does not build or modify it. + +To run the same click-through journey locally without touching installed apps or profiles: + +```sh +bash desktop/scripts/prepare-macos-acceptance.sh /tmp/umbod-ui-kit +python3 desktop/scripts/run-macos-ui-tests.py "desktop/dist/Umbod Dev.app" /tmp/umbod-ui-kit /tmp/umbod-ui-evidence --local-fixture +``` + +Build the dev bundle first. Use a new evidence directory per run. Xcode UI testing may request macOS automation/accessibility access on first use; do not bypass system controls. The CI gate still tests the unchanged release artifact. diff --git a/desktop/native/UITests/ConnectorJourney.swift b/desktop/native/UITests/ConnectorJourney.swift index 3d6a59f..68fa3b0 100644 --- a/desktop/native/UITests/ConnectorJourney.swift +++ b/desktop/native/UITests/ConnectorJourney.swift @@ -7,18 +7,18 @@ final class ConnectorJourney: XCTestCase { let env = ProcessInfo.processInfo.environment let appPath = try XCTUnwrap(env["UMBOD_UI_APP"]) let fixture = try XCTUnwrap(env["UMBOD_UI_FIXTURE"]) - let directory = FileManager.default.temporaryDirectory.appendingPathComponent("umbod-test-ui-" + UUID().uuidString) + let directory = URL(fileURLWithPath: try XCTUnwrap(env["UMBOD_UI_DATA_DIR"])) let app = XCUIApplication(url: URL(fileURLWithPath: appPath)) app.launchArguments = ["--data-dir", directory.path, "--test-credentials"] - defer { + addTeardownBlock { @MainActor in let screenshot = XCTAttachment(screenshot: app.screenshot()) screenshot.lifetime = .keepAlways - add(screenshot) + self.add(screenshot) app.terminate() try? FileManager.default.removeItem(at: directory) } app.launch() - XCTAssertTrue(app.staticTexts["Gateway running"].waitForExistence(timeout: 60)) + XCTAssertTrue(app.staticTexts["Gateway running"].waitForExistence(timeout: 60), app.staticTexts.allElementsBoundByIndex.map { String(describing: $0.value) }.joined(separator: "\n")) app.buttons["Connectors"].click() app.buttons["connectors.add"].click() let name = app.textFields["connector.name"] @@ -54,7 +54,7 @@ final class ConnectorJourney: XCTestCase { XCTAssertTrue(app.buttons["connector.save"].waitForNonExistence(timeout: 60)) app.terminate() app.launch() - XCTAssertTrue(app.staticTexts["Gateway running"].waitForExistence(timeout: 60)) + XCTAssertTrue(app.staticTexts["Gateway running"].waitForExistence(timeout: 60), app.staticTexts.allElementsBoundByIndex.map { String(describing: $0.value) }.joined(separator: "\n")) app.buttons["Connectors"].click() XCTAssertTrue(app.buttons["connector.row.UI fixture edited"].waitForExistence(timeout: 15)) XCTAssertFalse(app.buttons["connector.row.UI fixture"].exists, "Editing must not create a duplicate") diff --git a/desktop/scripts/run-macos-ui-tests.py b/desktop/scripts/run-macos-ui-tests.py index d5593cf..b5ab9cc 100644 --- a/desktop/scripts/run-macos-ui-tests.py +++ b/desktop/scripts/run-macos-ui-tests.py @@ -1,23 +1,48 @@ #!/usr/bin/env python3 """Run precompiled XCUITests against the exact installed artifact on disposable CI.""" +import argparse import json import os from pathlib import Path import plistlib import subprocess -import sys +import tempfile +import uuid -if os.environ.get("GITHUB_ACTIONS") != "true" or os.environ.get("RUNNER_ENVIRONMENT") != "github-hosted": - raise SystemExit("Release-identity UI tests require a disposable GitHub-hosted Mac") -app, kit, evidence = map(lambda value: Path(value).resolve(), sys.argv[1:]) +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("app", type=Path) +parser.add_argument("kit", type=Path) +parser.add_argument("evidence", type=Path) +parser.add_argument("--local-fixture", action="store_true", help="Copy and re-sign the app with a disposable bundle identity") +args = parser.parse_args() +if not args.local_fixture and (os.environ.get("GITHUB_ACTIONS") != "true" or os.environ.get("RUNNER_ENVIRONMENT") != "github-hosted"): + raise SystemExit("Release-identity UI tests require a disposable GitHub-hosted Mac; use --local-fixture locally") +app, kit, evidence = (value.resolve() for value in [args.app, args.kit, args.evidence]) +# Keep the temporary copy alive for the entire xcodebuild invocation. +local_copy = tempfile.TemporaryDirectory(prefix="umbod-test-ui-app-") if args.local_fixture else None +if local_copy: + copy = Path(local_copy.name) / "Umbod UI Fixture.app" + subprocess.run(["/usr/bin/ditto", str(app), str(copy)], check=True) + app = copy + info = app / "Contents/Info.plist" + metadata = plistlib.loads(info.read_bytes()) + metadata["CFBundleIdentifier"] += ".fixture." + uuid.uuid4().hex + info.write_bytes(plistlib.dumps(metadata)) + subprocess.run(["/usr/bin/codesign", "--force", "--sign", "-", str(app)], check=True) +subprocess.run(["/usr/bin/codesign", "--verify", "--deep", "--strict", str(app)], check=True) evidence.mkdir(parents=True, exist_ok=True) products = kit / "ui-tests" -plans = list(products.glob("*.xctestrun")) +plans = [path for path in products.glob("*.xctestrun") if path.name != "configured.xctestrun"] if len(plans) != 1: raise SystemExit("Expected exactly one precompiled UI test plan") plan = plistlib.loads(plans[0].read_bytes()) target = plan["ConnectorJourney"] +# The XCTest runner's Foundation temporaryDirectory lives in its app container. +# Create fixture data in the real user's temporary root, recognized by the app's +# deliberately strict FixtureVault guard, without weakening that guard. +data_directory = tempfile.TemporaryDirectory(prefix="umbod-test-ui-data-") target["EnvironmentVariables"].update({ + "UMBOD_UI_DATA_DIR": str(Path(data_directory.name).resolve()), "UMBOD_UI_APP": str(app), "UMBOD_UI_FIXTURE": str(kit / "scripts/fixture_stdio.py"), "UMBOD_UI_AGENT": str(kit / "scripts/fixture_agent.py"), @@ -41,3 +66,9 @@ if report.get("passedTests") != 1 or report.get("failedTests") != 0 or report.get("skippedTests") != 0: raise SystemExit("Expected one passing UI journey with no failures or skipped tests") print("PASS: packaged UI add → discover → enable → agent call → edit → relaunch") + +subprocess.run(["/usr/bin/codesign", "--verify", "--deep", "--strict", str(app)], check=True) +if local_copy: + local_copy.cleanup() + +data_directory.cleanup() From 813ac633781f0824db51340e813019667fbedb02 Mon Sep 17 00:00:00 2001 From: Kasper Junge Date: Sun, 11 Oct 2026 20:35:07 +0200 Subject: [PATCH 3/3] Gate macOS releases on browser OAuth and packaged UI journeys --- desktop/README.md | 4 +- desktop/docs/RELEASING.md | 2 +- .../native/Sources/Umbod/ConnectorsView.swift | 14 +- desktop/native/Sources/Umbod/UmbodApp.swift | 6 +- desktop/native/UITests/ConnectorJourney.swift | 120 +++++- .../project.pbxproj | 2 +- desktop/scripts/fixture_agent.py | 28 +- desktop/scripts/prepare-macos-acceptance.sh | 6 +- desktop/scripts/run-macos-ui-tests.py | 127 ++++-- desktop/scripts/ui_fixture.py | 385 ++++++++++++++++++ desktop/tests/test_ui_fixture.py | 156 +++++++ 11 files changed, 790 insertions(+), 60 deletions(-) mode change 100644 => 100755 desktop/scripts/run-macos-ui-tests.py create mode 100644 desktop/scripts/ui_fixture.py create mode 100644 desktop/tests/test_ui_fixture.py diff --git a/desktop/README.md b/desktop/README.md index e5d1182..1fdb170 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -63,7 +63,7 @@ The [upstream Windows run](https://github.com/kasperjunge/umbod-desktop/actions/ Release packaging collects dependency inventories and notices through `scripts/dependency-notices.py`; pinned supplemental sources are documented in `licenses/README.md`. `scripts/verify-python-bundle.py` rejects editable installs and imports outside the bundled interpreter before signing and again after installation. `scripts/audit-macos-bundle.py` verifies ARM64 slices, the macOS 14 deployment floor, required resources, executable permissions, internal symlinks and relocatable native dependencies. `Resources/build-metadata.json` records source and toolchain inputs. `scripts/prepare-macos-acceptance.sh` packages a precompiled test harness and fixtures; the separate `macos-acceptance` job runs `scripts/accept-macos-artifact.sh` on current macOS ARM64 with no checkout, using the DMG’s interpreter and the unchanged release identity. This required job also runs in the manual release workflow before publication. Local UI tests retain disposable bundle identities to protect user preferences. `scripts/install-smoke.py` installs the Mac DMG into disposable storage and verifies the packaged connector/agent flow, native window lifecycle, and signature integrity before and after execution. It never uses the login Keychain. -The acceptance kit also includes a precompiled XCUITest runner. `scripts/run-macos-ui-tests.py` runs it with `xcodebuild test-without-building` on the disposable hosted Mac. It clicks Add connector, saves a local fixture, discovers and enables its echo tool, calls it through the packaged agent bridge, edits the connector, and verifies persistence after relaunch. No API shortcut creates the connector. The required gate rejects failed or skipped journeys and preserves `.xcresult` diagnostics and screenshots. The runner uses temporary data and fixture credentials, never the login Keychain. Local UI testing uses `--local-fixture` to copy and re-sign the app with a unique bundle identity; release-identity execution remains restricted to disposable CI. The app is still built with SwiftPM, and the separate test project does not build or modify it. +The acceptance kit also includes a precompiled XCUITest runner. `scripts/run-macos-ui-tests.py` runs it with `xcodebuild test-without-building` on the disposable hosted Mac. It clicks Add connector, saves a local fixture, discovers and enables its echo tool, calls it through the packaged agent bridge, edits the connector, and verifies persistence after relaunch. A second journey adds a remote HTTPS MCP fixture, opens a real browser for synthetic consent, completes discovery/registration and the PKCE callback, enables and calls a tool, refreshes the token, and verifies authenticated calls after relaunch. `scripts/ui_fixture.py` owns the local OAuth server and runs the packaged agent outside the sandboxed XCTest runner; its control endpoints require an ephemeral capability. Its temporary certificate is trusted only by the test app process, with no Keychain or system trust changes. No API shortcut creates or enables connectors. This tests the standards-based sign-in flow rather than Wispr Flow’s live account service. The required gate rejects failed or skipped journeys and preserves `.xcresult` diagnostics and screenshots. The runner uses temporary data and fixture credentials, never the login Keychain. Local UI testing uses `--local-fixture` to copy and re-sign the app with a unique bundle identity; release-identity execution remains restricted to disposable CI. The app is still built with SwiftPM, and the separate test project does not build or modify it. To run the same click-through journey locally without touching installed apps or profiles: @@ -72,4 +72,4 @@ bash desktop/scripts/prepare-macos-acceptance.sh /tmp/umbod-ui-kit python3 desktop/scripts/run-macos-ui-tests.py "desktop/dist/Umbod Dev.app" /tmp/umbod-ui-kit /tmp/umbod-ui-evidence --local-fixture ``` -Build the dev bundle first. Use a new evidence directory per run. Xcode UI testing may request macOS automation/accessibility access on first use; do not bypass system controls. The CI gate still tests the unchanged release artifact. +Build the dev bundle first. Use a fresh kit directory per build and a new evidence directory per run. Xcode UI testing may request macOS automation/accessibility access on first use; do not bypass system controls. The CI gate still tests the unchanged release artifact. diff --git a/desktop/docs/RELEASING.md b/desktop/docs/RELEASING.md index 0108d87..a9aeb00 100644 --- a/desktop/docs/RELEASING.md +++ b/desktop/docs/RELEASING.md @@ -6,7 +6,7 @@ Desktop is versioned independently of chart, images and SDK. Update `desktop/VER 1. Resolve the source-license and dependency-notice blockers in [provenance](PROVENANCE.md), review the changes, and merge through the normal process. Do not copy historical personal docs or evidence. 2. Configure the GitHub `desktop-release` environment with required reviewers and a deployment branch rule permitting only `main`. The workflow also validates repository, ref, event, version, rights attestation, and exact confirmation. No signing secret is required for unsigned builds. No credentials belong in website code or manifests. -3. Run native desktop CI on the reviewed source. PR/push CI only uploads artifacts and has read-only permissions. Mac uses an Apple Silicon runner and Swift 6; Windows builds native x64 Rust and WPF, including its packaged smoke test. The Mac artifact-only acceptance job must also pass: it uses the exact DMG, no checkout or compiler, and an unchanged release app launched through LaunchServices on a fresh hosted runner. Its precompiled XCUITest journey must pass without skipped tests, including UI connector creation, tool enablement and a real agent call; inspect the uploaded `.xcresult` on failure. Both platform builds must pass; a local cross-build does not substitute for Windows tests. +3. Run native desktop CI on the reviewed source. PR/push CI only uploads artifacts and has read-only permissions. Mac uses an Apple Silicon runner and Swift 6; Windows builds native x64 Rust and WPF, including its packaged smoke test. The Mac artifact-only acceptance job must also pass: it uses the exact DMG, no checkout or compiler, and an unchanged release app launched through LaunchServices on a fresh hosted runner. Both precompiled XCUITest journeys must pass without skipped tests: local connector creation/edit/relaunch and browser OAuth sign-in/PKCE, token refresh, and credential persistence. Both must enable tools through the UI and make real calls through the packaged agent bridge; inspect the uploaded `.xcresult` on failure. Both platform builds must pass; a local cross-build does not substitute for Windows tests. 4. From `main`, explicitly dispatch **Desktop manual prerelease** with version `X.Y.Z`, mode `unsigned-prerelease`, rights confirmation enabled, and confirmation text `release desktop-vX.Y.Z unsigned-prerelease`. The workflow rebuilds/tests the dispatch SHA. It creates a new desktop tag, an explicitly UNSIGNED GitHub prerelease, versioned assets, release notes, SHA256SUMS and desktop-downloads.json. It never promotes repository-wide latest or overwrites existing tags/releases/assets. 5. Download the `website-manifest-update` artifact and follow [APPLY-MANIFEST.md](APPLY-MANIFEST.md). This is a PR-ready file tree; publication does not automatically edit or deploy the website. diff --git a/desktop/native/Sources/Umbod/ConnectorsView.swift b/desktop/native/Sources/Umbod/ConnectorsView.swift index 76852f0..b6bc978 100644 --- a/desktop/native/Sources/Umbod/ConnectorsView.swift +++ b/desktop/native/Sources/Umbod/ConnectorsView.swift @@ -10,9 +10,17 @@ struct ServersView: View { @State private var sheet: ConnectorSheet? @Binding var selectedConnector: String? @State private var removal: String? + // An offline draft receives its backend ID after first successful discovery. + // Keep the selected detail and sheet attached through its stable host identity. + private func connector(_ id: String) -> [String: Any]? { + model.servers.first { + $0["id"] as? String == id || + (id.hasPrefix("pending:") && $0["host_id"] as? String == String(id.dropFirst("pending:".count))) + } + } var body: some View { VStack(alignment: .leading, spacing: 24) { - if let id = selectedConnector, let connector = model.servers.first(where: { $0["id"] as? String == id }) { + if let id = selectedConnector, let connector = connector(id) { Button { selectedConnector = nil } label: { Label("All connectors", systemImage: "chevron.left") } ConnectorToolsView(connector: connector, manage: { sheet = .manage(id) }).id(id) } else { @@ -58,9 +66,9 @@ struct ServersView: View { switch route { case .add: ServerEditor(draft: ServerDraft()) case .edit(let id): - if let server = model.servers.first(where: { $0["id"] as? String == id }) { ServerEditor(draft: ServerDraft(server)) } + if let server = connector(id) { ServerEditor(draft: ServerDraft(server)) } case .manage(let id): - if let server = model.servers.first(where: { $0["id"] as? String == id }) { + if let server = connector(id) { ConnectorManagementView(server: server, edit: { sheet = .edit(id) }) } } diff --git a/desktop/native/Sources/Umbod/UmbodApp.swift b/desktop/native/Sources/Umbod/UmbodApp.swift index ec8fbad..b10af9f 100644 --- a/desktop/native/Sources/Umbod/UmbodApp.swift +++ b/desktop/native/Sources/Umbod/UmbodApp.swift @@ -232,12 +232,12 @@ struct ServerEditor: View { Text(editor.draft.id.isEmpty ? "Add connector" : "Edit connector").font(.title2.bold()) ScrollView { VStack(alignment: .leading, spacing: 16) { TextField("Name", text: $editor.draft.name).accessibilityIdentifier("connector.name") - Picker("Connection", selection: $editor.draft.transport) { Text("Local process · stdio").tag("stdio"); Text("Remote · Streamable HTTP").tag("http") } + Picker("Connection", selection: $editor.draft.transport) { Text("Local process · stdio").tag("stdio"); Text("Remote · Streamable HTTP").tag("http") }.accessibilityIdentifier("connector.transport") if editor.draft.transport == "stdio" { TextField("Executable (absolute path)", text: $editor.draft.command).accessibilityIdentifier("connector.command") Text("Arguments · one per line, no shell quoting").font(.caption).foregroundStyle(Brand.muted) - TextEditor(text: $editor.draft.arguments).accessibilityIdentifier("connector.arguments").font(.system(.body, design: .monospaced)).frame(height: 65).padding(8).background(Brand.canvas).overlay(RoundedRectangle(cornerRadius: 8).stroke(Brand.border)) - } else { TextField("Endpoint URL", text: $editor.draft.url) } + TextEditor(text: $editor.draft.arguments).accessibilityIdentifier("connector.arguments").font(.system(.body, design: .monospaced)).frame(height: 65).padding(8).background(Brand.canvas).overlay(RoundedRectangle(cornerRadius: 8).stroke(Brand.border).allowsHitTesting(false)) + } else { TextField("Endpoint URL", text: $editor.draft.url).accessibilityIdentifier("connector.url") } VStack(alignment: .leading, spacing: 12) { Text("Optional credential · stored only in Keychain").font(.system(size: 14, weight: .semibold)).padding(.top, 8) TextField(editor.draft.transport == "stdio" ? "Environment variable name" : "Header name (Authorization, X-API-Key…)", text: $editor.credentialName) diff --git a/desktop/native/UITests/ConnectorJourney.swift b/desktop/native/UITests/ConnectorJourney.swift index 68fa3b0..c439a8d 100644 --- a/desktop/native/UITests/ConnectorJourney.swift +++ b/desktop/native/UITests/ConnectorJourney.swift @@ -2,7 +2,14 @@ import XCTest /// Runs against the installed DMG app, never an app rebuilt by this Xcode project. final class ConnectorJourney: XCTestCase { - @MainActor func testAddEditAndReopenConnector() throws { + private func require(_ condition: Bool, _ message: String = "Required UI state was not reached", file: StaticString = #filePath, line: UInt = #line) throws { + guard condition else { + XCTFail(message, file: file, line: line) + throw NSError(domain: "UmbodUIJourney", code: 1, userInfo: [NSLocalizedDescriptionKey: message]) + } + } + + @MainActor func testAddEditAndReopenConnector() async throws { continueAfterFailure = false let env = ProcessInfo.processInfo.environment let appPath = try XCTUnwrap(env["UMBOD_UI_APP"]) @@ -18,11 +25,12 @@ final class ConnectorJourney: XCTestCase { try? FileManager.default.removeItem(at: directory) } app.launch() - XCTAssertTrue(app.staticTexts["Gateway running"].waitForExistence(timeout: 60), app.staticTexts.allElementsBoundByIndex.map { String(describing: $0.value) }.joined(separator: "\n")) + app.activate() + try require(app.staticTexts["Gateway running"].waitForExistence(timeout: 60), app.staticTexts.allElementsBoundByIndex.map { String(describing: $0.value) }.joined(separator: "\n")) app.buttons["Connectors"].click() app.buttons["connectors.add"].click() let name = app.textFields["connector.name"] - XCTAssertTrue(name.waitForExistence(timeout: 5)) + try require(name.waitForExistence(timeout: 5)) name.click(); name.typeText("UI fixture") let command = app.textFields["connector.command"] command.click(); command.typeText(appPath + "/Contents/Resources/python/bin/python3") @@ -30,33 +38,111 @@ final class ConnectorJourney: XCTestCase { arguments.click(); arguments.typeText("-B\n" + fixture) app.buttons["connector.save"].click() let row = app.buttons["connector.row.UI fixture"] - XCTAssertTrue(row.waitForExistence(timeout: 60), "Saving a new connector must close the editor and show the created connector") + try require(row.waitForExistence(timeout: 60), "Saving a new connector must close the editor and show the created connector") row.click() app.buttons["Connection settings"].click() app.buttons["Connect / rediscover"].click() - XCTAssertTrue(app.staticTexts["Connected"].waitForExistence(timeout: 60)) + try require(app.staticTexts["Connected"].waitForExistence(timeout: 60)) app.buttons["Done"].click() let toggle = app.descendants(matching: .any).matching(identifier: "tool.allow.echo").firstMatch - XCTAssertTrue(toggle.waitForExistence(timeout: 15)) + try require(toggle.waitForExistence(timeout: 15)) toggle.click() - let agent = Process() - agent.executableURL = URL(fileURLWithPath: appPath + "/Contents/Resources/python/bin/python3") - let profile = try String(contentsOfFile: appPath + "/Contents/Resources/umbod-profile", encoding: .utf8).trimmingCharacters(in: .whitespacesAndNewlines) - agent.arguments = ["-I", "-B", try XCTUnwrap(env["UMBOD_UI_AGENT"]), appPath + "/Contents/MacOS/umbod-desktop", directory.path, profile] - try agent.run() - agent.waitUntilExit() - XCTAssertEqual(agent.terminationStatus, 0, "An agent must be able to call the tool enabled through the UI") + try await callAgent("stdio") app.buttons["Connection settings"].click() app.buttons["Edit connection"].click() - XCTAssertTrue(name.waitForExistence(timeout: 5)) + try require(name.waitForExistence(timeout: 5)) name.click(); name.typeKey("a", modifierFlags: .command); name.typeText("UI fixture edited") app.buttons["connector.save"].click() - XCTAssertTrue(app.buttons["connector.save"].waitForNonExistence(timeout: 60)) + try require(app.buttons["connector.save"].waitForNonExistence(timeout: 60)) app.terminate() app.launch() - XCTAssertTrue(app.staticTexts["Gateway running"].waitForExistence(timeout: 60), app.staticTexts.allElementsBoundByIndex.map { String(describing: $0.value) }.joined(separator: "\n")) + app.activate() + try require(app.staticTexts["Gateway running"].waitForExistence(timeout: 60), app.staticTexts.allElementsBoundByIndex.map { String(describing: $0.value) }.joined(separator: "\n")) app.buttons["Connectors"].click() - XCTAssertTrue(app.buttons["connector.row.UI fixture edited"].waitForExistence(timeout: 15)) + try require(app.buttons["connector.row.UI fixture edited"].waitForExistence(timeout: 15)) XCTAssertFalse(app.buttons["connector.row.UI fixture"].exists, "Editing must not create a duplicate") } + + @MainActor private func oracle(_ path: String) async throws -> [String: Any] { + let env = ProcessInfo.processInfo.environment + var request = URLRequest(url: try XCTUnwrap(URL(string: try XCTUnwrap(env["UMBOD_UI_FIXTURE_ORIGIN"]) + "/test/" + path))) + request.httpMethod = path.hasPrefix("agent/") ? "POST" : "GET" + request.timeoutInterval = 65 + request.setValue("Bearer " + (try XCTUnwrap(env["UMBOD_UI_FIXTURE_CAPABILITY"])), forHTTPHeaderField: "Authorization") + let (data, response) = try await URLSession.shared.data(for: request) + XCTAssertEqual((response as? HTTPURLResponse)?.statusCode, 200, "Fixture oracle failed for \(path)") + return try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) + } + + @MainActor private func callAgent(_ journey: String) async throws { + let result = try await oracle("agent/" + journey) + XCTAssertEqual(result["passed"] as? Bool, true, "The packaged agent must call the UI-enabled tool") + } + + @MainActor func testBrowserSignInRefreshAndReopen() async throws { + continueAfterFailure = false + let env = ProcessInfo.processInfo.environment + let app = XCUIApplication(url: URL(fileURLWithPath: try XCTUnwrap(env["UMBOD_UI_APP"]))) + app.launchArguments = ["--data-dir", try XCTUnwrap(env["UMBOD_UI_OAUTH_DATA_DIR"]), "--test-credentials"] + app.launchEnvironment = ["SSL_CERT_FILE": try XCTUnwrap(env["UMBOD_UI_CA"])] + addTeardownBlock { @MainActor in + let screenshot = XCTAttachment(screenshot: app.screenshot()) + screenshot.lifetime = .keepAlways + self.add(screenshot) + app.terminate() + } + app.launch(); app.activate() + try require(app.staticTexts["Gateway running"].waitForExistence(timeout: 60)) + app.buttons["Connectors"].click() + app.buttons["connectors.add"].click() + app.textFields["connector.name"].click() + app.textFields["connector.name"].typeText("OAuth fixture") + app.popUpButtons["connector.transport"].click() + app.menuItems["Remote · Streamable HTTP"].click() + app.textFields["connector.url"].click() + app.textFields["connector.url"].typeText(try XCTUnwrap(env["UMBOD_UI_OAUTH_ENDPOINT"])) + app.buttons["connector.save"].click() + let row = app.buttons["connector.row.OAuth fixture"] + try require(row.waitForExistence(timeout: 60)) + row.click() + app.buttons["Connection settings"].click() + app.buttons["Sign in with browser"].click() + var signedIn = false + let deadline = Date().addingTimeInterval(60) + while Date() < deadline { + let state = try await oracle("oauth-ready") + if state["failed"] as? Bool == true { break } + if state["ready"] as? Bool == true { + XCTAssertEqual(state["browser_visits"] as? Int, 1) + XCTAssertEqual(state["exchanges"] as? Int, 1) + signedIn = true + break + } + try await Task.sleep(for: .milliseconds(250)) + } + guard signedIn else { XCTFail("The real browser redirect and PKCE exchange must finish"); return } + app.activate() + app.buttons["Connect / rediscover"].click() + try require(app.staticTexts["Connected"].waitForExistence(timeout: 60)) + app.buttons["Done"].click() + let toggle = app.descendants(matching: .any).matching(identifier: "tool.allow.echo").firstMatch + try require(toggle.waitForExistence(timeout: 15)) + toggle.click() + try await callAgent("oauth") + app.buttons["Connection settings"].click() + app.buttons["Refresh token"].click() + try require(app.staticTexts["Token refreshed and transport reconnected."].waitForExistence(timeout: 30)) + try await callAgent("oauth") + let refreshed = try await oracle("oauth-ready") + XCTAssertEqual(refreshed["refreshes"] as? Int, 1) + app.terminate() + app.launch(); app.activate() + try require(app.staticTexts["Gateway running"].waitForExistence(timeout: 60)) + app.buttons["Connectors"].click() + try require(row.waitForExistence(timeout: 15)) + try await callAgent("oauth") + let restored = try await oracle("oauth-ready") + XCTAssertEqual(restored["browser_visits"] as? Int, 1, "Stored OAuth credentials must survive relaunch without another browser sign-in") + } + } diff --git a/desktop/native/UITests/ConnectorJourney.xcodeproj/project.pbxproj b/desktop/native/UITests/ConnectorJourney.xcodeproj/project.pbxproj index a68386c..1f5ad2d 100644 --- a/desktop/native/UITests/ConnectorJourney.xcodeproj/project.pbxproj +++ b/desktop/native/UITests/ConnectorJourney.xcodeproj/project.pbxproj @@ -15,7 +15,7 @@ 100000000000000000000009 = {isa = PBXBuildFile; fileRef = 100000000000000000000005;}; 100000000000000000000010 = {isa = XCConfigurationList; buildConfigurations = (100000000000000000000012); defaultConfigurationIsVisible = 0; defaultConfigurationName = Debug;}; 100000000000000000000011 = {isa = XCConfigurationList; buildConfigurations = (100000000000000000000013); defaultConfigurationIsVisible = 0; defaultConfigurationName = Debug;}; - 100000000000000000000012 = {isa = XCBuildConfiguration; name = Debug; buildSettings = {SDKROOT = macosx; MACOSX_DEPLOYMENT_TARGET = 14.0; SWIFT_VERSION = 6.0; ARCHS = arm64; ONLY_ACTIVE_ARCH = YES;};}; + 100000000000000000000012 = {isa = XCBuildConfiguration; name = Debug; buildSettings = {ALWAYS_SEARCH_USER_PATHS = NO; SDKROOT = macosx; MACOSX_DEPLOYMENT_TARGET = 14.0; SWIFT_VERSION = 6.0; ARCHS = arm64; ONLY_ACTIVE_ARCH = YES;};}; 100000000000000000000013 = {isa = XCBuildConfiguration; name = Debug; buildSettings = {PRODUCT_BUNDLE_IDENTIFIER = app.umbod.tests.ConnectorJourney; PRODUCT_NAME = "$(TARGET_NAME)"; GENERATE_INFOPLIST_FILE = YES; CODE_SIGN_IDENTITY = "-"; CODE_SIGN_STYLE = Manual; ENABLE_HARDENED_RUNTIME = NO;};}; }; rootObject = 100000000000000000000001; diff --git a/desktop/scripts/fixture_agent.py b/desktop/scripts/fixture_agent.py index 493c213..8335708 100644 --- a/desktop/scripts/fixture_agent.py +++ b/desktop/scripts/fixture_agent.py @@ -1,20 +1,36 @@ """Call the UI-enabled echo tool through the packaged agent bridge.""" + import asyncio import sys +from pathlib import Path + from fastmcp import Client from fastmcp.client.transports import StdioTransport async def main(): - async with Client(StdioTransport(command=sys.argv[1], args=[ - "bridge", "--profile", sys.argv[3], "--data-dir", sys.argv[2], - "--test-credentials", - ]), mode="legacy") as client: + async with Client( + StdioTransport( + command=sys.argv[1], + args=[ + "bridge", + "--profile", + sys.argv[3], + "--data-dir", + sys.argv[2], + "--test-credentials", + ], + env={"TMPDIR": str(Path(sys.argv[2]).parent)}, + ), + mode="legacy", + ) as client: async with asyncio.timeout(30): while not (tools := await client.list_tools()): - await asyncio.sleep(.1) + await asyncio.sleep(0.1) assert len(tools) == 1, "Only the UI-enabled tool should be published" - result = await client.call_tool(tools[0].name, {"text": "ui-journey-fixture"}) + result = await client.call_tool( + tools[0].name, {"text": "ui-journey-fixture"} + ) assert "ui-journey-fixture" in str(result) diff --git a/desktop/scripts/prepare-macos-acceptance.sh b/desktop/scripts/prepare-macos-acceptance.sh index a789689..58ad9f2 100755 --- a/desktop/scripts/prepare-macos-acceptance.sh +++ b/desktop/scripts/prepare-macos-acceptance.sh @@ -2,10 +2,14 @@ set -euo pipefail cd "$(dirname "$0")/.." DEST="${1:?test kit output directory required}" +if [[ -e "$DEST/ui-tests" ]]; then + echo "Choose a fresh kit directory; an existing UI test kit must not be mixed with a new build." >&2 + exit 1 +fi mkdir -p "$DEST/scripts" "$DEST/tests/fixtures" swift build --package-path native --product UmbodTests cp native/.build/debug/UmbodTests "$DEST/UmbodTests" -cp scripts/{accept-macos-artifact.sh,audit-macos-bundle.py,verify-python-bundle.py,native-smoke.py,fixture_stdio.py,fixture_agent.py,run-macos-ui-tests.py} "$DEST/scripts/" +cp scripts/{accept-macos-artifact.sh,audit-macos-bundle.py,verify-python-bundle.py,native-smoke.py,fixture_stdio.py,fixture_agent.py,run-macos-ui-tests.py,ui_fixture.py} "$DEST/scripts/" cp tests/fixtures/update-manifest.json "$DEST/tests/fixtures/" # Only the test runner is compiled here. XCUITest launches the separately packaged app. diff --git a/desktop/scripts/run-macos-ui-tests.py b/desktop/scripts/run-macos-ui-tests.py old mode 100644 new mode 100755 index b5ab9cc..9e80452 --- a/desktop/scripts/run-macos-ui-tests.py +++ b/desktop/scripts/run-macos-ui-tests.py @@ -1,25 +1,40 @@ #!/usr/bin/env python3 """Run precompiled XCUITests against the exact installed artifact on disposable CI.""" + import argparse import json import os -from pathlib import Path import plistlib +import runpy import subprocess import tempfile import uuid +from pathlib import Path parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("app", type=Path) parser.add_argument("kit", type=Path) parser.add_argument("evidence", type=Path) -parser.add_argument("--local-fixture", action="store_true", help="Copy and re-sign the app with a disposable bundle identity") +parser.add_argument( + "--local-fixture", + action="store_true", + help="Copy and re-sign the app with a disposable bundle identity", +) args = parser.parse_args() -if not args.local_fixture and (os.environ.get("GITHUB_ACTIONS") != "true" or os.environ.get("RUNNER_ENVIRONMENT") != "github-hosted"): - raise SystemExit("Release-identity UI tests require a disposable GitHub-hosted Mac; use --local-fixture locally") +if not args.local_fixture and ( + os.environ.get("GITHUB_ACTIONS") != "true" + or os.environ.get("RUNNER_ENVIRONMENT") != "github-hosted" +): + raise SystemExit( + "Release-identity UI tests require a disposable GitHub-hosted Mac; use --local-fixture locally" + ) app, kit, evidence = (value.resolve() for value in [args.app, args.kit, args.evidence]) # Keep the temporary copy alive for the entire xcodebuild invocation. -local_copy = tempfile.TemporaryDirectory(prefix="umbod-test-ui-app-") if args.local_fixture else None +local_copy = ( + tempfile.TemporaryDirectory(prefix="umbod-test-ui-app-") + if args.local_fixture + else None +) if local_copy: copy = Path(local_copy.name) / "Umbod UI Fixture.app" subprocess.run(["/usr/bin/ditto", str(app), str(copy)], check=True) @@ -28,11 +43,17 @@ metadata = plistlib.loads(info.read_bytes()) metadata["CFBundleIdentifier"] += ".fixture." + uuid.uuid4().hex info.write_bytes(plistlib.dumps(metadata)) - subprocess.run(["/usr/bin/codesign", "--force", "--sign", "-", str(app)], check=True) -subprocess.run(["/usr/bin/codesign", "--verify", "--deep", "--strict", str(app)], check=True) + subprocess.run( + ["/usr/bin/codesign", "--force", "--sign", "-", str(app)], check=True + ) +subprocess.run( + ["/usr/bin/codesign", "--verify", "--deep", "--strict", str(app)], check=True +) evidence.mkdir(parents=True, exist_ok=True) products = kit / "ui-tests" -plans = [path for path in products.glob("*.xctestrun") if path.name != "configured.xctestrun"] +plans = [ + path for path in products.glob("*.xctestrun") if path.name != "configured.xctestrun" +] if len(plans) != 1: raise SystemExit("Expected exactly one precompiled UI test plan") plan = plistlib.loads(plans[0].read_bytes()) @@ -41,33 +62,87 @@ # Create fixture data in the real user's temporary root, recognized by the app's # deliberately strict FixtureVault guard, without weakening that guard. data_directory = tempfile.TemporaryDirectory(prefix="umbod-test-ui-data-") -target["EnvironmentVariables"].update({ - "UMBOD_UI_DATA_DIR": str(Path(data_directory.name).resolve()), - "UMBOD_UI_APP": str(app), - "UMBOD_UI_FIXTURE": str(kit / "scripts/fixture_stdio.py"), - "UMBOD_UI_AGENT": str(kit / "scripts/fixture_agent.py"), -}) +root = Path(data_directory.name).resolve() +directories = {name: root / ("umbod-test-" + name) for name in ("stdio", "oauth")} +Fixture = runpy.run_path(str(kit / "scripts/ui_fixture.py"))["Fixture"] +fixture = Fixture(root, app, kit, directories) +target["EnvironmentVariables"].update( + { + "UMBOD_UI_DATA_DIR": str(directories["stdio"]), + "UMBOD_UI_OAUTH_DATA_DIR": str(directories["oauth"]), + "UMBOD_UI_FIXTURE_ORIGIN": fixture.origin, + "UMBOD_UI_FIXTURE_CAPABILITY": fixture.capability, + "UMBOD_UI_OAUTH_ENDPOINT": fixture.endpoint, + "UMBOD_UI_CA": str(fixture.cert), + "UMBOD_UI_APP": str(app), + "UMBOD_UI_FIXTURE": str(kit / "scripts/fixture_stdio.py"), + "UMBOD_UI_AGENT": str(kit / "scripts/fixture_agent.py"), + } +) target["TestTimeoutsEnabled"] = True target["MaximumTestExecutionTimeAllowance"] = 180 target["UserAttachmentLifetime"] = "keepAlways" configured = products / "configured.xctestrun" configured.write_bytes(plistlib.dumps(plan)) result = evidence / "connector-journey.xcresult" -subprocess.run([ - "/usr/bin/xcodebuild", "test-without-building", "-xctestrun", str(configured), - "-destination", "platform=macOS,arch=arm64", "-parallel-testing-enabled", "NO", - "-resultBundlePath", str(result), -], check=True) -summary = subprocess.check_output([ - "/usr/bin/xcrun", "xcresulttool", "get", "test-results", "summary", "--path", str(result), -]) +try: + subprocess.run( + [ + "/usr/bin/xcodebuild", + "test-without-building", + "-xctestrun", + str(configured), + "-destination", + "platform=macOS,arch=arm64", + "-parallel-testing-enabled", + "NO", + "-resultBundlePath", + str(result), + ], + check=True, + ) +finally: + fixture.close() + (evidence / "fixture-requests.json").write_text( + json.dumps(fixture.requests, indent=2) + ) + (evidence / "oauth-fixture-audit.json").write_text( + json.dumps(fixture.audit, indent=2) + ) +summary = subprocess.check_output( + [ + "/usr/bin/xcrun", + "xcresulttool", + "get", + "test-results", + "summary", + "--path", + str(result), + ] +) (evidence / "summary.json").write_bytes(summary) report = json.loads(summary) -if report.get("passedTests") != 1 or report.get("failedTests") != 0 or report.get("skippedTests") != 0: - raise SystemExit("Expected one passing UI journey with no failures or skipped tests") -print("PASS: packaged UI add → discover → enable → agent call → edit → relaunch") +if ( + report.get("passedTests") != 2 + or report.get("failedTests") != 0 + or report.get("skippedTests") != 0 +): + raise SystemExit( + "Expected two passing UI journeys with no failures or skipped tests" + ) +assert ( + fixture.audit["browser_visits"] >= 1 + and fixture.audit["exchanges"] >= 1 + and fixture.audit["refreshes"] >= 1 + and fixture.audit["calls"] >= 2 +), "Browser, PKCE exchange, refresh and authenticated calls must all occur" +print( + "PASS: packaged local and browser OAuth UI journeys, agent calls, refresh, edit and relaunch" +) -subprocess.run(["/usr/bin/codesign", "--verify", "--deep", "--strict", str(app)], check=True) +subprocess.run( + ["/usr/bin/codesign", "--verify", "--deep", "--strict", str(app)], check=True +) if local_copy: local_copy.cleanup() diff --git a/desktop/scripts/ui_fixture.py b/desktop/scripts/ui_fixture.py new file mode 100644 index 0000000..4088f09 --- /dev/null +++ b/desktop/scripts/ui_fixture.py @@ -0,0 +1,385 @@ +"""Loopback-only OAuth/MCP fixture and agent oracle for packaged UI acceptance. + +Never bundled in Umbod. OAuth automatically consents to a synthetic account, but +only a browser executing the consent page can complete the redirect. Token +exchange still verifies registration, redirect URI, one-use code and S256 PKCE. +""" + +import base64 +import hashlib +import json +import secrets +import ssl +import subprocess +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlencode, urlsplit +from urllib.request import Request, urlopen + + +class Fixture: + def __init__(self, root, app, kit, directories): + self.root, self.app, self.kit = map(Path, (root, app, kit)) + self.directories = directories + self.capability = secrets.token_urlsafe(32) + self.clients, self.consents, self.codes = {}, {}, {} + self.access, self.refresh = set(), set() + self.audit = { + "browser_visits": 0, + "exchanges": 0, + "refreshes": 0, + "rejected": 0, + "calls": 0, + } + self.requests = [] + self.lock = threading.RLock() + fixture = self + + class Handler(BaseHTTPRequestHandler): + def setup(self): + # Negotiate in the worker thread: an idle/preconnected socket + # must not block the listener from accepting OAuth connections. + self.request.settimeout(10) + if isinstance(self.request, ssl.SSLSocket): + self.request.do_handshake() + super().setup() + + def log_message(self, *_args): + pass + + def do_GET(self): + fixture.handle(self) + + def do_POST(self): + fixture.handle(self) + + def do_DELETE(self): + self.send_response(204) + self.end_headers() + + self.http = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.https = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = f"http://127.0.0.1:{self.http.server_port}" + self.issuer = f"https://127.0.0.1:{self.https.server_port}" + self.endpoint = self.issuer + "/mcp" + self.cert = self.root / "fixture-ca.pem" + key = self.root / "fixture-key.pem" + config = self.root / "openssl.cnf" + config.write_text( + "[req]\nprompt=no\ndistinguished_name=dn\nx509_extensions=ext\n[dn]\nCN=Umbod isolated UI fixture\n[ext]\nsubjectAltName=IP:127.0.0.1\nbasicConstraints=critical,CA:TRUE\nkeyUsage=critical,digitalSignature,keyEncipherment,keyCertSign\n" + ) + subprocess.run( + [ + "/usr/bin/openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-config", + str(config), + "-keyout", + str(key), + "-out", + str(self.cert), + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(self.cert, key) + self.https.socket = context.wrap_socket( + self.https.socket, server_side=True, do_handshake_on_connect=False + ) + for server in (self.http, self.https): + threading.Thread(target=server.serve_forever, daemon=True).start() + + def close(self): + for server in (self.http, self.https): + server.shutdown() + server.server_close() + + def respond(self, request, status, body=None, headers=None): + if not request.path.startswith("/test/"): + self.requests.append( + { + "method": request.command, + "path": urlsplit(request.path).path, + "status": status, + } + ) + data = json.dumps(body or {}).encode() + request.send_response(status) + request.send_header("Content-Type", "application/json") + request.send_header("Content-Length", str(len(data))) + request.send_header("Cache-Control", "no-store") + for name, value in (headers or {}).items(): + request.send_header(name, value) + request.end_headers() + request.wfile.write(data) + + def token(self): + access, refresh = secrets.token_urlsafe(32), secrets.token_urlsafe(32) + self.access.add(access) + self.refresh.add(refresh) + return { + "access_token": access, + "refresh_token": refresh, + "token_type": "Bearer", + "expires_in": 3600, + "scope": "mcp", + } + + def handle(self, request): + path = urlsplit(request.path).path + raw = request.rfile.read(int(request.headers.get("Content-Length", "0"))) + if path.startswith("/test/"): + if request.headers.get("Authorization") != "Bearer " + self.capability: + return self.respond(request, 403) + if path == "/test/agent/stdio" or path == "/test/agent/oauth": + directory = self.directories[path.rsplit("/", 1)[1]] + profile = ( + (self.app / "Contents/Resources/umbod-profile").read_text().strip() + ) + result = subprocess.run( + [ + str(self.app / "Contents/Resources/python/bin/python3"), + "-I", + "-B", + str(self.kit / "scripts/fixture_agent.py"), + str(self.app / "Contents/MacOS/umbod-desktop"), + str(directory), + profile, + ], + capture_output=True, + timeout=60, + check=False, + ) + if result.returncode: + print( + result.stderr.decode(errors="replace")[-8000:], + file=sys.stderr, + flush=True, + ) + return self.respond( + request, + 200 if result.returncode == 0 else 500, + {"passed": result.returncode == 0}, + ) + if path == "/test/oauth-ready": + directory = self.directories["oauth"] + try: + record = json.loads((directory / "python-runtime.json").read_text()) + vault = json.loads( + (directory / "fixture-credentials.json").read_text() + ) + address = record["management"] + assert urlsplit("http://" + address).hostname == "127.0.0.1" + with urlopen( + Request( + "http://" + address + "/desktop/state", + headers={ + "Authorization": "Bearer " + vault["python:admin-token"] + }, + ), + timeout=2, + ) as response: + state = json.load(response) + ready = "signed_in" in state.get("oauth_status", {}).values() + failed = "failed" in state.get("oauth_status", {}).values() + except (OSError, ValueError, KeyError): + ready = False + failed = False + return self.respond( + request, 200, {"ready": ready, "failed": failed, **self.audit} + ) + return self.respond(request, 404) + with self.lock: + try: + return self.oauth_request(request, path, raw) + except (ValueError, KeyError, AssertionError) as error: + self.audit["last_error"] = type(error).__name__ + " at " + path + self.audit["rejected"] += 1 + return self.respond(request, 400, {"error": "invalid_request"}) + + def oauth_request(self, request, path, raw): + if path.startswith("/.well-known/oauth-protected-resource"): + return self.respond( + request, + 200, + { + "resource": self.endpoint, + "authorization_servers": [self.issuer], + "scopes_supported": ["mcp"], + "bearer_methods_supported": ["header"], + }, + ) + if path == "/.well-known/oauth-authorization-server": + return self.respond( + request, + 200, + { + "issuer": self.issuer, + "authorization_endpoint": self.origin + "/authorize", + "token_endpoint": self.issuer + "/token", + "registration_endpoint": self.issuer + "/register", + "response_types_supported": ["code"], + "grant_types_supported": ["authorization_code", "refresh_token"], + "code_challenge_methods_supported": ["S256"], + "token_endpoint_auth_methods_supported": ["none"], + "scopes_supported": ["mcp"], + }, + ) + if path == "/register" and request.command == "POST": + metadata = json.loads(raw) + for uri in metadata["redirect_uris"]: + parsed = urlsplit(uri) + assert ( + parsed.scheme == "http" + and parsed.hostname == "127.0.0.1" + and parsed.path.startswith("/desktop/oauth/callback/") + ) + client = secrets.token_urlsafe(16) + self.clients[client] = metadata + return self.respond( + request, + 201, + {**metadata, "client_id": client, "token_endpoint_auth_method": "none"}, + ) + if path == "/authorize" and request.command == "GET": + values = { + key: value[0] + for key, value in parse_qs(urlsplit(request.path).query).items() + } + assert ( + values["redirect_uri"] + in self.clients[values["client_id"]]["redirect_uris"] + ) + assert ( + values["response_type"] == "code" + and values["code_challenge_method"] == "S256" + and values["state"] + ) + assert values.get("resource", self.endpoint) == self.endpoint + nonce = secrets.token_urlsafe(32) + self.consents[nonce] = values + self.audit["browser_visits"] += 1 + # Synthetic account: no real password/account, no browser certificate exception. + # JavaScript completion proves this step actually passed through a browser. + html = f'Umbod test sign-in

Signing in to the local Umbod test fixture…

'.encode() + request.send_response(200) + request.send_header("Content-Type", "text/html; charset=utf-8") + request.send_header("Content-Length", str(len(html))) + request.send_header("Cache-Control", "no-store") + request.end_headers() + request.wfile.write(html) + return + if path == "/approve" and request.command == "POST": + values = self.consents.pop(parse_qs(raw.decode())["nonce"][0]) + code = secrets.token_urlsafe(32) + self.codes[code] = (values, time.monotonic()) + return self.respond( + request, + 302, + headers={ + "Location": values["redirect_uri"] + + "?" + + urlencode( + {"code": code, "state": values["state"], "iss": self.issuer} + ) + }, + ) + if path == "/token" and request.command == "POST": + values = {key: value[0] for key, value in parse_qs(raw.decode()).items()} + if values["grant_type"] == "refresh_token": + self.refresh.remove(values["refresh_token"]) + self.audit["refreshes"] += 1 + return self.respond(request, 200, self.token()) + assert values["grant_type"] == "authorization_code" + grant, created = self.codes.pop(values["code"]) + assert time.monotonic() - created < 60 + challenge = ( + base64.urlsafe_b64encode( + hashlib.sha256(values["code_verifier"].encode()).digest() + ) + .decode() + .rstrip("=") + ) + assert challenge == grant["code_challenge"] + assert ( + values["client_id"] == grant["client_id"] + and values["redirect_uri"] == grant["redirect_uri"] + ) + self.audit["exchanges"] += 1 + return self.respond(request, 200, self.token()) + if path == "/mcp": + if ( + request.headers.get("Authorization", "").removeprefix("Bearer ") + not in self.access + ): + self.audit["rejected"] += 1 + return self.respond( + request, + 401, + headers={ + "WWW-Authenticate": f'Bearer resource_metadata="{self.issuer}/.well-known/oauth-protected-resource"' + }, + ) + if request.command != "POST": + return self.respond(request, 405) + message = json.loads(raw) + if "id" not in message: + return self.respond(request, 202) + method = message["method"] + if method == "initialize": + result = { + "protocolVersion": message["params"]["protocolVersion"], + "capabilities": {"tools": {}}, + "serverInfo": {"name": "umbod-oauth-fixture", "version": "1"}, + } + elif method == "tools/list": + result = { + "tools": [ + { + "name": "echo", + "description": "OAuth fixture echo", + "inputSchema": { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + }, + } + ] + } + elif method == "tools/call": + self.audit["calls"] += 1 + result = { + "content": [ + {"type": "text", "text": message["params"]["arguments"]["text"]} + ] + } + elif method == "ping": + result = {} + else: + return self.respond( + request, + 200, + { + "jsonrpc": "2.0", + "id": message["id"], + "error": { + "code": -32601, + "message": "Unsupported fixture method", + }, + }, + ) + return self.respond( + request, 200, {"jsonrpc": "2.0", "id": message["id"], "result": result} + ) + return self.respond(request, 404) diff --git a/desktop/tests/test_ui_fixture.py b/desktop/tests/test_ui_fixture.py new file mode 100644 index 0000000..7d56401 --- /dev/null +++ b/desktop/tests/test_ui_fixture.py @@ -0,0 +1,156 @@ +"""The OAuth fixture must reject invalid grants rather than mask product bugs.""" + +import base64 +import hashlib +import importlib.util +import io +import json +import re +import socket +import ssl +import tempfile +import unittest +from pathlib import Path +from urllib.parse import parse_qs, urlencode, urlsplit +from urllib.request import urlopen + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts/ui_fixture.py" +spec = importlib.util.spec_from_file_location("ui_fixture", SCRIPT) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class Request: + def __init__(self, path, method="GET", body=b"", headers=None): + self.path, self.command = path, method + self.headers = {"Content-Length": str(len(body)), **(headers or {})} + self.rfile, self.wfile = io.BytesIO(body), io.BytesIO() + self.response_headers = {} + + def send_response(self, status): + self.status = status + + def send_header(self, key, value): + self.response_headers[key] = value + + def end_headers(self): + pass + + +class OAuthFixtureTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(prefix="umbod-test-oauth-fixture-") + root = Path(self.directory.name) + self.fixture = module.Fixture(root, root, root, {}) + self.addCleanup(self.directory.cleanup) + self.addCleanup(self.fixture.close) + + def send(self, path, method="GET", body=b"", headers=None): + request = Request(path, method, body, headers) + self.fixture.handle(request) + return request + + def test_idle_tls_connection_does_not_block_discovery(self): + with ( + socket.create_connection(("127.0.0.1", self.fixture.https.server_port)), + urlopen( + self.fixture.issuer + "/.well-known/oauth-authorization-server", + context=ssl.create_default_context(cafile=self.fixture.cert), + timeout=2, + ) as response, + ): + self.assertEqual(json.load(response)["issuer"], self.fixture.issuer) + + def grant(self): + redirect = "http://127.0.0.1:9/desktop/oauth/callback/fixture" + registration = self.send( + "/register", "POST", json.dumps({"redirect_uris": [redirect]}).encode() + ) + client = json.loads(registration.wfile.getvalue())["client_id"] + verifier = "fixture-verifier-" * 4 + challenge = ( + base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()) + .decode() + .rstrip("=") + ) + page = self.send( + "/authorize?" + + urlencode( + { + "client_id": client, + "redirect_uri": redirect, + "response_type": "code", + "code_challenge_method": "S256", + "code_challenge": challenge, + "state": "state-fixture", + "resource": self.fixture.endpoint, + } + ) + ) + self.assertEqual(page.status, 200) + html = page.wfile.getvalue().decode() + self.assertIn("document.forms[0].requestSubmit()", html) + nonce = re.search(r'name="nonce" value="([^"]+)"', html)[1] + approved = self.send("/approve", "POST", urlencode({"nonce": nonce}).encode()) + response = parse_qs(urlsplit(approved.response_headers["Location"]).query) + self.assertEqual(response["state"], ["state-fixture"]) + self.assertEqual(response["iss"], [self.fixture.issuer]) + return { + "grant_type": "authorization_code", + "code": response["code"][0], + "code_verifier": verifier, + "client_id": client, + "redirect_uri": redirect, + } + + def test_pkce_one_use_codes_refresh_rotation_and_protected_mcp(self): + self.assertEqual(self.send("/mcp", "POST").status, 401) + grant = self.grant() + wrong = {**grant, "code_verifier": "wrong"} + self.assertEqual( + self.send("/token", "POST", urlencode(wrong).encode()).status, 400 + ) + self.assertEqual( + self.send("/token", "POST", urlencode(grant).encode()).status, 400 + ) + grant = self.grant() + exchange = self.send("/token", "POST", urlencode(grant).encode()) + self.assertEqual(exchange.status, 200) + tokens = json.loads(exchange.wfile.getvalue()) + self.assertEqual( + self.send("/token", "POST", urlencode(grant).encode()).status, 400 + ) + refresh = urlencode( + {"grant_type": "refresh_token", "refresh_token": tokens["refresh_token"]} + ).encode() + self.assertEqual(self.send("/token", "POST", refresh).status, 200) + self.assertEqual(self.send("/token", "POST", refresh).status, 400) + message = json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "echo", "arguments": {"text": "fixture"}}, + } + ).encode() + self.assertEqual( + self.send( + "/mcp", + "POST", + message, + {"Authorization": "Bearer " + tokens["access_token"]}, + ).status, + 200, + ) + self.assertEqual(self.fixture.audit["calls"], 1) + + def test_rejects_unregistered_redirects_and_unauthorized_test_controls(self): + bad = self.send( + "/register", + "POST", + json.dumps({"redirect_uris": ["https://example.com/callback"]}).encode(), + ) + self.assertEqual(bad.status, 400) + self.assertEqual(self.send("/authorize?client_id=unknown").status, 400) + self.assertEqual(self.send("/test/agent/stdio", "POST").status, 403) + self.assertEqual(self.send("/test/oauth-ready").status, 403)