Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .claude-plugin/marketplace.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"name": "agentops-accelerator",
"source": "../../plugins/agentops",
"description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.",
"version": "0.3.7",
"version": "0.3.8",
"keywords": [
"agentops",
"evaluation",
Expand Down
2 changes: 1 addition & 1 deletion .github/plugin/marketplace.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"name": "agentops-accelerator",
"source": "../../plugins/agentops",
"description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.",
"version": "0.3.7",
"version": "0.3.8",
"keywords": [
"agentops",
"evaluation",
Expand Down
18 changes: 18 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,24 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres

## [Unreleased]

## [0.3.8] - 2026-06-04

### Fixed
- **`agentops init` now handles blank required wizard values gracefully.** If
the user presses Enter without an existing Foundry endpoint or agent default,
the wizard explains that AgentOps needs the missing value and re-prompts
instead of proceeding to a later persistence failure. Scripted blank flags
such as `--agent ""` now exit with the same friendly message and no traceback.
- **`agentops init` no longer depends on undeclared PyYAML.** The setup wizard
now reads and writes `agentops.yaml` through the repository's `ruamel.yaml`
helpers, fixing the ugly `No module named 'yaml'` traceback seen in clean
installs.

### Changed
- **AgentOps brand tagline sequence now reads `Evaluate :: Ship :: Observe ::
Own`.** The startup/explain banner now matches the intended product story
order.

## [0.3.7] - 2026-06-01

### Fixed
Expand Down
2 changes: 1 addition & 1 deletion plugins/agentops/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"name": "agentops-accelerator",
"displayName": "AgentOps Accelerator — Skills for GitHub Copilot",
"description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.",
"version": "0.3.7",
"version": "0.3.8",
"publisher": "AgentOpsAccelerator",
"icon": "icon.png",
"license": "MIT",
Expand Down
2 changes: 1 addition & 1 deletion plugins/agentops/plugin.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "agentops-accelerator",
"description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.",
"version": "0.3.7",
"version": "0.3.8",
"author": {
"name": "AgentOps Accelerator",
"url": "https://github.com/Azure/agentops"
Expand Down
36 changes: 32 additions & 4 deletions src/agentops/cli/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -1365,6 +1365,7 @@ def cmd_init(
AGENT_TITLE,
DATASET_TITLE,
PROJECT_ENDPOINT_TITLE,
REQUIRED_CONFIGURATION_MESSAGE,
WizardAnswers,
apply_answers,
discover_defaults,
Expand Down Expand Up @@ -1476,16 +1477,37 @@ def cmd_init(
if any_flag:
# Scripted mode — validate then apply.
if project_endpoint is not None:
if not project_endpoint.strip():
typer.echo(
f"{_cli_error('Error')}: --project-endpoint is required. "
f"{REQUIRED_CONFIGURATION_MESSAGE}",
err=True,
)
raise typer.Exit(code=1)
err = validate_project_endpoint(project_endpoint)
if err:
typer.echo(f"{_cli_error('Error')}: --project-endpoint: {err}", err=True)
raise typer.Exit(code=1)
if agent is not None:
if not agent.strip():
typer.echo(
f"{_cli_error('Error')}: --agent is required. "
f"{REQUIRED_CONFIGURATION_MESSAGE}",
err=True,
)
raise typer.Exit(code=1)
err = validate_agent(agent)
if err:
typer.echo(f"{_cli_error('Error')}: --agent: {err}", err=True)
raise typer.Exit(code=1)
if dataset is not None:
if not dataset.strip():
typer.echo(
f"{_cli_error('Error')}: --dataset is required. "
f"{REQUIRED_CONFIGURATION_MESSAGE}",
err=True,
)
raise typer.Exit(code=1)
err = validate_dataset(dataset, workspace)
if err:
typer.echo(f"{_cli_error('Error')}: --dataset: {err}", err=True)
Expand Down Expand Up @@ -1605,8 +1627,14 @@ def _wizard_echo(msg: str) -> None:
default_env_name=target_env_name,
azd_env_name=azd_env_name,
)
except RuntimeError as exc:
typer.echo(f"{_cli_error('Error')}: {exc}", err=True)
except Exception as exc: # noqa: BLE001
typer.echo(
f"{_cli_error('Error')}: could not save AgentOps configuration. "
f"{REQUIRED_CONFIGURATION_MESSAGE}",
err=True,
)
if str(exc):
typer.echo(f"{_cli_warn('Details')}: {exc}", err=True)
raise typer.Exit(code=1)

typer.echo("")
Expand Down Expand Up @@ -4362,8 +4390,8 @@ def _colorize_block(
# fallback keeps the same words on a single line.
def _agentops_tagline() -> str:
if _terminal_unicode_enabled():
return "Evaluate · Observe · Diagnose · Ship — every Foundry agent."
return "Evaluate :: Observe :: Diagnose :: Ship -- every Foundry agent."
return "Evaluate · Ship · Observe · Own — every Foundry agent."
return "Evaluate :: Ship :: Observe :: Own -- every Foundry agent."


def _render_brand_block(
Expand Down
38 changes: 24 additions & 14 deletions src/agentops/services/setup_wizard.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,12 @@
ENV_KEY_PROJECT_ENDPOINT = "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT"
ENV_KEY_APPINSIGHTS = "APPLICATIONINSIGHTS_CONNECTION_STRING"

REQUIRED_CONFIGURATION_MESSAGE = (
"AgentOps needs a Foundry project endpoint, an agent, and a dataset path "
"before it can finish configuration. Enter the missing value, or press "
"Ctrl+C to cancel and re-run `agentops init` later."
)


# ---------------------------------------------------------------------------
# Data models
Expand Down Expand Up @@ -356,18 +362,16 @@ def _read_agentops_yaml(workspace: Path) -> dict:
if not path.exists():
return {}
try:
import yaml # noqa: PLC0415
except Exception: # noqa: BLE001
return {}
try:
data = yaml.safe_load(path.read_text(encoding="utf-8"))
from agentops.utils.yaml import load_yaml # noqa: PLC0415

data = load_yaml(path)
except Exception: # noqa: BLE001
return {}
return data if isinstance(data, dict) else {}


def _write_agentops_yaml(path: Path, data: dict) -> None:
import yaml # noqa: PLC0415
from agentops.utils.yaml import save_yaml # noqa: PLC0415

path.parent.mkdir(parents=True, exist_ok=True)
# Preserve simple field order for readability: version, agent, dataset,
Expand All @@ -381,10 +385,7 @@ def _write_agentops_yaml(path: Path, data: dict) -> None:
for key, value in data.items():
if key not in ordered:
ordered[key] = value
path.write_text(
yaml.safe_dump(ordered, sort_keys=False, default_flow_style=False),
encoding="utf-8",
)
save_yaml(path, ordered)


_AGENTOPS_ENV_HEADER = (
Expand Down Expand Up @@ -601,7 +602,11 @@ def _confirm_existing(label: str, value: str, secret: bool = False) -> None:
raw = prompt("Foundry project endpoint", effective_endpoint_default)
value = raw.strip()
if not value:
break # keep current / leave blank
if effective_endpoint_default:
break # keep current
echo(" ! Foundry project endpoint is required.")
echo(" ! " + REQUIRED_CONFIGURATION_MESSAGE)
continue
err = validate_project_endpoint(value)
if err:
echo(" ! " + err)
Expand All @@ -623,7 +628,11 @@ def _confirm_existing(label: str, value: str, secret: bool = False) -> None:
raw = prompt("Agent", defaults.agent)
value = raw.strip()
if not value:
break
if defaults.agent:
break # keep current
echo(" ! Agent is required.")
echo(" ! " + REQUIRED_CONFIGURATION_MESSAGE)
continue
err = validate_agent(value)
if err:
echo(" ! " + err)
Expand All @@ -641,11 +650,12 @@ def _confirm_existing(label: str, value: str, secret: bool = False) -> None:
echo("")
echo(DATASET_TITLE)
echo(_indent(DATASET_HELP))
dataset_default = defaults.dataset or ".agentops/data/smoke.jsonl"
while True:
raw = prompt("Dataset path", defaults.dataset or ".agentops/data/smoke.jsonl")
raw = prompt("Dataset path", dataset_default)
value = raw.strip()
if not value:
break
value = dataset_default
err = validate_dataset(value, workspace)
if err:
echo(" ! " + err)
Expand Down
14 changes: 12 additions & 2 deletions tests/unit/test_init_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,16 @@ def test_init_scripted_validates_project_endpoint(tmp_path: Path) -> None:
assert "Project endpoint" in result.output


def test_init_scripted_blank_required_value_is_friendly(tmp_path: Path) -> None:
"""Blank scripted values fail with a concise message, not a traceback."""
result = runner.invoke(app, ["init", "--dir", str(tmp_path), "--agent", ""])

assert result.exit_code == 1
assert "--agent is required" in result.output
assert "AgentOps needs a Foundry project endpoint" in result.output
assert "Traceback" not in result.output


def test_init_scripted_with_custom_azd_env(tmp_path: Path) -> None:
"""`--azd-env qa` writes to the explicit azd env instead of the local env."""
runner.invoke(app, ["init", "--dir", str(tmp_path), "--no-prompt"])
Expand Down Expand Up @@ -293,7 +303,7 @@ def test_init_prints_brand_banner(tmp_path: Path, monkeypatch) -> None:
# ASCII letterforms from _AGENTOPS_PLAIN_BANNER (figlet "Standard").
assert "____ _____ _ _ _____" in text
# The catchphrase, ASCII fallback variant.
assert "Evaluate :: Observe :: Diagnose :: Ship -- every Foundry agent." in text
assert "Evaluate :: Ship :: Observe :: Own -- every Foundry agent." in text


def test_brand_tagline_is_used_by_explain_pages(monkeypatch) -> None:
Expand All @@ -305,7 +315,7 @@ def test_brand_tagline_is_used_by_explain_pages(monkeypatch) -> None:
result = runner.invoke(app, ["init", "explain", "--no-pager"])
assert result.exit_code == 0
text = _strip_ansi(result.stdout)
assert "Evaluate :: Observe :: Diagnose :: Ship -- every Foundry agent." in text
assert "Evaluate :: Ship :: Observe :: Own -- every Foundry agent." in text


# ---------------------------------------------------------------------------
Expand Down
78 changes: 74 additions & 4 deletions tests/unit/test_setup_wizard.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import json
import os
import builtins
from pathlib import Path

import pytest
Expand Down Expand Up @@ -242,6 +243,29 @@ def test_apply_answers_writes_agent_and_dataset_to_yaml(tmp_path: Path):
assert "dataset: .agentops/data/smoke.jsonl" in text


def test_apply_answers_writes_yaml_without_pyyaml(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
real_import = builtins.__import__

def _import(name, *args, **kwargs): # noqa: ANN001
if name == "yaml":
raise ModuleNotFoundError("No module named 'yaml'")
return real_import(name, *args, **kwargs)

monkeypatch.setattr(builtins, "__import__", _import)

result = apply_answers(
tmp_path,
WizardAnswers(agent="my-bot:7", dataset=".agentops/data/smoke.jsonl"),
)

assert result.yaml_updated is True
assert "agent: my-bot:7" in (tmp_path / "agentops.yaml").read_text(
encoding="utf-8"
)


def test_apply_answers_does_not_write_project_endpoint_to_yaml(tmp_path: Path):
"""Environment-specific endpoints stay out of agentops.yaml."""
answers = WizardAnswers(
Expand Down Expand Up @@ -465,7 +489,7 @@ def test_run_wizard_does_not_prompt_for_appinsights(
assert "Application Insights" not in "\n".join(messages)


def test_run_wizard_empty_input_keeps_current(
def test_run_wizard_empty_input_reprompts_required_missing_endpoint(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.delenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", raising=False)
Expand All @@ -479,14 +503,46 @@ def test_run_wizard_empty_input_keeps_current(
# With idempotent skip-on-default, agent/dataset are silently reused.
# Only the still-empty project endpoint gets asked; App Insights is left
# for runtime discovery or explicit non-interactive configuration.
prompt = _scripted_prompt([""])
prompt = _scripted_prompt(["", "https://acct.services.ai.azure.com/api/projects/p"])
messages: list[str] = []
answers = run_wizard(
tmp_path, prompt=prompt, echo=lambda _msg: None, reconfigure=False
tmp_path, prompt=prompt, echo=messages.append, reconfigure=False
)
assert answers.project_endpoint is None
assert answers.project_endpoint == "https://acct.services.ai.azure.com/api/projects/p"
assert answers.agent is None
assert answers.dataset is None
assert answers.appinsights_connection_string is None
assert "Foundry project endpoint is required." in "\n".join(messages)


def test_run_wizard_reprompts_blank_required_values(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.delenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", raising=False)
monkeypatch.delenv("APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False)

data_dir = tmp_path / ".agentops" / "data"
data_dir.mkdir(parents=True)
(data_dir / "smoke.jsonl").write_text("{}\n", encoding="utf-8")
prompt = _scripted_prompt(
[
"",
"https://acct.services.ai.azure.com/api/projects/p",
"",
"my-bot:9",
"",
]
)
messages: list[str] = []

answers = run_wizard(tmp_path, prompt=prompt, echo=messages.append)

assert answers.project_endpoint == "https://acct.services.ai.azure.com/api/projects/p"
assert answers.agent == "my-bot:9"
assert answers.dataset == ".agentops/data/smoke.jsonl"
output = "\n".join(messages)
assert "Foundry project endpoint is required." in output
assert "Agent is required." in output


def test_run_wizard_force_prompt_fields_reasks_seed_agent_and_dataset(
Expand Down Expand Up @@ -543,6 +599,20 @@ def test_run_wizard_appinsights_is_not_interactive_even_when_missing(
"""The wizard should not ask for App Insights just to leave it blank."""
monkeypatch.delenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", raising=False)
monkeypatch.delenv("APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False)
_seed_azd_env(
tmp_path,
"dev",
{
"AZURE_AI_FOUNDRY_PROJECT_ENDPOINT": (
"https://acct.services.ai.azure.com/api/projects/p"
),
},
)
(tmp_path / "agentops.yaml").write_text(
"version: 1\nagent: keep:1\ndataset: keep.jsonl\n",
encoding="utf-8",
)
(tmp_path / "keep.jsonl").write_text("{}\n", encoding="utf-8")

messages: list[str] = []
prompt_calls: list[str] = []
Expand Down
Loading