diff --git a/cu-cli/CHANGELOG.md b/cu-cli/CHANGELOG.md index ae49137..3c72151 100644 --- a/cu-cli/CHANGELOG.md +++ b/cu-cli/CHANGELOG.md @@ -4,6 +4,9 @@ ### Features Added +- Added result limits, continuation tokens, and exact or prefix ID filtering to + `cu analyzer list`, while preserving complete array output for unlimited JSON + queries. - Added HTTPS and Azure Blob SAS URL inputs to `cu analyze` through repeatable `--url` options or standalone positional shortcuts, including URL-based analysis for large audio and video files without downloading them through the CLI. diff --git a/cu-cli/README.md b/cu-cli/README.md index 2c09492..f30e0c5 100644 --- a/cu-cli/README.md +++ b/cu-cli/README.md @@ -132,8 +132,32 @@ List the prebuilt analyzers available to the configured resource: ```bash cu analyzer list + +# Limit output and filter analyzer IDs by a case-sensitive prefix. +cu analyzer list --id-prefix prebuilt-tax --limit 25 + +# Retrieve one exact analyzer ID without scanning the analyzer collection. +cu analyzer list --id prebuilt-layout --json ``` +With `--limit`, JSON output is an object containing `items` and +`continuationToken`. Pass the token back with the same filters and sort order to +read the next result page: + +```bash +cu analyzer list \ + --id-prefix prebuilt-tax \ + --limit 25 \ + --continuation-token \ + --json +``` + +Without `--limit`, `--json` retains the original complete JSON array output. +The service currently exposes collection paging but no list filter or sort +parameters, so prefix filtering and sorting require the CLI to read all service +pages. Continuation results reflect the current analyzer collection and are not +a snapshot. + Start with the `prebuilt-layout` content extraction analyzer. It extracts text, paragraphs, tables, figures, and document structure without requiring a language model or embeddings model. `-a` is the short form of `--analyzer`: diff --git a/cu-cli/docs/usage-guide.md b/cu-cli/docs/usage-guide.md index 370e450..d4c3e45 100644 --- a/cu-cli/docs/usage-guide.md +++ b/cu-cli/docs/usage-guide.md @@ -552,6 +552,26 @@ Hyphens are reserved for service-provided prebuilt analyzer IDs. # List analyzers available on the selected resource. cu analyzer list +# Return one analyzer by exact ID. This uses the analyzer GET operation. +cu analyzer list --id invoice_v1 --json + +# Return up to 20 matching custom analyzers. +cu analyzer list \ + --id-prefix invoice_ \ + --kind custom \ + --sort-by analyzerId \ + --limit 20 \ + --json + +# Continue the same query with the token returned by the previous command. +cu analyzer list \ + --id-prefix invoice_ \ + --kind custom \ + --sort-by analyzerId \ + --limit 20 \ + --continuation-token \ + --json + # Print one analyzer definition. cu analyzer show invoice_v1 @@ -559,6 +579,27 @@ cu analyzer show invoice_v1 cu analyzer delete invoice_v1 ``` +`--id` and `--id-prefix` are mutually exclusive, and ID matching is +case-sensitive. A limited JSON result has this shape: + +```json +{ + "items": [], + "continuationToken": "" +} +``` + +Repeat the same `--id`, `--id-prefix`, `--kind`, and `--sort-by` values when +using a continuation token. The page size may change. A missing +`continuationToken` value means there are no more matches. Without `--limit`, +the command reads all pages and preserves the original JSON array output. + +The Content Understanding list API currently provides `nextLink` paging but no +server-side analyzer ID filter or sort parameters. The CLI therefore reads all +service pages before applying prefix, kind, and sort options. Continuation is a +stable CLI cursor over the current collection, not a service snapshot; analyzers +created, deleted, or modified between commands can affect later pages. + ### Create a schema Generate a starter schema: diff --git a/cu-cli/packages/core/src/cu_cli_core/command_spec.py b/cu-cli/packages/core/src/cu_cli_core/command_spec.py index a469e4a..1faf2ed 100644 --- a/cu-cli/packages/core/src/cu_cli_core/command_spec.py +++ b/cu-cli/packages/core/src/cu_cli_core/command_spec.py @@ -386,6 +386,18 @@ def _profile_name_arguments(option_help: str) -> tuple[ArgumentSpec, ...]: operation="cu_cli_core.operations.analyzers#list_analyzers", request_type="cu_cli_core.contracts#AnalyzerListRequest", arguments=( + ArgumentSpec( + "--id", + field="analyzer_id", + parser_name="analyzer_id", + help="Return the analyzer with this exact ID.", + ), + ArgumentSpec( + "--id-prefix", + field="id_prefix", + parser_name="id_prefix", + help="Filter analyzer IDs by a case-sensitive prefix.", + ), ArgumentSpec( "--kind", field="kind", @@ -402,11 +414,25 @@ def _profile_name_arguments(option_help: str) -> tuple[ArgumentSpec, ...]: default="analyzerId", choices=("analyzerId", "createdAt", "lastModifiedAt"), ), + ArgumentSpec( + "--limit", + field="limit", + parser_name="limit", + help="Maximum number of analyzers to return.", + value_type=ArgumentValueType.INTEGER, + minimum=1, + ), + ArgumentSpec( + "--continuation-token", + field="continuation_token", + parser_name="continuation_token", + help="Continue a limited query using the token from the previous result.", + ), ArgumentSpec( "--json", field="json", parser_name="json_output", - help="Write the complete result as JSON.", + help="Write JSON; limited results include a continuation token.", value_type=ArgumentValueType.BOOLEAN, classification=SurfaceClassification.FRONTEND_PRESENTATION, ), diff --git a/cu-cli/packages/core/src/cu_cli_core/contracts.py b/cu-cli/packages/core/src/cu_cli_core/contracts.py index dee72bb..639988b 100644 --- a/cu-cli/packages/core/src/cu_cli_core/contracts.py +++ b/cu-cli/packages/core/src/cu_cli_core/contracts.py @@ -127,12 +127,37 @@ def __post_init__(self) -> None: class AnalyzerListRequest: kind: str = "all" sort_by: str = "analyzerId" + analyzer_id: str | None = None + id_prefix: str | None = None + limit: int | None = None + continuation_token: str | None = None def __post_init__(self) -> None: if self.kind not in {"all", "prebuilt", "custom"}: raise ValueError(f"invalid analyzer kind: {self.kind}") if self.sort_by not in {"analyzerId", "createdAt", "lastModifiedAt"}: raise ValueError(f"invalid analyzer sort field: {self.sort_by}") + if self.analyzer_id is not None and self.id_prefix is not None: + raise ValueError("--id and --id-prefix cannot be used together.") + for field_name in ("analyzer_id", "id_prefix", "continuation_token"): + value = getattr(self, field_name) + if value is None: + continue + normalized = value.strip() + if not normalized: + option = field_name.replace("analyzer_id", "id").replace("_", "-") + raise ValueError(f"--{option} cannot be empty.") + object.__setattr__(self, field_name, normalized) + if self.limit is not None and self.limit < 1: + raise ValueError("--limit must be greater than zero.") + if self.continuation_token is not None and self.limit is None: + raise ValueError("--continuation-token requires --limit.") + + +@dataclass(frozen=True) +class AnalyzerListResult: + items: tuple[Any, ...] + continuation_token: str | None = None @dataclass(frozen=True) diff --git a/cu-cli/packages/core/src/cu_cli_core/operations/analyzers.py b/cu-cli/packages/core/src/cu_cli_core/operations/analyzers.py index 992cdc7..a31713d 100644 --- a/cu-cli/packages/core/src/cu_cli_core/operations/analyzers.py +++ b/cu-cli/packages/core/src/cu_cli_core/operations/analyzers.py @@ -5,9 +5,23 @@ from __future__ import annotations +import base64 +import binascii +import hashlib +import json from typing import Any -from ..errors import ConflictError, NotFoundError, ServiceError +from ..contracts import AnalyzerListResult +from ..errors import ConflictError, NotFoundError, ServiceError, ValidationError + + +_CONTINUATION_TOKEN_VERSION = 1 +_MAX_CONTINUATION_TOKEN_LENGTH = 4096 +_SORT_KEYS = { + "analyzerId": ("analyzer_id", "analyzerId"), + "createdAt": ("created_at", "createdAt"), + "lastModifiedAt": ("last_modified_at", "lastModifiedAt"), +} def _value(analyzer: Any, *keys: str) -> str: @@ -28,16 +42,118 @@ def list_analyzers( *, kind: str = "all", sort_by: str = "analyzerId", -) -> list[Any]: - items = list(client.list_analyzers()) + analyzer_id: str | None = None, + id_prefix: str | None = None, + limit: int | None = None, + continuation_token: str | None = None, +) -> AnalyzerListResult: + query_fingerprint = _query_fingerprint(kind, sort_by, analyzer_id, id_prefix) + cursor = ( + _decode_continuation_token(continuation_token, query_fingerprint) + if continuation_token is not None + else None + ) + + if analyzer_id is None: + items = list(client.list_analyzers()) + else: + from azure.core.exceptions import ResourceNotFoundError + + try: + items = [client.get_analyzer(analyzer_id)] + except ResourceNotFoundError: + items = [] + if kind != "all": items = [item for item in items if analyzer_kind(item) == kind] - sort_keys = { - "analyzerId": ("analyzer_id", "analyzerId"), - "createdAt": ("created_at", "createdAt"), - "lastModifiedAt": ("last_modified_at", "lastModifiedAt"), - } - return sorted(items, key=lambda item: _value(item, *sort_keys[sort_by])) + if id_prefix is not None: + items = [ + item + for item in items + if _value(item, "analyzer_id", "analyzerId").startswith(id_prefix) + ] + + items.sort(key=lambda item: _sort_key(item, sort_by)) + if cursor is not None: + items = [item for item in items if _sort_key(item, sort_by) > cursor] + + page_items = items if limit is None else items[:limit] + next_token = None + if limit is not None and len(items) > limit: + next_token = _encode_continuation_token( + _sort_key(page_items[-1], sort_by), + query_fingerprint, + ) + return AnalyzerListResult(tuple(page_items), next_token) + + +def _sort_key(analyzer: Any, sort_by: str) -> tuple[str, str]: + analyzer_id = _value(analyzer, "analyzer_id", "analyzerId") + return _value(analyzer, *_SORT_KEYS[sort_by]), analyzer_id + + +def _query_fingerprint( + kind: str, + sort_by: str, + analyzer_id: str | None, + id_prefix: str | None, +) -> str: + query = json.dumps( + { + "analyzerId": analyzer_id, + "idPrefix": id_prefix, + "kind": kind, + "sortBy": sort_by, + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(query).hexdigest() + + +def _encode_continuation_token(cursor: tuple[str, str], query_fingerprint: str) -> str: + payload = json.dumps( + { + "version": _CONTINUATION_TOKEN_VERSION, + "query": query_fingerprint, + "after": list(cursor), + }, + separators=(",", ":"), + ).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def _decode_continuation_token( + token: str, + query_fingerprint: str, +) -> tuple[str, str]: + if len(token) > _MAX_CONTINUATION_TOKEN_LENGTH: + raise ValidationError("invalid continuation token.") + try: + padding = "=" * (-len(token) % 4) + decoded = base64.b64decode( + (token + padding).encode("ascii"), + altchars=b"-_", + validate=True, + ) + payload = json.loads(decoded.decode("utf-8")) + except (binascii.Error, UnicodeError, ValueError): + raise ValidationError("invalid continuation token.") from None + + if not isinstance(payload, dict) or payload.get("version") != _CONTINUATION_TOKEN_VERSION: + raise ValidationError("invalid continuation token.") + if payload.get("query") != query_fingerprint: + raise ValidationError( + "continuation token does not match the current analyzer filters and sort order." + ) + cursor = payload.get("after") + if ( + not isinstance(cursor, list) + or len(cursor) != 2 + or not all(isinstance(value, str) for value in cursor) + ): + raise ValidationError("invalid continuation token.") + return cursor[0], cursor[1] def analyzer_kind(analyzer: Any) -> str: diff --git a/cu-cli/packages/core/tests/test_command_spec.py b/cu-cli/packages/core/tests/test_command_spec.py index 0666de6..96ec208 100644 --- a/cu-cli/packages/core/tests/test_command_spec.py +++ b/cu-cli/packages/core/tests/test_command_spec.py @@ -199,11 +199,19 @@ def test_defaults_set_does_not_implicitly_select_profile_mappings(): def test_frontend_presentation_arguments_are_not_bound_to_core_request(): request = build_request( ANALYZER_LIST, - {"kind": "custom", "sort_by": "createdAt", "json_output": True}, + { + "id_prefix": "invoice_", + "kind": "custom", + "sort_by": "createdAt", + "limit": 25, + "json_output": True, + }, ) + assert request.id_prefix == "invoice_" assert request.kind == "custom" assert request.sort_by == "createdAt" + assert request.limit == 25 @pytest.mark.parametrize( diff --git a/cu-cli/packages/core/tests/test_operations.py b/cu-cli/packages/core/tests/test_operations.py index d7e0a89..c93ef8a 100644 --- a/cu-cli/packages/core/tests/test_operations.py +++ b/cu-cli/packages/core/tests/test_operations.py @@ -1,11 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from types import SimpleNamespace from unittest.mock import Mock import pytest +from azure.core.exceptions import ResourceNotFoundError -from cu_cli_core.operations.analyzers import get_analyzer +from cu_cli_core.errors import ValidationError +from cu_cli_core.operations.analyzers import get_analyzer, list_analyzers pytestmark = pytest.mark.unit @@ -18,3 +21,114 @@ def test_get_analyzer_delegates_to_injected_client(): client.get_analyzer.assert_called_once_with("invoice-v1") assert result == {"analyzerId": "invoice-v1"} + + +def _analyzer( + analyzer_id: str, + created_at: str = "2026-01-01", + last_modified_at: str = "2026-01-01", +): + return SimpleNamespace( + analyzer_id=analyzer_id, + created_at=created_at, + last_modified_at=last_modified_at, + ) + + +class _PagedAnalyzerClient: + def __init__(self, pages): + self.pages = pages + self.list_calls = 0 + self.get_calls = [] + + def list_analyzers(self): + self.list_calls += 1 + return (item for page in self.pages for item in page) + + def get_analyzer(self, analyzer_id): + self.get_calls.append(analyzer_id) + for page in self.pages: + for item in page: + if item.analyzer_id == analyzer_id: + return item + raise ResourceNotFoundError("not found") + + +def _ids(result): + return [item.analyzer_id for item in result.items] + + +def test_list_analyzers_pages_filtered_results_without_duplicates_or_gaps(): + client = _PagedAnalyzerClient( + [ + [_analyzer("invoice_c"), _analyzer("other")], + [_analyzer("invoice_a"), _analyzer("invoice_b")], + ] + ) + + first = list_analyzers(client, id_prefix="invoice_", limit=2) + second = list_analyzers( + client, + id_prefix="invoice_", + limit=2, + continuation_token=first.continuation_token, + ) + + assert _ids(first) == ["invoice_a", "invoice_b"] + assert first.continuation_token is not None + assert _ids(second) == ["invoice_c"] + assert second.continuation_token is None + assert client.list_calls == 2 + + +def test_list_analyzers_exact_id_uses_get_and_honors_kind(): + client = _PagedAnalyzerClient([[_analyzer("custom_v1"), _analyzer("prebuilt-layout")]]) + + found = list_analyzers(client, analyzer_id="custom_v1", kind="custom") + excluded = list_analyzers(client, analyzer_id="custom_v1", kind="prebuilt") + missing = list_analyzers(client, analyzer_id="missing") + + assert _ids(found) == ["custom_v1"] + assert _ids(excluded) == [] + assert _ids(missing) == [] + assert client.get_calls == ["custom_v1", "custom_v1", "missing"] + assert client.list_calls == 0 + + +def test_list_analyzers_rejects_invalid_token_before_service_call(): + client = _PagedAnalyzerClient([[_analyzer("custom_v1")]]) + + with pytest.raises(ValidationError, match="invalid continuation token"): + list_analyzers(client, limit=1, continuation_token="not-a-token") + + assert client.list_calls == 0 + + +def test_list_analyzers_rejects_token_when_query_changes(): + client = _PagedAnalyzerClient([[_analyzer("a"), _analyzer("b")]]) + first = list_analyzers(client, limit=1) + + with pytest.raises(ValidationError, match="does not match"): + list_analyzers( + client, + kind="custom", + limit=1, + continuation_token=first.continuation_token, + ) + + +def test_list_analyzers_uses_id_as_stable_sort_tiebreaker(): + client = _PagedAnalyzerClient( + [[_analyzer("b", created_at="same"), _analyzer("a", created_at="same")]] + ) + + first = list_analyzers(client, sort_by="createdAt", limit=1) + second = list_analyzers( + client, + sort_by="createdAt", + limit=1, + continuation_token=first.continuation_token, + ) + + assert _ids(first) == ["a"] + assert _ids(second) == ["b"] diff --git a/cu-cli/packages/standalone/src/cu_cli/commands/analyzer.py b/cu-cli/packages/standalone/src/cu_cli/commands/analyzer.py index f22988b..40f99d7 100644 --- a/cu-cli/packages/standalone/src/cu_cli/commands/analyzer.py +++ b/cu-cli/packages/standalone/src/cu_cli/commands/analyzer.py @@ -191,32 +191,58 @@ def analyzer_group() -> None: epilog=common_commands( ("cu analyzer list", "List all analyzers as a table."), ("cu analyzer list --kind custom", "List only custom analyzers."), - ("cu analyzer list --json", "List analyzers as machine-readable JSON."), + ("cu analyzer list --json", "List all analyzers as machine-readable JSON."), + ("cu analyzer list --id-prefix invoice --limit 50", "List a filtered result page."), ), ) @with_command_arguments(ANALYZER_LIST) @with_auth_options @friendly_errors def cmd_list( - kind, sort_by, json_output, endpoint, api_key, api_version, entra, profile_name, - show_runtime_context, show_calling_time + analyzer_id, id_prefix, kind, sort_by, limit, continuation_token, json_output, + endpoint, api_key, api_version, entra, profile_name, show_runtime_context, + show_calling_time ) -> None: - request = build_request( - ANALYZER_LIST, - {"kind": kind, "sort_by": sort_by, "json_output": json_output}, - ) + try: + request = build_request( + ANALYZER_LIST, + { + "analyzer_id": analyzer_id, + "id_prefix": id_prefix, + "kind": kind, + "sort_by": sort_by, + "limit": limit, + "continuation_token": continuation_token, + "json_output": json_output, + }, + ) + except CommandBindingError as exc: + raise CuCliError(str(exc), exit_code=VALIDATION_FAILURE) from exc client = _client(endpoint, api_key, api_version, entra, profile_name, show_runtime_context) with calling_time(show_calling_time) as calling_timer: - items = resolve_identifier(ANALYZER_LIST.operation)( + result = resolve_identifier(ANALYZER_LIST.operation)( client, kind=request.kind, sort_by=request.sort_by, + analyzer_id=request.analyzer_id, + id_prefix=request.id_prefix, + limit=request.limit, + continuation_token=request.continuation_token, ) if json_output: - dump_json([a.as_dict() for a in items]) + items = [a.as_dict() for a in result.items] + if request.limit is None: + dump_json(items) + else: + dump_json({"items": items, "continuationToken": result.continuation_token}) else: - console.print(analyzer_table(items)) - console.print(f"\n[dim]{len(items)} analyzer(s)[/dim]") + console.print(analyzer_table(result.items)) + console.print(f"\n[dim]{len(result.items)} analyzer(s)[/dim]") + if result.continuation_token is not None: + console.print( + "[dim]continuation token:[/dim] " + f"[cyan]{result.continuation_token}[/cyan]" + ) calling_timer.print() diff --git a/cu-cli/packages/standalone/tests/unit/test_cli.py b/cu-cli/packages/standalone/tests/unit/test_cli.py index 060b170..367ac65 100644 --- a/cu-cli/packages/standalone/tests/unit/test_cli.py +++ b/cu-cli/packages/standalone/tests/unit/test_cli.py @@ -2001,13 +2001,27 @@ def as_dict(self): class _FakeListAnalyzerClient: + def __init__(self): + self.list_calls = 0 + self.get_calls = [] + def list_analyzers(self): + self.list_calls += 1 return [ _FakeListAnalyzer("my_custom_v1", "2026-01-03T00:00:00Z", "2026-01-01T00:00:00Z"), _FakeListAnalyzer("prebuilt-invoice", "2026-01-01T00:00:00Z", "2026-01-03T00:00:00Z"), _FakeListAnalyzer("prebuilt-document", "2026-01-02T00:00:00Z", "2026-01-02T00:00:00Z"), ] + def get_analyzer(self, analyzer_id): + self.get_calls.append(analyzer_id) + for analyzer in self.list_analyzers(): + if analyzer.analyzer_id == analyzer_id: + return analyzer + from azure.core.exceptions import ResourceNotFoundError + + raise ResourceNotFoundError("not found") + class _FakeConfigListClient: def get_defaults(self): @@ -2253,6 +2267,90 @@ def test_analyzer_list_kind_all_is_default(monkeypatch): assert len(payload) == 3 # no filtering by default +def test_analyzer_list_limited_json_can_continue(monkeypatch): + client = _FakeListAnalyzerClient() + monkeypatch.setattr("cu_cli.commands.analyzer._client", lambda *_a, **_k: client) + + first = _run("analyzer", "list", "--json", "--limit", "2") + assert first.exit_code == 0, first.output + first_payload = json.loads(first.output) + token = first_payload["continuationToken"] + assert [item["analyzerId"] for item in first_payload["items"]] == [ + "my_custom_v1", + "prebuilt-document", + ] + assert token + + second = _run( + "analyzer", "list", "--json", "--limit", "2", + "--continuation-token", token, + ) + assert second.exit_code == 0, second.output + second_payload = json.loads(second.output) + assert [item["analyzerId"] for item in second_payload["items"]] == [ + "prebuilt-invoice" + ] + assert second_payload["continuationToken"] is None + + +def test_analyzer_list_limited_table_prints_continuation_token(monkeypatch): + monkeypatch.setattr("cu_cli.commands.analyzer._client", lambda *_a, **_k: _FakeListAnalyzerClient()) + + result = _run("analyzer", "list", "--limit", "1") + + assert result.exit_code == 0, result.output + assert "1 analyzer(s)" in result.output + assert "continuation token:" in result.output + + +def test_analyzer_list_combines_prefix_kind_and_sort(monkeypatch): + monkeypatch.setattr("cu_cli.commands.analyzer._client", lambda *_a, **_k: _FakeListAnalyzerClient()) + + result = _run( + "analyzer", "list", "--json", "--id-prefix", "prebuilt-", + "--kind", "prebuilt", "--sort-by", "createdAt", + ) + + assert result.exit_code == 0, result.output + assert [item["analyzerId"] for item in json.loads(result.output)] == [ + "prebuilt-invoice", + "prebuilt-document", + ] + + +def test_analyzer_list_exact_id_uses_single_resource_lookup(monkeypatch): + client = _FakeListAnalyzerClient() + monkeypatch.setattr("cu_cli.commands.analyzer._client", lambda *_a, **_k: client) + + result = _run("analyzer", "list", "--json", "--id", "prebuilt-invoice") + + assert result.exit_code == 0, result.output + assert [item["analyzerId"] for item in json.loads(result.output)] == [ + "prebuilt-invoice" + ] + assert client.get_calls == ["prebuilt-invoice"] + + +@pytest.mark.parametrize( + "arguments, message", + [ + (("--limit", "0"), "0 is not in the range"), + (("--continuation-token", "bad"), "requires --limit"), + (("--id", "one", "--id-prefix", "o"), "cannot be used together"), + (("--limit", "1", "--continuation-token", "bad"), "invalid continuation token"), + ], +) +def test_analyzer_list_rejects_invalid_paging_arguments(monkeypatch, arguments, message): + client = _FakeListAnalyzerClient() + monkeypatch.setattr("cu_cli.commands.analyzer._client", lambda *_a, **_k: client) + + result = _run("analyzer", "list", *arguments) + + assert result.exit_code == 2 + assert message in result.output + assert client.list_calls == 0 + + def test_analyzer_list_sort_by_analyzer_id(monkeypatch): monkeypatch.setattr("cu_cli.commands.analyzer._client", lambda *_a, **_k: _FakeListAnalyzerClient()) res = _run("analyzer", "list", "--json", "--sort-by", "analyzerId")