diff --git a/.claude/skills/use-cmeta-cli/SKILL.md b/.claude/skills/use-cmeta-cli/SKILL.md index 28f5e9c..79d905c 100644 --- a/.claude/skills/use-cmeta-cli/SKILL.md +++ b/.claude/skills/use-cmeta-cli/SKILL.md @@ -89,14 +89,15 @@ Parsed by `cmeta/utils/cli.py::parse_cmd`. Rules: | `--key` | boolean `True` | | `--key-` | boolean `False` | | `--no-key` | boolean `False` | -| `--key=value` / `--key value` | string / int (repeats overwrite — last wins) | +| `--key=value` | string / int (repeats overwrite — last wins); `--key value` is the flag `--key` (True) + a positional `value` | | `--key,=v1,v2,v3` | list `[v1, v2, v3]` (trailing comma on the key requests split) | | `--key,=` | empty list `[]` | | `--parent.child=v` | nested dict `{'parent': {'child': v}}` | -| `key=value` (no dashes) | same as `--key=value` | +| `key=value` (no dashes) | same as `--key=value` when the part before `=` looks like a key (a letter or `_`, then letters, digits, `_ . , -`); other text with `=` (`"echo a=b"`, a URL with a query) is positional | +| `-5`, `-1.5` | positional (a negative number is not a flag) | | `@somefile.yaml` | inline the YAML/JSON file as flags (deep-merged) | | `@@somefile.yaml` | same, then delete the file | -| `--` | everything after → `unparsed` list | +| `--` | everything after → `unparsed` list, for the commands that read it (others say "takes no arguments after --") | | bare token | positional → `arg1`, `arg2`, ... | Examples: diff --git a/CHANGELOG.md b/CHANGELOG.md index f25489e..5bf76cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,25 @@ All notable changes to cMeta are documented here, newest first. +## 0.36.0 +- **The command line: free text with `=` stays text, a negative number is a value, a control flag takes a word, + and `--` is refused in words.** Four small corrections of the parser, none of which changes a command that + worked: (1) a bare `key=value` is a flag only when the part before the first `=` looks like a key (a letter or + `_`, then letters, digits, `_`, `.`, `,`, `-`) - `cx host run h "echo a=b"`, a URL with a query string or a + Python one-liner with `==` was cut into a parameter named by its first words ("unexpected keyword argument + 'echo a'") and is one positional argument now; every key that worked (`name=v`, `a.b=v`, `tags,=a,b`, + `name=John Smith`, `path=C:\x\y`) works as before; (2) a negative number (`-5`, `-1.5`) is a positional + argument, not a flag named `5` (`--delta=-5` was and is a value); (3) the engine's own control flags + (`--quiet`, `--verbose`, `--json`, `--debug`, ...) given a word - `--quiet=yes`, `--json=no` - took the value + converted for the flag before them or failed with "cannot access local variable 'converted_value'" (a typo); + they read yes/no, true/false, on/off, 1/0 now and refuse any other word by name; (4) text after `--` + reaches a command as `unparsed`, and a keyword-style command that has no such parameter says `" + " takes no arguments after "--"` instead of Python's TypeError. The docs said `--key value` was a + form: it never was (`--key value` is the flag `--key` and a positional `value`) and the docs say so now. + Tests: `tests/core_tests/test_cli_parse_table.py` - the table of every token kind, every bool control flag + with every word, the three command styles with text after `--`, and the command line end to end. + + ## 0.35.2 - **Any lock timeout is waited for as given, and it is the whole wait.** `CMETA_LOCK_TIMEOUT`, `CMETA_INDEX_LOCK_TIMEOUT` and the cache wait of cmeta-aops (`CMETA_CACHE_WAIT_TIMEOUT`) took `inf`, `nan` or a diff --git a/CITATION.cff b/CITATION.cff index 74a5669..ddb76ae 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -13,7 +13,7 @@ authors: website: "https://cTuning.ai/@gfursin" affiliation: "cTuning Labs" orcid: "https://orcid.org/0000-0001-7719-1624" -version: "0.35.2" +version: "0.36.0" date-released: "2026-10-13" license: Apache-2.0 repository-code: "https://github.com/cTuningLabs/cmeta" diff --git a/README.md b/README.md index 014d8c1..5b34047 100644 --- a/README.md +++ b/README.md @@ -435,7 +435,8 @@ command): ### Flag syntax at a glance - Booleans: `--flag` (True), `--flag-` (False), `--no-flag` (False). -- Strings: `--key=value` or `--key value` (repeats overwrite — last wins). +- Strings: `--key=value` (repeats overwrite — last wins); `--key value` is the + flag `--key` and a positional `value`. - Lists: `--key,=v1,v2,v3` (trailing comma on the key). - Nested dicts: `--parent.child=value` merges into `{'parent': {'child': ...}}`. - Reusable param files: `@input.yaml` / `@input.json` deep-merges into the diff --git a/cmeta/core.py b/cmeta/core.py index 9ff2c37..2f31a2d 100644 --- a/cmeta/core.py +++ b/cmeta/core.py @@ -871,9 +871,31 @@ def access( self.logger.debug(f'Calling {command_func_name}() @ {filename}:{start_line}-{end_line}') self.logger.debug(f' with parameters {command_params} ...') + # The text after "--" reaches a command as "unparsed": a keyword-style function without such a + # parameter (and without **kwargs) cannot take it - say so in words, not with Python's TypeError + keyword_style = command_func_name.endswith('_') and not command_func_name.endswith('__') + + if keyword_style and 'unparsed' in command_params: + try: + takes_unparsed = any(p.name == 'unparsed' or p.kind == inspect.Parameter.VAR_KEYWORD + for p in inspect.signature(func).parameters.values()) + except (TypeError, ValueError): + takes_unparsed = True # no signature to read: the call itself will tell + + if not takes_unparsed: + r = utils.names.restore_cmeta_obj(cmeta_ref_parts, key='artifact', fail_on_error = self_fail_on_error) + if self.catch_error(r): return r + category_str = r['obj'] + + extra_flags_help = category_meta.get('extra_flags_help', '') + + return self.error(f'"{category_str} {command}" takes no arguments after "--" (the text after "--" ' + f'is passed to the command unparsed, and this command does not read it).\n\n' + f'Rerun with {extra_flags_help}--help to view API usage and options') + try: command_params['ctx'] = ctx - if command_func_name.endswith('_') and not command_func_name.endswith('__'): + if keyword_style: result = func(**command_params) else: result = func(command_params) diff --git a/cmeta/internal-repo/_cmr.yaml b/cmeta/internal-repo/_cmr.yaml index fb55847..d002f96 100644 --- a/cmeta/internal-repo/_cmr.yaml +++ b/cmeta/internal-repo/_cmr.yaml @@ -3,5 +3,5 @@ authors: "[Grigori Fursin](https://cTuning.ai/@gfursin)" category: repo,f4f792ab40c7498f copyright: "2025-2026 Grigori Fursin and cTuning Labs. See the cMeta COPYRIGHT and LICENSE files for details." permanent: True -version: '0.35.2' +version: '0.36.0' version_date: '20261013' diff --git a/cmeta/internal-repo/category/utils/api/native_host.py b/cmeta/internal-repo/category/utils/api/native_host.py new file mode 100644 index 0000000..a810684 --- /dev/null +++ b/cmeta/internal-repo/category/utils/api/native_host.py @@ -0,0 +1,234 @@ +""" +Run and register cMeta artifacts as Mozilla native messaging hosts. + +cMeta author and developer: (C) 2025-2026 Grigori Fursin + +See the cMeta COPYRIGHT and LICENSE files in the project root for details. +""" + +import importlib.util +import json +import os +from pathlib import Path +import re +import struct +import subprocess +import sys + + +MAX_INPUT = 4 * 1024 * 1024 +MAX_OUTPUT = 1024 * 1024 - 1 +REF_PATTERN = re.compile(r'^[A-Za-z0-9_.,:-]+::[A-Za-z0-9_.,:-]+$') +HOST_PATTERN = re.compile(r'^[a-z0-9_.-]+$') + + +def resolve(cm, ref): + """Resolve a host artifact in the active cMeta home, then validate its declaration.""" + if not isinstance(ref, str) or not REF_PATTERN.fullmatch(ref): + raise ValueError('Give one cMeta category::artifact reference (UIDs are recommended).') + category, artifact = ref.split('::', 1) + found = cm.access({'category': category, 'command': 'find', 'arg1': artifact, + 'con': False, 'quiet': True}) + if found.get('return'): + raise RuntimeError(found.get('error') or 'Native host artifact lookup failed.') + matches = found.get('artifacts', []) + if len(matches) != 1: + raise RuntimeError('Expected one native host artifact in the active cMeta home; found ' + str(len(matches)) + '.') + root = Path(matches[0]['path']).resolve() + declaration = json.loads((root / 'native-host.json').read_text(encoding='utf-8')) + name = declaration.get('name') + extensions = declaration.get('allowed_extensions') + entry_name = declaration.get('entry') + checks = declaration.get('checks', [{'command': 'ping'}]) + if not isinstance(name, str) or not HOST_PATTERN.fullmatch(name): + raise ValueError('Invalid native host name in native-host.json.') + if not isinstance(extensions, list) or not extensions or not all(isinstance(x, str) and x for x in extensions): + raise ValueError('Invalid allowed_extensions in native-host.json.') + if not isinstance(entry_name, str) or not entry_name.endswith('.py'): + raise ValueError('Invalid Python entry in native-host.json.') + entry = (root / entry_name).resolve() + if root not in entry.parents or not entry.is_file(): + raise ValueError('Native host entry must be a Python file inside its artifact.') + if not isinstance(checks, list) or not checks or not all(isinstance(x, dict) for x in checks): + raise ValueError('Invalid checks in native-host.json.') + return {'ref': ref, 'root': root, 'entry': entry, 'name': name, + 'description': declaration.get('description', name), + 'allowed_extensions': extensions, 'checks': checks} + + +def locations(name): + """Keep launchers independent of the repository and selected cMeta home.""" + if os.name == 'nt': + base = Path(os.environ.get('LOCALAPPDATA', str(Path.home() / 'AppData/Local'))) / 'cmeta/native-messaging' + return base / (name + '.bat'), base / (name + '.json') + if sys.platform == 'darwin': + launcher = Path.home() / 'Library/Application Support/cmeta/native-messaging' / (name + '.sh') + manifest = Path.home() / 'Library/Application Support/Mozilla/NativeMessagingHosts' / (name + '.json') + return launcher, manifest + data = Path(os.environ.get('XDG_DATA_HOME', str(Path.home() / '.local/share'))) + launcher = data / 'cmeta/native-messaging' / (name + '.sh') + manifest = Path.home() / '.mozilla/native-messaging-hosts' / (name + '.json') + return launcher, manifest + + +def _registered_manifest(name): + if os.name != 'nt': + return None + import winreg + key_name = 'Software\\Mozilla\\NativeMessagingHosts\\' + name + try: + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_name) as key: + return Path(winreg.QueryValueEx(key, '')[0]) + except FileNotFoundError: + return None + + +def _write_registration(name, manifest): + if os.name == 'nt': + import winreg + key_name = 'Software\\Mozilla\\NativeMessagingHosts\\' + name + with winreg.CreateKey(winreg.HKEY_CURRENT_USER, key_name) as key: + winreg.SetValueEx(key, '', 0, winreg.REG_SZ, str(manifest)) + + +def _owned_previous(previous, host): + """Permit migration only from this same artifact's legacy registration.""" + if not previous or not previous.is_file(): + return False + try: + data = json.loads(previous.read_text(encoding='utf-8')) + launcher = Path(data['path']).resolve() + current_launcher, _ = locations(host['name']) + same_name = data.get('name') == host['name'] + same_extensions = data.get('allowed_extensions') == host['allowed_extensions'] + same_artifact = host['root'] in launcher.parents + stable_launcher = launcher == current_launcher.resolve() + return same_name and same_extensions and (same_artifact or stable_launcher) + except (OSError, ValueError, KeyError): + return False + + +def install(cm, ref, apply=False): + host = resolve(cm, ref) + launcher, manifest = locations(host['name']) + previous = _registered_manifest(host['name']) if os.name == 'nt' else (manifest if manifest.exists() else None) + if previous and previous.resolve() != manifest.resolve() and not _owned_previous(previous, host): + raise RuntimeError('Native host name is registered to a different artifact: ' + str(previous)) + if manifest.exists() and not _owned_previous(manifest, host): + raise RuntimeError('Refusing to overwrite a different native host manifest: ' + str(manifest)) + expected = {'name': host['name'], 'description': host['description'], + 'path': str(launcher), 'type': 'stdio', + 'allowed_extensions': host['allowed_extensions']} + # A cRef contains only shell-safe characters. The active cMeta CLI on PATH + # resolves the current home and artifact afresh on every browser request. + args = ' utils native-host-run ' + ref + ' --quiet' + if os.name == 'nt': + content = ('@echo off\r\nsetlocal DisableDelayedExpansion\r\n' + 'where.exe cx >nul 2>nul\r\nif errorlevel 1 goto try_cmeta\r\n' + 'cx' + args + '\r\nexit /b %errorlevel%\r\n' + ':try_cmeta\r\nwhere.exe cmeta >nul 2>nul\r\n' + 'if errorlevel 1 goto missing\r\ncmeta' + args + '\r\n' + 'exit /b %errorlevel%\r\n:missing\r\n' + '>&2 echo cMeta CLI cx or cmeta is not on PATH.\r\nexit /b 127\r\n') + else: + content = ('#!/bin/sh\n' + 'if command -v cx >/dev/null 2>&1; then exec cx' + args + '; fi\n' + 'if command -v cmeta >/dev/null 2>&1; then exec cmeta' + args + '; fi\n' + 'echo "cMeta CLI cx or cmeta is not on PATH." >&2\nexit 127\n') + print('Active cMeta home:', cm.home_path) + print('Native host artifact:', host['root']) + print('Native host manifest:', manifest) + if not apply: + print('Review only. Add --apply to register the host.') + return {'return': 0, 'manifest': str(manifest), 'applied': False} + launcher.parent.mkdir(parents=True, exist_ok=True) + manifest.parent.mkdir(parents=True, exist_ok=True) + launcher.write_bytes(content.encode('mbcs' if os.name == 'nt' else 'utf-8')) + if os.name != 'nt': + launcher.chmod(0o700) + manifest.write_text(json.dumps(expected, indent=2) + '\n', encoding='utf-8') + _write_registration(host['name'], manifest) + return {'return': 0, 'manifest': str(manifest), 'applied': True} + + +def _send(command, request, home): + body = json.dumps(request).encode('utf-8') + env = os.environ.copy() + env['CMETA_HOME'] = str(home) + result = subprocess.run(command, input=struct.pack('=I', len(body)) + body, + capture_output=True, timeout=30, env=env, + creationflags=subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0) + if result.returncode or len(result.stdout) < 4: + raise RuntimeError(result.stderr.decode(errors='replace') or 'Native host returned no frame.') + count = struct.unpack('=I', result.stdout[:4])[0] + if count != len(result.stdout) - 4: + raise RuntimeError('Invalid native host framing.') + response = json.loads(result.stdout[4:]) + if response.get('return'): + raise RuntimeError(response.get('error') or str(response)) + return response + + +def check(cm, ref): + host = resolve(cm, ref) + launcher, manifest = locations(host['name']) + if not launcher.is_file() or not manifest.is_file(): + raise RuntimeError('Native host is not installed. Run native-host-install --apply first.') + registered = _registered_manifest(host['name']) if os.name == 'nt' else manifest + if not registered or registered.resolve() != manifest.resolve(): + raise RuntimeError('Native host registration points elsewhere.') + data = json.loads(manifest.read_text(encoding='utf-8')) + if data.get('path') != str(launcher) or data.get('allowed_extensions') != host['allowed_extensions']: + raise RuntimeError('Native host manifest does not match this artifact.') + if os.name == 'nt': + command = subprocess.list2cmdline([os.environ.get('COMSPEC', 'cmd.exe')]) + \ + ' /d /v:off /s /c ""' + str(launcher) + '""' + else: + command = [str(launcher)] + for request in host['checks']: + _send(command, request, cm.home_path) + print('Native host and backend checks passed:', host['name']) + print('Active cMeta home:', cm.home_path) + return {'return': 0, 'host': host['name']} + + +def _read_exact(stream, count): + result = bytearray() + while len(result) < count: + chunk = stream.read(count - len(result)) + if not chunk: + raise ValueError('Incomplete native message.') + result.extend(chunk) + return bytes(result) + + +def serve(cm, ref): + """Preserve a private binary stdout stream before loading any plugin code.""" + if os.name == 'nt': + import msvcrt + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + wire = os.fdopen(os.dup(sys.stdout.fileno()), 'wb', buffering=0) + os.dup2(sys.stderr.fileno(), sys.stdout.fileno()) + try: + host = resolve(cm, ref) + size = struct.unpack('=I', _read_exact(sys.stdin.buffer, 4))[0] + if size > MAX_INPUT: + raise ValueError('Native message is too large.') + request = json.loads(_read_exact(sys.stdin.buffer, size).decode('utf-8')) + spec = importlib.util.spec_from_file_location('cmeta_native_host_' + host['name'], host['entry']) + if spec is None or spec.loader is None: + raise RuntimeError('Could not load the native host entry.') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + response = module.dispatch(request, cm=cm) + if not isinstance(response, dict): + raise TypeError('Native host dispatch must return a dictionary.') + except Exception as exc: + response = {'return': 1, 'error': str(exc)} + packet = json.dumps(response, ensure_ascii=False).encode('utf-8') + if len(packet) > MAX_OUTPUT: + packet = json.dumps({'return': 1, 'error': 'The result exceeds the native messaging limit.'}).encode('utf-8') + wire.write(struct.pack('=I', len(packet)) + packet) + wire.flush() + return {'return': 0} diff --git a/cmeta/internal-repo/category/utils/api/v1.py b/cmeta/internal-repo/category/utils/api/v1.py index 6fc0287..137401c 100644 --- a/cmeta/internal-repo/category/utils/api/v1.py +++ b/cmeta/internal-repo/category/utils/api/v1.py @@ -41,6 +41,27 @@ def __init__( """ super().__init__(*args, module_file_path = __file__, **kwargs) + def native_host_install_(self, ctx, arg1=None, apply=False): + """Register a native messaging host declared by a cMeta artifact.""" + from . import native_host + try: + return native_host.install(self.cm, arg1, apply=apply) + except Exception as exc: + return self.cm.error(str(exc)) + + def native_host_check_(self, ctx, arg1=None): + """Check native messaging transport and the artifact's declared requests.""" + from . import native_host + try: + return native_host.check(self.cm, arg1) + except Exception as exc: + return self.cm.error(str(exc)) + + def native_host_run_(self, ctx, arg1=None): + """Serve one framed native message using cx's active cMeta home.""" + from . import native_host + return native_host.serve(self.cm, arg1) + ############################################################ def test( self, diff --git a/cmeta/utils/cli.py b/cmeta/utils/cli.py index d819249..922206a 100644 --- a/cmeta/utils/cli.py +++ b/cmeta/utils/cli.py @@ -6,11 +6,21 @@ See the cMeta COPYRIGHT and LICENSE files in the project root for details. """ +import re + from typing import Dict, Any, List, Optional from .files import safe_read_file from .common import deep_merge, _error +# A bare key=value is a flag only when the part before the first "=" looks like a key: a letter or "_", then +# letters, digits, "_", "." (nested keys), "," (list keys) or "-". Free text that holds "=" - a command, a URL +# with a query string, an expression - stays one positional argument. +KEY_LIKE = re.compile(r'^[^\W\d][\w.,-]*$') + +# A negative number is a value (a positional argument), not a flag +NEGATIVE_NUMBER = re.compile(r'^-\d+(\.\d+)?$') + def parse_cmd( cmd, # Command string to parse, list of arguments, or None. fail_on_error: bool = False, # If True, raise exception on error. @@ -20,8 +30,9 @@ def parse_cmd( Supports various argument formats: - - Positional arguments: stored in 'positional_arguments' list - - Flags: --key=value, -key=value, key=value + - Positional arguments: stored in the 'args' list; a negative number (-5, -1.5) is one too + - Flags: --key=value, -key=value, key=value (a bare key=value only when the part before "=" looks like a + key - KEY_LIKE: free text with "=", such as "echo a=b" or a URL with a query, stays positional) - Boolean flags: --key (True), --key- (False), --no-key (False) - List flags: --key, (creates list) - Nested keys: --parent.child=value @@ -90,6 +101,12 @@ def parse_cmd( elif argument.startswith('---'): return _error(f'Flag "{argument}" has unknown prefix "---"', 1, None, fail_on_error) + # A negative number is a positional argument, not a flag + elif NEGATIVE_NUMBER.match(argument): + if params.get('args') is None: + params['args'] = [] + params['args'].append(argument) + # Handle double-dash flags (--key=value) elif argument.startswith('-'): flag_content = argument[2:] if argument.startswith('--') else argument[1:] @@ -109,8 +126,8 @@ def parse_cmd( # Handle non-flag arguments else: - # Check if it's a key=value pair without dashes - if '=' in argument: + # A key=value pair without dashes - when the part before "=" looks like a key + if '=' in argument and KEY_LIKE.match(argument.split('=', 1)[0]): r = split_flag(argument, params, fix_keys=True) if r['return']>0: return r else: @@ -310,9 +327,9 @@ def check_params( tmp_converted_value = input_value.lower().strip() if tmp_converted_value in ['1', 'on', 'true', 'yes']: - convert_value = True + converted_value = True elif tmp_converted_value in ['0', 'off', 'false', 'no']: - convert_value = False + converted_value = False else: return _error(f"Invalid bool value '{input_value}' for '{main_key}'", 1, None, fail_on_error) diff --git a/cmeta/version.py b/cmeta/version.py index d4b9a10..288da7f 100644 --- a/cmeta/version.py +++ b/cmeta/version.py @@ -6,5 +6,5 @@ See the cMeta COPYRIGHT and LICENSE files in the project root for details. """ -__version__ = "0.35.2" +__version__ = "0.36.0" __release_date__ = "2026-10-13" diff --git a/cmeta/versions.json b/cmeta/versions.json index 89cb7dd..1ef4e07 100644 --- a/cmeta/versions.json +++ b/cmeta/versions.json @@ -113,6 +113,10 @@ { "version": "0.35.2", "version_date": "20261013" + }, + { + "version": "0.36.0", + "version_date": "20261013" } ] } diff --git a/docs/common-commands.md b/docs/common-commands.md index 5c5c231..e2b6e57 100644 --- a/docs/common-commands.md +++ b/docs/common-commands.md @@ -245,7 +245,13 @@ not a failure. Details and shell examples in ## Flag syntax - Booleans: `--flag` (True), `--flag-` or `--no-flag` (False). -- Strings: `--key=value` or `--key value` (last wins). +- Strings: `--key=value` (last wins). `--key value` is the flag `--key` and a + positional `value`. +- Bare `key=value` works when the part before `=` looks like a key; other text + with `=` (`"echo a=b"`, a URL with a query) and negative numbers (`-5`) are + positional. +- `--` passes everything after it to the command untouched (`unparsed`), for + the commands that read it. - Lists: `--key,=v1,v2,v3` (note the comma on the *key*). - Nested: `--parent.child=value` → `{'parent': {'child': value}}`. - Param files: `@input.yaml` / `@input.json` deep-merge into the params diff --git a/docs/en/api/cmeta_config.rst b/docs/en/api/cmeta_config.rst index d13fad7..5382853 100644 --- a/docs/en/api/cmeta_config.rst +++ b/docs/en/api/cmeta_config.rst @@ -16,6 +16,8 @@ Functions .. autofunction:: cmeta.config.home_from_env +.. autofunction:: cmeta.config.home_path + .. autofunction:: cmeta.config.is_on .. autofunction:: cmeta.config.set_logging diff --git a/docs/en/api/cmeta_utils_sys.rst b/docs/en/api/cmeta_utils_sys.rst index f6ea2fe..4568a41 100644 --- a/docs/en/api/cmeta_utils_sys.rst +++ b/docs/en/api/cmeta_utils_sys.rst @@ -49,3 +49,8 @@ Functions .. autofunction:: cmeta.utils.sys.run_command_with_timeout_tree_kill_on_windows .. autofunction:: cmeta.utils.sys.system_code_pages + +Constants +--------- + +.. autodata:: cmeta.utils.sys.RUN_STOP_GRACE_SECONDS diff --git a/docs/en/api/index.rst b/docs/en/api/index.rst index db17947..c8bcd81 100644 --- a/docs/en/api/index.rst +++ b/docs/en/api/index.rst @@ -1,7 +1,7 @@ -API Reference (v0.35.2) +API Reference (v0.36.0) ======================= -This section contains the complete API reference for cMeta v0.35.2. +This section contains the complete API reference for cMeta v0.36.0. .. toctree:: :maxdepth: 2 diff --git a/docs/en/index.rst b/docs/en/index.rst index 3e21c1d..0cf5735 100644 --- a/docs/en/index.rst +++ b/docs/en/index.rst @@ -1,11 +1,11 @@ cMeta Documentation =================== -Welcome to cMeta v0.35.2 documentation! +Welcome to cMeta v0.36.0 documentation! .. toctree:: :maxdepth: 2 - :caption: cMeta v0.35.2 Contents: + :caption: cMeta v0.36.0 Contents: home diff --git a/docs/history.md b/docs/history.md index 722c151..95ae3ff 100644 --- a/docs/history.md +++ b/docs/history.md @@ -82,7 +82,7 @@ produces APA and BibTeX automatically. The BibTeX entry: author = {Fursin, Grigori}, title = {{cMeta (Common Meta Framework)}}, year = {2026}, - version = {0.35.2}, + version = {0.36.0}, license = {Apache-2.0}, url = {https://github.com/cTuningLabs/cmeta}, note = {cTuning Labs} diff --git a/docs/installation.md b/docs/installation.md index fc08b7d..cef5ddb 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -168,8 +168,8 @@ repositories) into these commands. **A specific version**, to pin or to go back: ```bash -uv tool install --force "cmeta[server]==0.35.2" -pip install "cmeta==0.35.2" +uv tool install --force "cmeta[server]==0.36.0" +pip install "cmeta==0.36.0" ``` **Moving an install from git to PyPI** (or back) is a reinstall from the other diff --git a/docs/using-cmeta.md b/docs/using-cmeta.md index 97902dc..d59add5 100644 --- a/docs/using-cmeta.md +++ b/docs/using-cmeta.md @@ -121,10 +121,12 @@ Parsed by `cmeta/utils/cli.py::parse_cmd`. Full rules: - `--no-flag` → **False** (alternative form). - `--flag=false` / `--flag=true` → literal string values `"false"` / `"true"`. (Don't rely on this for boolean semantics — use the trailing-dash / `--no-` - forms above.) + forms above.) The engine's own control flags (`--quiet`, `--verbose`, + `--json`, ...) read `yes`/`no`, `true`/`false`, `on`/`off`, `1`/`0` as booleans. **Strings and ints** -- `--key=value` or `--key value`. +- `--key=value`, with the `=`: `--key value` is the flag `--key` (True) + followed by the positional `value` — there is no separate-value form. - Repeated `--key=v1 --key=v2` → **last value wins**. For real lists, use: - `--key,=v1,v2,v3` → list `[v1, v2, v3]`. Trailing comma on the *key* signals CSV split. `--key,=` alone → `[]`. @@ -134,15 +136,25 @@ Parsed by `cmeta/utils/cli.py::parse_cmd`. Full rules: - Multiple `--k.a=1 --k.b=2 --k.c.d=3` merge into `{'k': {'a':'1','b':'2','c':{'d':'3'}}}`. **Positional tokens** — anything not consumed as a flag becomes `arg1`, `arg2`, -... in order. First two are conventionally `` and ``. +... in order. First two are conventionally `` and ``. A +negative number (`-5`, `-1.5`) is a positional token too, not a flag +(`--delta=-5` is a value, as always). **Hyphen / underscore equivalence** — `--log-level` == `--log_level`. -**Bare `key=value`** (no leading dashes) is treated the same as `--key=value`. +**Bare `key=value`** (no leading dashes) is treated the same as `--key=value` +when the part before the first `=` looks like a key: a letter or `_`, then +letters, digits, `_`, `.`, `,` or `-` (`name=v`, `a.b=v`, `tags,=a,b`). The +value may hold anything (`path=C:\x\y`, `url=https://x/f?id=42`). Any other +token with an `=` — a command such as `"echo a=b"`, a URL with a query string, +an expression — is one positional token, kept as it is. -**`--` separator** — everything after a lone `--` is captured under `unparsed` -(rarely needed; use for pass-through to sub-processes like -`cx app run -- --sub-flag`). +**`--` separator** — everything after a lone `--` reaches the command +untouched, as the list `unparsed`, for the commands that pass words on to a +program of their own (in cmeta-aops: `cx task run cmd -- echo a=b`, +`cx tool run -- --its-own-flag`, `cxt run-ai -- `). A +command that does not read it stops with `" " takes no +arguments after "--"`. **`@file` inclusion (YAML or JSON)** — replace a bunch of `--k=v` flags with a single reusable file: @@ -193,7 +205,7 @@ All shipped in `cmeta/internal-repo/` and available immediately after install. | **category** | The category system itself — `add`, `find`, `ls`, `rm`, `mv`. | | **repo** | Content repositories — `get`, `clone`, `pull`, `checkout`, `status`, `zip`/`unzip`, `plug`/`unplug`, `space`, `list`. | | **config** | Named configuration artifacts — `get`, `set`, `unset`, `read`, `show`. | -| **utils** | Helper commands — UID/UUID, JSON⇄YAML, clipboard helpers. | +| **utils** | Helper commands — UID/UUID, JSON⇄YAML, clipboard helpers, native messaging hosts. | | **app** | Runnable applications. Ships the `cserver` local web app. | | **script** | Portable shell / Python scripts. | | **asset** | Data / model / file assets. | @@ -209,6 +221,40 @@ Every category inherits the standard base commands: `find`, `list`, `read`, `index`, `test`. Categories add their own commands on top (e.g. `repo get`, `app run`). +### Native messaging hosts for cMeta add-ons + +An add-on artifact can declare `native-host.json` with its Mozilla host name, +allowed extension IDs, a Python `entry` inside the artifact, and optional +non-interactive `checks`. Its entry exports `dispatch(request, cm)` and returns +a JSON-serializable dictionary. For example: + +```json +{ + "name": "example_cmeta_host", + "description": "Example cMeta add-on", + "entry": "bridge/host.py", + "allowed_extensions": ["example@example.org"], + "checks": [{"command": "ping"}] +} +``` + +Use the artifact's UID-only `::` reference so a +rename does not break registration: + +```bash +cx --print-home +cx utils native-host-install :: --apply +cx utils native-host-check :: +``` + +The per-user launcher calls `cx utils native-host-run` using that reference. +Each request uses the cMeta home active in the desktop session and resolves +the artifact from its repository anew. No Python executable or repository path +is saved in the launcher. Mozilla still requires an absolute manifest path; +cMeta keeps that manifest in per-user application data. Keep `cx` or `cmeta` on the +desktop session's PATH and ensure its active home contains the artifact. +`native-host-check` sends the declared requests without opening an AI session. + --- ## 5. Working with artifacts diff --git a/tests/core_tests/test_cli_parse_table.py b/tests/core_tests/test_cli_parse_table.py new file mode 100644 index 0000000..e7006b2 --- /dev/null +++ b/tests/core_tests/test_cli_parse_table.py @@ -0,0 +1,288 @@ +""" +The command line of cMeta pinned in tables: what `parse_cmd` makes of each kind of token, how the engine's own control +flags read a word given as their value, and what a command says when it gets text after "--" that it does not read. + +The rules (point 4 of the hardening work, after cMeta 0.35.2): + +- a bare `key=value` is a flag only when the part before the first "=" looks like a key (a letter or "_", then + letters, digits, "_", ".", "," or "-"); any other token with "=" - a command, a URL with a query string, an + expression - is one positional argument, kept as it is; +- a negative number (-5, -1.5) is a positional argument, not a flag; +- a control flag (--quiet, --verbose, --json, ...) given a word takes yes/no, true/false, on/off, 1/0 - before, + such a word crashed the conversion or took the value of the flag converted before it; +- text after "--" goes to the command as `unparsed`; a keyword-style command without such a parameter (and without + **kwargs) says '" " takes no arguments after "--"' instead of Python's TypeError. + +Everything else is as it always was, and the table says so: every form that worked keeps its result. + +Licensed under the Apache License, Version 2.0. +See the cMeta COPYRIGHT and LICENSE files in the project root for details. +""" + +import os +import random +import re +import subprocess +import sys + +import pytest + +from cmeta import CMeta +from cmeta import config +from cmeta.utils import cli +from cmeta.utils.cli import parse_cmd, check_params + + +# (tokens, the params parse_cmd makes of them) +TABLE = [ + # Free text with "=" stays one positional argument + (['host', 'run', 'h', 'echo a=b'], {'args': ['host', 'run', 'h', 'echo a=b']}), + (['download', 'https://x/file?id=42'], {'args': ['download', 'https://x/file?id=42']}), + (['task', 'run', 'cmd', 'python -c "print(1==1)"'], {'args': ['task', 'run', 'cmd', 'python -c "print(1==1)"']}), + (['x', 'a b=c'], {'args': ['x', 'a b=c']}), + (['x', '=v'], {'args': ['x', '=v']}), + (['x', '1x=2'], {'args': ['x', '1x=2']}), + (['x', 'x@y=1'], {'args': ['x', 'x@y=1']}), + (['x', 'C:\\a=b'], {'args': ['x', 'C:\\a=b']}), + (['x', '/p/a=b'], {'args': ['x', '/p/a=b']}), + + # Negative numbers are positional; a value may still be negative + (['task', 'run', 'x', '-5', '-1.5'], {'args': ['task', 'run', 'x', '-5', '-1.5']}), + (['x', '--delta=-5'], {'args': ['x'], 'delta': '-5'}), + (['x', '-5x'], {'args': ['x'], '5x': True}), + + # Bare key=value as before, the value may hold anything + (['x', 'name=v'], {'args': ['x'], 'name': 'v'}), + (['x', 'name=John Smith'], {'args': ['x'], 'name': 'John Smith'}), + (['x', 'path=C:\\x\\y'], {'args': ['x'], 'path': 'C:\\x\\y'}), + (['x', 'url=https://a/b?c=d'], {'args': ['x'], 'url': 'https://a/b?c=d'}), + (['x', 'a.b=v'], {'args': ['x'], 'a': {'b': 'v'}}), + (['x', 'tags,=a,b'], {'args': ['x'], 'tags': ['a', 'b']}), + (['x', 'log-level=debug'], {'args': ['x'], 'log_level': 'debug'}), + (['x', '_private=1'], {'args': ['x'], '_private': '1'}), + (['x', 'na\u00efve=1'], {'args': ['x'], 'na\u00efve': '1'}), + + # Dash flags as before + (['x', '--quiet=yes'], {'args': ['x'], 'quiet': 'yes'}), + (['x', '--x'], {'args': ['x'], 'x': True}), + (['x', '--y-'], {'args': ['x'], 'y': False}), + (['x', '--no-z'], {'args': ['x'], 'z': False}), + (['x', '-q'], {'args': ['x'], 'q': True}), + (['x', '--cmd=echo a=b'], {'args': ['x'], 'cmd': 'echo a=b'}), + (['x', '--tags,=a,b'], {'args': ['x'], 'tags': ['a', 'b']}), + (['x', '--tags,='], {'args': ['x'], 'tags': []}), + (['x', '--a.b-c=1'], {'args': ['x'], 'a': {'b-c': '1'}}), + (['x', '--log-level=debug'], {'args': ['x'], 'log_level': 'debug'}), + (['x', '--k=v1', '--k=v2'], {'args': ['x'], 'k': 'v2'}), + + # "--key value" is not a form: a flag and a positional argument + (['x', '--name', 'foo'], {'args': ['x', 'foo'], 'name': True}), + + # "--" ends the parsing + (['x', '--', 'echo', 'a=b', '-5', '--y'], {'args': ['x'], 'unparsed': ['echo', 'a=b', '-5', '--y']}), + (['x', '--'], {'args': ['x'], 'unparsed': []}), + + # Plain positional arguments + (['repo', 'list'], {'args': ['repo', 'list']}), + ([], {}), +] + + +@pytest.mark.parametrize('tokens, params', TABLE, ids=[' '.join(t) or '(none)' for t, _ in TABLE]) +def test_what_the_parser_makes_of_each_token(tokens, params): + r = parse_cmd(tokens) + assert r['return'] == 0, r.get('error') + assert r['params'] == params + + +def test_a_triple_dash_is_still_an_error(): + r = parse_cmd(['x', '---y']) + assert r['return'] > 0 and '"---y"' in r['error'] + + +# The parser as it was, against the parser as it is: exactly two kinds of token changed their meaning + +def parse_as_before(tokens): + """The rules before this change: every token with "=" was a flag, and every token that starts with "-".""" + with pytest.MonkeyPatch.context() as m: + m.setattr(cli, 'KEY_LIKE', re.compile(r'^.*$', re.S)) + m.setattr(cli, 'NEGATIVE_NUMBER', re.compile(r'(?!x)x')) + return parse_cmd(tokens) + + +def changed_meaning(token): + """True for a token the change is meant for: a negative number, or a dash-less token with "=" whose part + before the "=" does not look like a key.""" + if cli.NEGATIVE_NUMBER.match(token): + return True + return '=' in token and not token.startswith('-') and not cli.KEY_LIKE.match(token.split('=', 1)[0]) + + +PIECES = ['a', 'B', 'z', '_', '1', '0', '5', '.', ',', '-', '--', '=', ' ', '/', ':', '?', '\\', '"', "'", '+', '#', + 'no-', 'naïve', 'C:', 'http://x', '(', ')', '==', '!', '*'] + + +def random_tokens(rng): + tokens = [] + for _ in range(rng.randint(1, 4)): + token = ''.join(rng.choice(PIECES) for _ in range(rng.randint(1, 6))) + if token.startswith('@'): # a file inclusion reads (or deletes) a file: not a parsing matter + token = 'x' + token + tokens.append(token) + return tokens + + +def test_only_free_text_with_equals_and_negative_numbers_changed_their_meaning(): + rng = random.Random(20261013) + changed, same = 0, 0 + + for _ in range(20000): + tokens = random_tokens(rng) + before, now = parse_as_before(tokens), parse_cmd(tokens) + + # "--" ends the parsing: a changed token after it never mattered + parsed = tokens[:tokens.index('--')] if '--' in tokens else tokens + meant = [t for t in parsed if changed_meaning(t)] + + if not meant: + assert before == now, (tokens, before, now) + same += 1 + elif before['return'] == 0 and now['return'] == 0: + # The changed tokens are positional arguments now, and nothing else changed + assert before != now, (tokens, now) + for t in meant: + assert t in now['params'].get('args', []), (tokens, now) + changed += 1 + # else: a parsing error in one of them (a "---" prefix, a key set as a value and then as a dict) - the + # error is the same text in both or comes from the changed token itself; nothing to compare + + assert changed > 500 and same > 5000, (changed, same) # both kinds of case were really exercised + + +# The control flags of the engine, the words they take + +BOOL_CONTROL_FLAGS = [item for desc in (config.params_desc, config.params_command_desc, config.params_command2_desc, + config.params_init_desc, config.params_command3_desc) + for item in desc if item and item.get('type') == bool] + +WORDS = {'yes': True, 'Yes': True, 'true': True, 'on': True, '1': True, ' YES ': True, + 'no': False, 'false': False, 'OFF': False, '0': False} + + +@pytest.mark.parametrize('item', BOOL_CONTROL_FLAGS, ids=lambda i: i['key']) +def test_every_bool_control_flag_takes_a_word(item): + desc = [item] + + for word, value in WORDS.items(): + r = check_params({item['key']: word}, desc) + assert r['return'] == 0, (item['key'], word, r.get('error')) + assert r['checked_params'][item['key']] is value, (item['key'], word) + + r = check_params({item['key']: 'maybe'}, desc) + assert r['return'] > 0 and "Invalid bool value 'maybe'" in r['error'] + + +def test_a_word_flag_after_another_flag_keeps_its_own_value(): + # Before: quiet took the value converted for verbose just before it + r = check_params({'verbose': False, 'quiet': 'yes'}, config.params_desc) + assert r['return'] == 0, r.get('error') + assert r['checked_params']['verbose'] is False and r['checked_params']['quiet'] is True + + +# Text after "--": the commands that read it and the ones that do not + +CATEGORY_UID = '0123456789abcdef' + +API_V1 = '''from cmeta.category import InitCategory + +class Category(InitCategory): + def __init__(self, *args, **kwargs): + super().__init__(*args, module_file_path = __file__, **kwargs) + + def plain_(self, ctx, arg1 = None): + return {'return': 0, 'arg1': arg1} + + def words_(self, ctx, arg1 = None, unparsed = None): + return {'return': 0, 'arg1': arg1, 'got': unparsed} + + def anything_(self, ctx, **kwargs): + return {'return': 0, 'got': kwargs.get('unparsed')} + + def asdict__(self, params): + return {'return': 0, 'got': params.get('unparsed')} +''' + + +@pytest.fixture() +def home(tmp_path, monkeypatch): + """A home of the test with a repository whose category has commands of each style.""" + for var in ('CMETA_HOME', 'CMETA_HOME2', 'VIRTUAL_ENV', 'CONDA_PREFIX', 'CMETA_DEBUG', 'CMETA_VERBOSE'): + monkeypatch.delenv(var, raising=False) + + home = tmp_path / 'home' + monkeypatch.setenv('CMETA_HOME', str(home)) + + repo = tmp_path / 'repo-p4' + category = repo / 'category' / 'p4cat' + (category / 'api').mkdir(parents=True) + (repo / '_cmr.yaml').write_text('artifact: repo-p4,1111111111111111\ncategory: repo,f4f792ab40c7498f\n', encoding='utf-8') + (category / '_cmeta.yaml').write_text(f'artifact: p4cat,{CATEGORY_UID}\ncategory: category,dd9ea50e7f76467f\n' + 'last_api_version: 1\nbase_category_default_api_versions:\n "1": 1\n', + encoding='utf-8') + (category / 'api' / 'v1.py').write_text(API_V1, encoding='utf-8') + + cm = CMeta(home=str(home)) + r = cm.access({'category': 'repo', 'command': 'plug', 'arg1': str(repo), 'con': False}) + assert r['return'] == 0, r.get('error') + + return {'cm': cm, 'home': home} + + +def test_text_after_dashes_reaches_the_commands_that_read_it(home): + cm = home['cm'] + + for command in ('words', 'anything', 'asdict'): + r = cm.access({'category': 'p4cat', 'command': command, 'unparsed': ['echo', 'a=b'], 'con': False}) + assert r['return'] == 0, (command, r.get('error')) + assert r['got'] == ['echo', 'a=b'], command + + +def test_a_command_that_does_not_read_it_says_so(home): + r = home['cm'].access({'category': 'p4cat', 'command': 'plain', 'unparsed': ['x'], 'con': False}) + assert r['return'] > 0 + assert 'takes no arguments after "--"' in r['error'] and 'p4cat' in r['error'] and 'plain' in r['error'], r['error'] + + # Without "--" the same command runs + r = home['cm'].access({'category': 'p4cat', 'command': 'plain', 'arg1': 'x', 'con': False}) + assert r['return'] == 0 and r['arg1'] == 'x' + + +def cx(args, env_home): + env = {k: v for k, v in os.environ.items() if not k.startswith('CMETA_')} + env.update({'CMETA_HOME': str(env_home), 'PYTHONIOENCODING': 'utf-8'}) + p = subprocess.run([sys.executable, '-m', 'cmeta'] + list(args), env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=600) + return p.returncode, p.stdout.decode('utf-8', 'replace'), p.stderr.decode('utf-8', 'replace') + + +def test_the_command_line_end_to_end(home): + # A control flag given a word + rc, out, err = cx(['repo', 'list', '--quiet=yes', '--json=no'], home['home']) + assert rc == 0, out + err + + rc, out, err = cx(['repo', 'list', '--quiet=maybe'], home['home']) + assert rc != 0 and "Invalid bool value 'maybe'" in err, out + err + + # Free text with "=" and a negative number reach the command as positional arguments + rc, out, err = cx(['p4cat', 'plain', 'echo a=b'], home['home']) + assert rc == 0, out + err + + rc, out, err = cx(['p4cat', 'plain', '-5'], home['home']) + assert rc == 0, out + err + + # Text after "--": read by one command, refused in words by another + rc, out, err = cx(['p4cat', 'words', '--', 'echo', 'a=b'], home['home']) + assert rc == 0, out + err + + rc, out, err = cx(['p4cat', 'plain', '--', 'x'], home['home']) + assert rc != 0 and 'takes no arguments after "--"' in err, out + err diff --git a/tests/internal_repo_tests/test_native_host.py b/tests/internal_repo_tests/test_native_host.py new file mode 100644 index 0000000..5a9c441 --- /dev/null +++ b/tests/internal_repo_tests/test_native_host.py @@ -0,0 +1,92 @@ +""" +Tests for portable cMeta native messaging hosts. + +cMeta author and developer: (C) 2025-2026 Grigori Fursin + +See the cMeta COPYRIGHT and LICENSE files in the project root for details. +""" + +import importlib.util +import json +from pathlib import Path +import sys + +import cmeta +import pytest + + +MODULE = Path(cmeta.__file__).parent / 'internal-repo/category/utils/api/native_host.py' +SPEC = importlib.util.spec_from_file_location('test_cmeta_native_host', MODULE) +native_host = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(native_host) +REF = '1111111111111111::2222222222222222' + + +class FakeCM: + def __init__(self, artifact): + self.artifact = artifact + self.home_path = artifact.parent + + def access(self, params): + assert params['category'] == '1111111111111111' + assert params['arg1'] == '2222222222222222' + return {'return': 0, 'artifacts': [{'path': str(self.artifact)}]} + + +def artifact_at(path, entry='bridge/host.py'): + path.mkdir(parents=True) + (path / 'bridge').mkdir() + (path / 'bridge/host.py').write_text('def dispatch(request, cm): return request\n', encoding='utf-8') + (path / 'native-host.json').write_text(json.dumps({ + 'name': 'example_host', 'description': 'Example host', 'entry': entry, + 'allowed_extensions': ['example@example.org'], + 'checks': [{'command': 'ping'}], + }), encoding='utf-8') + return path + + +def test_resolves_same_uid_from_current_home(tmp_path): + first = artifact_at(tmp_path / 'first' / 'plugin') + second = artifact_at(tmp_path / 'second' / 'renamed-plugin') + cm = FakeCM(first) + assert native_host.resolve(cm, REF)['root'] == first + cm.artifact = second + cm.home_path = second.parent + assert native_host.resolve(cm, REF)['root'] == second + + +def test_entry_cannot_escape_artifact(tmp_path): + root = artifact_at(tmp_path / 'plugin', entry='../outside.py') + (tmp_path / 'outside.py').write_text('def dispatch(request, cm): return request\n', encoding='utf-8') + with pytest.raises(ValueError, match='inside its artifact'): + native_host.resolve(FakeCM(root), REF) + + +def test_launcher_stores_only_cx_and_uid_reference(tmp_path, monkeypatch): + root = artifact_at(tmp_path / 'home' / 'plugin') + launcher = tmp_path / 'registration' / 'example_host.bat' + manifest = tmp_path / 'registration' / 'example_host.json' + monkeypatch.setattr(native_host, 'locations', lambda name: (launcher, manifest)) + monkeypatch.setattr(native_host, '_registered_manifest', lambda name: None) + monkeypatch.setattr(native_host, '_write_registration', lambda name, path: None) + native_host.install(FakeCM(root), REF, apply=True) + contents = launcher.read_bytes().decode('mbcs' if sys.platform == 'win32' else 'utf-8') + assert 'cx utils native-host-run ' + REF in contents + assert 'cmeta utils native-host-run ' + REF in contents + assert str(root) not in contents + assert 'python' not in contents.lower() + assert json.loads(manifest.read_text(encoding='utf-8'))['path'] == str(launcher) + + +def test_check_request_uses_selected_home(tmp_path): + script = tmp_path / 'reply.py' + script.write_text('''import json,os,struct,sys +raw=sys.stdin.buffer.read() +request=json.loads(raw[4:]) +reply=json.dumps({'return':0,'home':os.environ['CMETA_HOME'],'request':request}).encode() +sys.stdout.buffer.write(struct.pack('=I',len(reply))+reply) +''', encoding='utf-8') + home = tmp_path / 'selected-home' + response = native_host._send([sys.executable, str(script)], {'command': 'ping'}, home) + assert response['home'] == str(home) + assert response['request'] == {'command': 'ping'}