diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 4cfb491..2e33155 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -1,26 +1,51 @@ name: test-hdxcli -run-name: ${{ github.actor }} acceptance tests -on: [push] +run-name: ${{ github.actor }} tests +on: + push: + workflow_dispatch: + inputs: + acceptance: + description: "Also run the live-cluster acceptance suite (tests/command_line_interface)" + type: boolean + default: false jobs: - run-tests: + # Hermetic unit tests. These cover the maintained surface (the migrate command and its + # helpers) and need no cluster, so they run on every push. + unit-tests: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4.3.0 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.10' - name: "Install poetry" run: python3 -m pip install poetry==2.0.1 - name: "Install hdxcli dependencies" run: python3 -m poetry install - - name: "Set environment for tests" - run: | - echo "PYTHONPATH=$GITHUB_WORKSPACE/src" >> $GITHUB_ENV - - name: "Run tests" - run: poetry run python3 -m pytest -v + - name: "Run unit tests" + run: poetry run python3 -m pytest -v tests --ignore=tests/command_line_interface env: + PYTHONPATH: ${{ github.workspace }}/src + + # Live-cluster acceptance suite for the legacy CRUD commands. It requires the + # HDXCLI_TESTS_CLUSTER_* secrets and a reachable cluster, so it only runs on demand. + acceptance-tests: + if: github.event_name == 'workflow_dispatch' && inputs.acceptance + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.10' + - name: "Install poetry" + run: python3 -m pip install poetry==2.0.1 + - name: "Install hdxcli dependencies" + run: python3 -m poetry install + - name: "Run acceptance tests" + run: poetry run python3 -m pytest -v tests/command_line_interface + env: + PYTHONPATH: ${{ github.workspace }}/src HDXCLI_TESTS_CLUSTER_SSL_ACTIVE: ${{secrets.HDXCLI_TESTS_CLUSTER_SSL_ACTIVE}} HDXCLI_TESTS_CLUSTER_PASSWORD: ${{secrets.HDXCLI_TESTS_CLUSTER_PASSWORD}} HDXCLI_TESTS_CLUSTER_USERNAME: ${{secrets.HDXCLI_TESTS_CLUSTER_USERNAME}} - PYTHONPATH: ${{env.PYTHONPATH}} HDXCLI_TESTS_CLUSTER_HOSTNAME: ${{secrets.HDXCLI_TESTS_CLUSTER_HOSTNAME}} diff --git a/pyproject.toml b/pyproject.toml index e582a60..9ba6356 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -version = "1.0.83" +version = "1.0.84" name = "hdxcli" requires-python = ">=3.10,<4.0" maintainers = [ @@ -10,7 +10,7 @@ maintainers = [ [tool.poetry] name = "hdxcli" -version = "1.0.83" +version = "1.0.84" description = "Hydrolix command line utility to do CRUD operations on projects, tables, transforms and other resources in Hydrolix clusters" authors = ["German Diago Gomez ", "Agustin Actis "] license = "Apache-2.0" diff --git a/src/hdx_cli/cli_interface/migrate/commands.py b/src/hdx_cli/cli_interface/migrate/commands.py index 7bc8a41..f4d5c73 100644 --- a/src/hdx_cli/cli_interface/migrate/commands.py +++ b/src/hdx_cli/cli_interface/migrate/commands.py @@ -7,12 +7,12 @@ from hdx_cli.cli_interface.migrate.data import migrate_data from hdx_cli.cli_interface.migrate.helpers import MigrationData, get_catalog from hdx_cli.cli_interface.migrate.rc.rc_manager import RcloneAPIConfig -from hdx_cli.cli_interface.migrate.resources import get_resources, create_resources +from hdx_cli.cli_interface.migrate.resources import create_resources, get_resources from hdx_cli.cli_interface.migrate.validator import validations from hdx_cli.config.profile_settings import is_valid_hostname from hdx_cli.library_api.common.exceptions import InvalidHostnameException from hdx_cli.library_api.common.logging import get_logger -from hdx_cli.library_api.utility.decorators import report_error_and_exit, ensure_logged_in +from hdx_cli.library_api.utility.decorators import ensure_logged_in, report_error_and_exit logger = get_logger() @@ -79,6 +79,14 @@ def validate_hostname(ctx, params, hostname: str) -> str: type=click.Choice(["http", "https"], case_sensitive=False), help="URI scheme for the target cluster (http or https).", ) +@click.option( + "--target-customer", + "-tc", + "target_customer", + default=None, + help="Name or UUID of the customer on the target cluster to assign to the migrated " + "project. Required by clusters v6.3+; if omitted, you are prompted to choose one.", +) @click.option( "--allow-merge", type=bool, @@ -169,6 +177,7 @@ def migrate( target_username: str, target_password: str, target_uri_scheme: str, + target_customer: str, allow_merge: bool, only: str, with_functions: bool, @@ -205,6 +214,11 @@ def migrate( - Target Cluster: Specify the destination with `--target-profile` or with individual connection details (`--target-hostname`, `--target-username`, etc.). \b + - Target Customer (`--target-customer`): Clusters v6.3+ require every project to + belong to a customer. Provide the target customer by name or UUID; when omitted, + the command lists the target's customers and prompts for one, offering to create + it if it does not exist. + \b - Migration Scope (`--only`): - *resources*: Migrates only the project, table, and other definitions. - *data*: Migrates only the data, assuming resources already exist. @@ -239,7 +253,9 @@ def migrate( """ source_profile = ctx.parent.obj["usercontext"] has_target_profile = target_profile_name is not None - has_all_cluster_options = all([target_hostname, target_username, target_password, target_uri_scheme]) + has_all_cluster_options = all( + [target_hostname, target_username, target_password, target_uri_scheme] + ) if not has_target_profile and not has_all_cluster_options: raise click.BadParameter( @@ -320,7 +336,8 @@ def migrate( source_data, reuse_partitions, migrate_functions=with_functions, - migrate_dictionaries=with_dictionaries + migrate_dictionaries=with_dictionaries, + target_customer=target_customer, ) if only != "resources": migrate_data( diff --git a/src/hdx_cli/cli_interface/migrate/customer.py b/src/hdx_cli/cli_interface/migrate/customer.py new file mode 100644 index 0000000..b518216 --- /dev/null +++ b/src/hdx_cli/cli_interface/migrate/customer.py @@ -0,0 +1,385 @@ +"""Customer resolution and membership handling for migrations. + +Since Hydrolix v6.3 (HDX-11681), project creation requires an existing +Customer on the target cluster, and tables may only reference storages and +credentials that are members of the project's customer (HDX-11643). This +module resolves which customer a migrated project should belong to and +registers the storages/credentials the migrated table references. +""" + +import json +from urllib.parse import urlparse + +from hdx_cli.library_api.common.exceptions import ( + ActionNotAvailableException, + HdxCliException, + HttpException, +) +from hdx_cli.library_api.common.generic_resource import access_resource_detailed +from hdx_cli.library_api.common.logging import get_logger +from hdx_cli.library_api.common.storage import get_storage_default +from hdx_cli.models import ProfileUserContext + +from ..common.undecorated_click_commands import basic_create, basic_get, basic_options +from .helpers import confirm_action + +logger = get_logger() + +CUSTOMERS_PATH = "/config/v1/customers/" +# Customers seeded by init_ci; never a sensible default for a migrated project. +CI_CUSTOMER_NAMES = ("hydro", "sample_project") +PICKER_ATTEMPTS = 3 + + +def customer_field_status(profile: ProfileUserContext, projects_path: str) -> tuple[bool, bool]: + """Whether the target's project endpoint supports and requires a customer. + + Read from the OPTIONS metadata of the projects endpoint, so version + differences are handled without hardcoded version checks: + + - ``supported`` is False on pre-6.1 clusters (no 'customer' field at all). + - ``required`` is True only where the server rejects a project created + without a customer (6.4+). Clusters where the field is present but + optional (6.1-6.3.x auto-assign the default customer) report False, so + migrations to them keep working untouched. + + Both are False when the metadata cannot be fetched. + """ + try: + structure = basic_options(profile, projects_path) + except (HttpException, ActionNotAvailableException) as exc: + logger.debug(f"Could not fetch project options from target: {exc}") + return False, False + field = structure.get("customer") + if field is None: + return False, False + return True, bool(field.get("required")) + + +def get_target_customer( + profile: ProfileUserContext, + projects_path: str, + customer_name_or_uuid: str | None = None, + existing_project: dict | None = None, +) -> dict | None: + """Resolve the Customer for the migrated project on the target cluster. + + Returns the customer body (with 'uuid' and 'name'), or None when no + customer applies (the target does not support them, or the field is + optional and none was requested so the server assigns the default). + """ + if existing_project is not None: + return _customer_of_existing_project(profile, existing_project, customer_name_or_uuid) + + supported, required = customer_field_status(profile, projects_path) + if not supported: + # Pre-6.1 target: no customer concept at all. + if customer_name_or_uuid: + logger.debug( + "The target cluster does not support customers on projects, " + "ignoring --target-customer." + ) + return None + + # The field exists. Honor an explicit choice on any such cluster (it is + # settable whether or not it is required). + if customer_name_or_uuid: + return _resolve_or_create_from_option( + profile, list_customers(profile), customer_name_or_uuid + ) + + # No explicit choice: only prompt when the server would otherwise reject + # the project. Optional-field clusters (6.1-6.3.x) auto-assign the default. + if not required: + return None + return _interactive_pick_customer(profile, list_customers(profile)) + + +def list_customers(profile: ProfileUserContext) -> list[dict]: + try: + return basic_get(profile, CUSTOMERS_PATH, pagination=False) or [] + except HttpException as exc: + raise HdxCliException(f"Unable to list customers on the target cluster: {exc}") from exc + + +def find_customer(customers: list[dict], name_or_uuid: str) -> dict | None: + # The server slugifies customer names on create (spaces become dashes), so + # match the value as typed and in its slugified form. This lets a name with + # a space still resolve to the customer the server actually stored. + candidates = {name_or_uuid, name_or_uuid.replace(" ", "-")} + for customer in customers: + if candidates & {customer.get("uuid"), customer.get("name")}: + return customer + return None + + +def default_customer_candidate(customers: list[dict]) -> dict | None: + """The obvious pick: the single customer that is not seeded by init_ci.""" + non_ci = [c for c in customers if c.get("name") not in CI_CUSTOMER_NAMES] + return non_ci[0] if len(non_ci) == 1 else None + + +def _customer_of_existing_project( + profile: ProfileUserContext, existing_project: dict, customer_name_or_uuid: str | None +) -> dict | None: + customer_id = existing_project.get("customer") + if not customer_id: + # Pre-6.3 target, or a legacy project without a customer. + return None + + customer = find_customer(list_customers(profile), customer_id) + if not customer: + customer = {"uuid": customer_id, "name": customer_id} + if customer_name_or_uuid and customer_name_or_uuid not in ( + customer.get("uuid"), + customer.get("name"), + ): + logger.info( + f"Warning: the target project already belongs to customer " + f"'{customer.get('name')}' and this cannot be changed; " + "ignoring --target-customer." + ) + return customer + + +def _resolve_or_create_from_option( + profile: ProfileUserContext, customers: list[dict], customer_name_or_uuid: str +) -> dict: + customer = find_customer(customers, customer_name_or_uuid) + if customer: + return customer + + available = ", ".join(sorted(c.get("name", "") for c in customers)) or "none" + logger.info("") + logger.info(f"Customer '{customer_name_or_uuid}' was not found on the target cluster.") + logger.info(f"Available customers: {available}.") + if not confirm_action(f"Create customer '{customer_name_or_uuid}' on the target cluster?"): + raise HdxCliException( + f"Customer '{customer_name_or_uuid}' does not exist on the target cluster. " + "Rerun with --target-customer set to one of the available customers, " + "or ask an administrator to create it." + ) + return _create_customer(profile, customer_name_or_uuid) + + +def _interactive_pick_customer(profile: ProfileUserContext, customers: list[dict]) -> dict: + logger.info("In progress") + logger.info("") + header = " Customer Settings " + logger.info(f"{header:*^40}") + logger.info("* The target cluster requires projects to belong to a customer.") + if customers: + logger.info("* Available customers on the target cluster:") + for customer in customers: + logger.info(f"* {customer.get('name')} ({customer.get('uuid')})") + else: + logger.info("* There are no customers on the target cluster yet.") + logger.info("*") + + default_customer = default_customer_candidate(customers) + for _ in range(PICKER_ATTEMPTS): + if default_customer: + logger.info(f"* Customer name or UUID ({default_customer.get('name')}): [!i]") + else: + logger.info("* Customer name or UUID (created if it does not exist): [!i]") + user_input = input().strip() + + if not user_input: + if not default_customer: + logger.info("* A customer is required. Please try again.") + continue + selection = default_customer + else: + selection = find_customer(customers, user_input) + + if selection: + if confirm_action( + f"* Assign customer '{selection.get('name')}' to the migrated project?" + ): + logger.info(f'{"*" * 40:<42} -> [!n]') + return selection + continue + + if confirm_action(f"* Customer '{user_input}' does not exist. Create it?"): + created = _create_customer(profile, user_input) + logger.info(f'{"*" * 40:<42} -> [!n]') + return created + + raise HdxCliException( + "Attempt limit reached. No customer was selected for the migrated project. " + "Rerun with --target-customer to provide one directly." + ) + + +def _create_customer(profile: ProfileUserContext, customer_name: str) -> dict: + try: + response = basic_create(profile, CUSTOMERS_PATH, customer_name) + except HttpException as exc: + if exc.error_code in (401, 403): + raise HdxCliException( + f"Insufficient permissions to create customer '{customer_name}' on the " + "target cluster. Ask an administrator to create it, then rerun the " + f"migration with --target-customer {customer_name}." + ) from exc + raise + + # Use the created resource straight from the POST response: the server may + # have slugified the name (e.g. spaces to dashes), so reading it back by the + # typed name would miss it and leave the target already mutated. + try: + created = response.json() + except (ValueError, AttributeError): + created = {} + if not created.get("uuid"): + raise HdxCliException( + f"Customer '{customer_name}' was created but the response did not " "include its id." + ) + return {"uuid": created["uuid"], "name": created.get("name", customer_name)} + + +def collect_storage_map_ids(table_body: dict) -> set[str]: + storage_map = table_body.get("settings", {}).get("storage_map") or {} + storage_ids = set() + if storage_map.get("default_storage_id"): + storage_ids.add(storage_map["default_storage_id"]) + storage_ids.update((storage_map.get("column_value_mapping") or {}).keys()) + storage_ids.update(storage_map.get("spread_list") or []) + return storage_ids + + +def collect_autoingest_credential_ids(table_body: dict) -> set[str]: + credential_ids = set() + for entry in table_body.get("settings", {}).get("autoingest") or []: + if isinstance(entry, dict): + for key in ("source_credential_id", "bucket_credential_id"): + if entry.get(key): + credential_ids.add(entry[key]) + return credential_ids + + +def _is_member(resource: dict, customer: dict) -> bool: + members = resource.get("customers") or [] + return customer.get("uuid") in {m.get("uuid") for m in members if isinstance(m, dict)} + + +def ensure_storage_memberships( + profile: ProfileUserContext, customer: dict, table_body: dict, target_storages: list[dict] +) -> str: + """Register the table's storages with the target customer when needed. + + Tables may only reference storages that are members of the project's + customer; the cluster default storage is exempt from that validation. + Returns a short status message for the caller's progress line. + """ + storage_ids = collect_storage_map_ids(table_body) + default_storage_id, _ = get_storage_default(target_storages) + storage_ids.discard(default_storage_id) + if not storage_ids: + return "Not needed" + + for storage_id in sorted(storage_ids): + storage = next((s for s in target_storages if s.get("uuid") == storage_id), None) + if storage is None: + # Unknown storage id: let the table creation surface the real error. + continue + if "customers" not in storage: + # This storage does not expose memberships; skip it and let the + # table creation surface any error rather than abandoning the rest. + logger.debug(f"Storage '{storage_id}' has no 'customers' field; skipping.") + continue + if _is_member(storage, customer): + continue + _confirm_and_add_membership( + profile, + customer, + "add_storage", + {"storages": [{"uuid": storage_id}]}, + f"storage '{storage.get('name')}' ({storage_id})", + "This makes the storage (and its credential) usable by that customer's users.", + ) + return "Done" + + +def ensure_credential_memberships( + profile: ProfileUserContext, customer: dict, credential_ids: set[str] +) -> None: + """Register the given credentials (e.g. from autoingest) with the customer.""" + if not credential_ids: + return + + credentials, _ = access_resource_detailed(profile, [("credentials", None)]) + for credential_id in sorted(credential_ids): + credential = next((c for c in credentials if c.get("uuid") == credential_id), None) + if credential is None: + # Unknown credential id: let the table creation surface the real error. + continue + if "customers" not in credential: + # This credential does not expose memberships; skip it and keep + # going rather than abandoning the remaining credentials. + logger.debug(f"Credential '{credential_id}' has no 'customers' field; skipping.") + continue + if _is_member(credential, customer): + continue + _confirm_and_add_membership( + profile, + customer, + "add_credential", + {"credentials": [{"uuid": credential_id}]}, + f"credential '{credential.get('name')}' ({credential_id})", + "This makes the credential usable by that customer's users.", + ) + + +def _confirm_and_add_membership( + profile: ProfileUserContext, + customer: dict, + action: str, + body: dict, + resource_description: str, + consequence: str, +) -> None: + customer_name = customer.get("name") + action_path = f"{CUSTOMERS_PATH}{customer.get('uuid')}/{action}/" + manual_remediation = ( + f"ask an administrator to run: POST {action_path} " + f"with body {json.dumps(body)}, then rerun the migration" + ) + + logger.info("") + logger.info(f"The {resource_description} is not registered with customer '{customer_name}'.") + logger.info(consequence) + if not confirm_action(f"Register it with customer '{customer_name}'?"): + raise HdxCliException( + f"The {resource_description} must be registered with customer " + f"'{customer_name}' before the table can be created. To do it manually, " + f"{manual_remediation}." + ) + + try: + basic_create(profile, action_path, body=body) + except HttpException as exc: + if exc.error_code in (401, 403): + raise HdxCliException( + f"Insufficient permissions to register the {resource_description} " + f"with customer '{customer_name}'. To do it manually, {manual_remediation}." + ) from exc + if exc.error_code in (404, 405): + # The customer add_storage/add_credential actions only exist on + # clusters that enforce membership (6.4+). Older clusters expose a + # 'customers' field but manage membership server-side (they attach + # the storage/credential to the customer at table-create time and do + # not reject non-members), so there is nothing to do here: skip. + logger.debug( + f"Membership registration is not available on this target " + f"({action}); it is managed automatically. Skipping." + ) + return + raise + + +def target_projects_context(profile: ProfileUserContext) -> tuple[dict | None, str]: + """The existing target project (if any) and the target projects path.""" + projects, projects_url = access_resource_detailed(profile, [("projects", None)]) + projects_path = urlparse(projects_url).path + existing_project = next((p for p in projects if p.get("name") == profile.projectname), None) + return existing_project, projects_path diff --git a/src/hdx_cli/cli_interface/migrate/resources.py b/src/hdx_cli/cli_interface/migrate/resources.py index 69ffbe9..a1040b0 100644 --- a/src/hdx_cli/cli_interface/migrate/resources.py +++ b/src/hdx_cli/cli_interface/migrate/resources.py @@ -6,6 +6,13 @@ basic_create_file, basic_get, ) +from hdx_cli.cli_interface.migrate.customer import ( + collect_autoingest_credential_ids, + ensure_credential_memberships, + ensure_storage_memberships, + get_target_customer, + target_projects_context, +) from hdx_cli.cli_interface.migrate.helpers import MigrationData, confirm_action from hdx_cli.cli_interface.migrate.resource_adapter import ( adapt_resource_to_api_structure, @@ -42,12 +49,33 @@ def create_resources( reuse_partitions: bool = False, migrate_functions: bool = False, migrate_dictionaries: bool = False, + target_customer: str = None, ) -> None: logger.info(f'{" Resource Creation ":=^50}') logger.info(f"Target Cluster: {target_profile.hostname}") + # CUSTOMER + # Since v6.3, projects belong to a customer and tables may only reference + # storages/credentials that are members of it. Resolve the customer and + # register memberships before creating anything. + existing_project, target_projects_path = target_projects_context(target_profile) + logger.info(f"{' Target customer':<42} -> [!n]") + customer = get_target_customer( + target_profile, target_projects_path, target_customer, existing_project + ) + logger.info(customer.get("name") if customer else "Not required") + + if customer: + logger.info(f"{' Storage registration':<42} -> [!n]") + message = ensure_storage_memberships( + target_profile, customer, source_data.table, target_data.storages + ) + logger.info(message) + # PROJECT - _create_project(target_profile, source_data.project, reuse_partitions) + _create_project( + target_profile, source_data.project, target_projects_path, customer, reuse_partitions + ) target_data.project, _ = access_resource_detailed( target_profile, [("projects", target_profile.projectname)] ) @@ -63,7 +91,7 @@ def create_resources( _create_dictionaries(target_profile, source_profile, source_data.dictionaries) # TABLE - _create_table(target_profile, source_data.table, reuse_partitions) + _create_table(target_profile, source_data.table, reuse_partitions, customer) target_data.table, _ = access_resource_detailed( target_profile, [("projects", target_profile.projectname), ("tables", target_profile.tablename)], @@ -78,13 +106,26 @@ def create_resources( def _create_project( - target_profile: ProfileUserContext, source_project_body: dict, reuse_partitions: bool + target_profile: ProfileUserContext, + source_project_body: dict, + target_projects_path: str, + customer: dict, + reuse_partitions: bool, ) -> None: logger.info(f"{f' Project: {target_profile.projectname[:31]}':<42} -> [!n]") - _, target_projects_url = access_resource_detailed(target_profile, [("projects", None)]) - target_projects_path = urlparse(target_projects_url).path target_project_body = copy.deepcopy(source_project_body) + # The source's customer/org/deployment_id never make sense on the target + # cluster. The right customer (when the target requires one) was already + # resolved. hdx_deployment_id must be dropped explicitly: some cluster + # versions expose it as writable in the projects metadata, so the adapter + # would otherwise copy the source's value and the create fails as a + # duplicate; the target derives its own. + target_project_body.pop("customer", None) + target_project_body.pop("org", None) + target_project_body.pop("hdx_deployment_id", None) + if customer: + target_project_body["customer"] = customer.get("uuid") adapted_project = adapt_resource_to_api_structure( target_profile, target_projects_path, target_project_body @@ -100,9 +141,16 @@ def _create_project( ) logger.info("Done") except HttpException as exc: - if exc.error_code != 400 or "already exists" not in str(exc.message): - raise exc - logger.info("Exists, skipping") + message = str(exc.message) + if exc.error_code == 400 and "already exists" in message: + logger.info("Exists, skipping") + return + if exc.error_code == 400 and "customer" in message: + raise HdxCliException( + f"The target cluster rejected the project creation: {message} " + "Use --target-customer to choose a customer on the target cluster." + ) from exc + raise exc def _create_functions( @@ -215,7 +263,10 @@ def _create_dictionary_file( def _create_table( - target_profile: ProfileUserContext, source_table_body: dict, reuse_partitions: bool + target_profile: ProfileUserContext, + source_table_body: dict, + reuse_partitions: bool, + customer: dict = None, ) -> None: logger.info(f"{f' Table: {target_profile.tablename[:33]}':<42} -> [!n]") @@ -230,6 +281,12 @@ def _create_table( ) normalized_table = normalize_table(adapted_table, reuse_partitions) + # Autoingest credentials are only known after the interactive + # normalization above, so their membership check happens here. + if customer: + credential_ids = collect_autoingest_credential_ids(normalized_table) + ensure_credential_memberships(target_profile, customer, credential_ids) + basic_create( target_profile, target_tables_path, target_profile.tablename, body=normalized_table ) diff --git a/tests/test_migrate_customer.py b/tests/test_migrate_customer.py new file mode 100644 index 0000000..dc76f17 --- /dev/null +++ b/tests/test_migrate_customer.py @@ -0,0 +1,253 @@ +import pytest + +from hdx_cli.cli_interface.migrate import customer as customer_module +from hdx_cli.cli_interface.migrate.customer import ( + collect_autoingest_credential_ids, + collect_storage_map_ids, + customer_field_status, + default_customer_candidate, + find_customer, + get_target_customer, +) +from hdx_cli.library_api.common.exceptions import ActionNotAvailableException + +CUSTOMERS = [ + {"uuid": "aaaa-1111", "name": "hydro"}, + {"uuid": "bbbb-2222", "name": "sample_project"}, + {"uuid": "cccc-3333", "name": "acme"}, +] + + +class TestFindCustomer: + def test_finds_by_name(self): + assert find_customer(CUSTOMERS, "acme")["uuid"] == "cccc-3333" + + def test_finds_by_uuid(self): + assert find_customer(CUSTOMERS, "aaaa-1111")["name"] == "hydro" + + def test_returns_none_when_missing(self): + assert find_customer(CUSTOMERS, "unknown") is None + + def test_empty_list(self): + assert find_customer([], "acme") is None + + def test_matches_slugified_name(self): + # The server stores "Acme Corp" as "Acme-Corp"; typing the spaced form + # must still resolve to it. + customers = [{"uuid": "dddd-4444", "name": "Acme-Corp"}] + assert find_customer(customers, "Acme Corp")["uuid"] == "dddd-4444" + + +class TestDefaultCustomerCandidate: + def test_single_non_ci_customer_is_default(self): + assert default_customer_candidate(CUSTOMERS)["name"] == "acme" + + def test_no_default_with_multiple_non_ci_customers(self): + customers = CUSTOMERS + [{"uuid": "dddd-4444", "name": "globex"}] + assert default_customer_candidate(customers) is None + + def test_no_default_with_only_ci_customers(self): + assert default_customer_candidate(CUSTOMERS[:2]) is None + + +class TestCollectStorageMapIds: + def test_collects_all_storage_map_sources(self): + table = { + "settings": { + "storage_map": { + "default_storage_id": "st-1", + "column_value_mapping": {"st-2": ["1", "2"]}, + "spread_list": ["st-3"], + } + } + } + assert collect_storage_map_ids(table) == {"st-1", "st-2", "st-3"} + + def test_empty_without_storage_map(self): + assert collect_storage_map_ids({"settings": {}}) == set() + assert collect_storage_map_ids({}) == set() + + def test_ignores_missing_keys(self): + table = {"settings": {"storage_map": {"default_storage_id": "st-1"}}} + assert collect_storage_map_ids(table) == {"st-1"} + + +class TestCollectAutoingestCredentialIds: + def test_collects_both_credential_kinds(self): + table = { + "settings": { + "autoingest": [ + {"source_credential_id": "cr-1", "bucket_credential_id": "cr-2"}, + {"source_credential_id": "cr-3"}, + ] + } + } + assert collect_autoingest_credential_ids(table) == {"cr-1", "cr-2", "cr-3"} + + def test_empty_without_autoingest(self): + assert collect_autoingest_credential_ids({"settings": {}}) == set() + + def test_ignores_non_dict_entries(self): + table = {"settings": {"autoingest": ["bogus", {"source_credential_id": "cr-1"}]}} + assert collect_autoingest_credential_ids(table) == {"cr-1"} + + +class TestCustomerFieldStatus: + def test_required_when_options_mark_it_required(self, monkeypatch): + # 6.4+: field present and required. + monkeypatch.setattr( + customer_module, + "basic_options", + lambda profile, path: {"name": {}, "customer": {"required": True}}, + ) + assert customer_field_status(None, "/projects/") == (True, True) + + def test_supported_but_optional(self, monkeypatch): + # 6.1-6.3.x: field present but optional (server auto-assigns default). + monkeypatch.setattr( + customer_module, + "basic_options", + lambda profile, path: {"name": {}, "customer": {"required": False}}, + ) + assert customer_field_status(None, "/projects/") == (True, False) + + def test_unsupported_when_options_lack_customer(self, monkeypatch): + # Pre-6.1: no customer field at all. + monkeypatch.setattr(customer_module, "basic_options", lambda profile, path: {"name": {}}) + assert customer_field_status(None, "/projects/") == (False, False) + + def test_unsupported_when_options_unavailable(self, monkeypatch): + def raise_unavailable(profile, path): + raise ActionNotAvailableException("no options") + + monkeypatch.setattr(customer_module, "basic_options", raise_unavailable) + assert customer_field_status(None, "/projects/") == (False, False) + + +class TestGetTargetCustomer: + def test_none_when_target_does_not_support_customer(self, monkeypatch): + monkeypatch.setattr( + customer_module, "customer_field_status", lambda profile, path: (False, False) + ) + assert get_target_customer(None, "/projects/", "acme") is None + + def test_none_when_optional_and_no_flag(self, monkeypatch): + # Regression guard: an optional-field cluster (6.1-6.3.x) with no + # --target-customer must NOT drop into the interactive picker; the + # server auto-assigns the default. + monkeypatch.setattr( + customer_module, "customer_field_status", lambda profile, path: (True, False) + ) + # list_customers/input must never be reached; leave them unpatched so a + # network/EOF error would fail the test loudly. + assert get_target_customer(None, "/projects/", None) is None + + def test_flag_honored_when_optional(self, monkeypatch): + # An explicit --target-customer is honored even when the field is + # optional (settable on 6.1-6.3.x). + monkeypatch.setattr( + customer_module, "customer_field_status", lambda profile, path: (True, False) + ) + monkeypatch.setattr(customer_module, "list_customers", lambda profile: CUSTOMERS) + customer = get_target_customer(None, "/projects/", "acme") + assert customer["uuid"] == "cccc-3333" + + def test_option_resolves_by_name_when_required(self, monkeypatch): + monkeypatch.setattr( + customer_module, "customer_field_status", lambda profile, path: (True, True) + ) + monkeypatch.setattr(customer_module, "list_customers", lambda profile: CUSTOMERS) + customer = get_target_customer(None, "/projects/", "acme") + assert customer["uuid"] == "cccc-3333" + + def test_existing_project_customer_wins(self, monkeypatch): + monkeypatch.setattr(customer_module, "list_customers", lambda profile: CUSTOMERS) + existing_project = {"name": "proj", "customer": "cccc-3333"} + customer = get_target_customer( + None, "/projects/", "hydro", existing_project=existing_project + ) + assert customer["name"] == "acme" + + def test_existing_project_without_customer_yields_none(self, monkeypatch): + existing_project = {"name": "proj"} + assert ( + get_target_customer(None, "/projects/", None, existing_project=existing_project) is None + ) + + def test_existing_project_with_unknown_customer_keeps_uuid(self, monkeypatch): + monkeypatch.setattr(customer_module, "list_customers", lambda profile: []) + existing_project = {"name": "proj", "customer": "zzzz-9999"} + customer = get_target_customer(None, "/projects/", None, existing_project=existing_project) + assert customer == {"uuid": "zzzz-9999", "name": "zzzz-9999"} + + +class _FakeResponse: + def __init__(self, payload): + self._payload = payload + + def json(self): + return self._payload + + +class TestMembershipRegistration: + """ensure_storage_memberships against a non-member, non-default storage.""" + + def _setup(self, monkeypatch, create_side_effect): + monkeypatch.setattr(customer_module, "get_storage_default", lambda s: ("st-default", {})) + monkeypatch.setattr(customer_module, "confirm_action", lambda *a, **k: True) + monkeypatch.setattr(customer_module, "basic_create", create_side_effect) + + def _run(self): + table_body = { + "settings": { + "storage_map": { + "default_storage_id": "st-default", + "column_value_mapping": {"st-9": ["1"]}, + } + } + } + target_storages = [{"uuid": "st-9", "name": "S9", "customers": []}] + return customer_module.ensure_storage_memberships( + None, {"uuid": "cust-1", "name": "acme"}, table_body, target_storages + ) + + def test_404_is_skipped_not_fatal(self, monkeypatch): + from hdx_cli.library_api.common.exceptions import HttpException + + def raise_404(*a, **k): + raise HttpException(404, "Not Found") + + self._setup(monkeypatch, raise_404) + # Older clusters (6.1-6.3) lack the add_storage action; the missing + # endpoint must not abort the migration. + assert self._run() == "Done" + + def test_403_still_raises(self, monkeypatch): + from hdx_cli.library_api.common.exceptions import HdxCliException, HttpException + + def raise_403(*a, **k): + raise HttpException(403, "Forbidden") + + self._setup(monkeypatch, raise_403) + with pytest.raises(HdxCliException): + self._run() + + +class TestCreateCustomer: + def test_uses_response_body_with_slugified_name(self, monkeypatch): + # Server slugifies "Acme Corp" -> "Acme-Corp" and returns the real + # object; _create_customer must trust that, not re-read by typed name. + monkeypatch.setattr( + customer_module, + "basic_create", + lambda *a, **k: _FakeResponse({"uuid": "new-1", "name": "Acme-Corp"}), + ) + result = customer_module._create_customer(None, "Acme Corp") + assert result == {"uuid": "new-1", "name": "Acme-Corp"} + + def test_raises_when_response_has_no_uuid(self, monkeypatch): + from hdx_cli.library_api.common.exceptions import HdxCliException + + monkeypatch.setattr(customer_module, "basic_create", lambda *a, **k: _FakeResponse({})) + with pytest.raises(HdxCliException): + customer_module._create_customer(None, "acme") diff --git a/tests/test_migrate_resources.py b/tests/test_migrate_resources.py new file mode 100644 index 0000000..766a4e7 --- /dev/null +++ b/tests/test_migrate_resources.py @@ -0,0 +1,50 @@ +from types import SimpleNamespace + +from hdx_cli.cli_interface.migrate import resources as resources_module + + +def _run_create_project(monkeypatch, source_body, customer): + """Call _create_project with the adapter/normalizer/HTTP layer stubbed out, + returning the body that would have been POSTed to the target.""" + captured = {} + + # Pass the body through the adapter and normalizer unchanged so we can + # assert on exactly what _create_project prepared. + monkeypatch.setattr( + resources_module, "adapt_resource_to_api_structure", lambda profile, path, body: body + ) + monkeypatch.setattr(resources_module, "normalize_project", lambda body, reuse_partitions: body) + + def fake_basic_create(profile, path, name, *, body): + captured["body"] = body + + monkeypatch.setattr(resources_module, "basic_create", fake_basic_create) + + profile = SimpleNamespace(projectname="target_proj") + resources_module._create_project( + profile, source_body, "/config/v1/orgs/o/projects/", customer, False + ) + return captured["body"] + + +class TestCreateProjectBody: + def test_strips_cross_cluster_identity_fields(self, monkeypatch): + source_body = { + "name": "source_proj", + "customer": "source-customer-uuid", + "org": "source-org-uuid", + "hdx_deployment_id": "ns__source_proj", + "description": "keep me", + } + body = _run_create_project(monkeypatch, source_body, customer=None) + assert "customer" not in body + assert "org" not in body + assert "hdx_deployment_id" not in body + assert body["description"] == "keep me" + + def test_injects_resolved_customer(self, monkeypatch): + source_body = {"name": "source_proj", "customer": "source-customer-uuid"} + body = _run_create_project( + monkeypatch, source_body, customer={"uuid": "target-customer-uuid", "name": "acme"} + ) + assert body["customer"] == "target-customer-uuid"