diff --git a/docs/cyberark-pam-import.md b/docs/cyberark-pam-import.md index a95b16e00..86f34960a 100644 --- a/docs/cyberark-pam-import.md +++ b/docs/cyberark-pam-import.md @@ -62,8 +62,13 @@ Keeper Vault/ | PaloAltoNetworks, CiscoIOS, CiscoASA, JuniperJunos, F5BigIP, CheckPointGAIA | pamMachine | ssh | 22 | | CyberArk (internal) | pamMachine | ssh | 22 | | BusinessWebsite | login | — | — | -| (empty platformId) | pamMachine | ssh | 22 | -| (unknown platformId) | pamMachine | ssh | 22 | +| Custom/renamed platform resolved by CyberArk metadata or keyword matching | inferred PAM type | inferred | inferred | +| Empty/unknown platform with no metadata or keyword match | login | — | — | + +For an unresolved platform, fields not already represented by the login's title, +login, password, URL, or CyberArk notes are preserved as text custom fields. +Nested account metadata is flattened using dotted field names. Raw secret-bearing +payload keys are excluded from custom fields. ### Field Mapping @@ -170,6 +175,9 @@ pam project cyberark-import pvwa.company.com --dry-run --output import.json --in # Filter specific safes pam project cyberark-import pvwa.company.com --safes "Production,Staging" --exclude-safes "Archive*" +# Import into Nested Share Folders (folders, records, rotation, PAM config) +pam project cyberark-import pvwa.company.com --name "CyberArk Migration" --gateway "My Gateway" --nsf + # Extend existing project pam project cyberark-import pvwa.company.com --config @@ -199,7 +207,8 @@ pam project cyberark-cleanup --name "CyberArk Migration" --dry-run | `--name`, `-n` | Project name | | `--config`, `-c` | Extend existing PAM config UID | | `--gateway`, `-g` | Gateway name or UID | -| `--folder-mode` | flat, exact, ksm (default) | +| `--folder-mode` | flat, exact, ksm, safe (default: safe) | +| `--nsf` | Create project folders/records/PAM config in Nested Share Folders | | `--safes` | Include only these safes (comma/glob) | | `--exclude-safes` | Exclude safes (comma/glob) | | `--list-safes` | List safes and exit | @@ -273,5 +282,3 @@ Before building the import JSON, the importer warns about: - Rate limit handling: automatic retry on HTTP 429 with exponential backoff - Pagination cap: MAX_FETCH_RECORDS (50,000) prevents OOM attacks --- - - diff --git a/keepercommander/__main__.py b/keepercommander/__main__.py index b627a7947..58772abf9 100644 --- a/keepercommander/__main__.py +++ b/keepercommander/__main__.py @@ -201,6 +201,14 @@ def main(from_package=False): if from_package: sys.excepthook = handle_exceptions + # Internal: background service mode for a frozen (PyInstaller) executable - see service_app.py. + is_frozen = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') + if is_frozen: + from .service.core.service_app import SERVICE_MODE_FLAG, run_background_service + if len(sys.argv) > 1 and sys.argv[1] == SERVICE_MODE_FLAG: + run_background_service() + return + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) opts, flags = parser.parse_known_args(sys.argv[1:]) diff --git a/keepercommander/api.py b/keepercommander/api.py index 1e45403f0..4d611e5fb 100644 --- a/keepercommander/api.py +++ b/keepercommander/api.py @@ -43,6 +43,7 @@ from .sync_down import sync_down from .team import Team from .ttk import TTK +from .sanitization import sanitize_protobuf_json current_milli_time = lambda: int(round(time.time() * 1000)) @@ -881,7 +882,8 @@ def communicate_rest(params, request, endpoint, *, rs_type=None, payload_version if request: if logging.getLogger().level <= logging.DEBUG: js = google.protobuf.json_format.MessageToJson(request) - logging.debug('>>> [RQ] %s: %s', endpoint, js) + sanitized_js = sanitize_protobuf_json(js) + logging.debug('>>> [RQ] %s: %s', endpoint, sanitized_js) api_request_payload.payload = request.SerializeToString() if isinstance(payload_version, int): api_request_payload.apiVersion = payload_version @@ -894,7 +896,8 @@ def communicate_rest(params, request, endpoint, *, rs_type=None, payload_version proto_rs.ParseFromString(rs) if logging.getLogger().level <= logging.DEBUG: js = google.protobuf.json_format.MessageToJson(proto_rs) - logging.debug('>>> [RS] %s: %s', endpoint, js) + sanitized_js = sanitize_protobuf_json(js) + logging.debug('>>> [RS] %s: %s', endpoint, sanitized_js) return proto_rs else: return rs diff --git a/keepercommander/commands/discoveryrotation.py b/keepercommander/commands/discoveryrotation.py index 9be7a7711..6f714dc03 100644 --- a/keepercommander/commands/discoveryrotation.py +++ b/keepercommander/commands/discoveryrotation.py @@ -2429,7 +2429,7 @@ def print_root_rotation_setting(params, is_verbose=False, format_type='table'): for c in configurations: # type: vault.TypedRecord if c.record_type in ('pamAwsConfiguration', 'pamAzureConfiguration', 'pamGcpConfiguration', 'pamDomainConfiguration', 'pamNetworkConfiguration', 'pamOciConfiguration', - 'pamGitHubConfiguration'): + 'pamGitHubConfiguration', 'pamHashiCorpConfiguration'): facade.record = c folder_info = resolve_pam_config_folder_info( params, facade, c.record_uid) @@ -2489,7 +2489,7 @@ def print_root_rotation_setting(params, is_verbose=False, format_type='table'): common_parser = argparse.ArgumentParser(add_help=False) common_parser.add_argument('--environment', '-env', dest='config_type', action='store', - choices=['local', 'aws', 'azure', 'gcp', 'domain', 'oci', 'github'], help='PAM Configuration Type') + choices=['local', 'aws', 'azure', 'gcp', 'domain', 'oci', 'github', 'hashicorp'], help='PAM Configuration Type') common_parser.add_argument('--title', '-t', dest='title', action='store', help='Title of the PAM Configuration') common_parser.add_argument('--gateway', '-g', dest='gateway_uid', action='store', help='Gateway UID or Name') common_parser.add_argument('--shared-folder', '-sf', dest='shared_folder_uid', action='store', @@ -2555,12 +2555,23 @@ def print_root_rotation_setting(params, is_verbose=False, format_type='table'): github_group.add_argument('--github-base-url', dest='github_base_url', action='store', help='GitHub Base URL') +hashicorp_group = common_parser.add_argument_group('hashicorp', 'HashiCorp Vault configuration') +hashicorp_group.add_argument('--hashicorp-id', dest='hashicorp_id', action='store', help='HashiCorp Id') +hashicorp_group.add_argument('--vault-base-url', dest='vault_base_url', action='store', + help='Vault Base URL (e.g., https://vault.company.com:8200)') +hashicorp_group.add_argument('--vault-token', dest='vault_token', action='store', + help='Vault Token (optional; syncIdentity takes precedence)') +hashicorp_group.add_argument('--vault-namespace', dest='vault_namespace', action='store', + help='Vault Namespace (optional; leave blank for Community Edition)') +hashicorp_group.add_argument('--vault-mount-path', dest='vault_mount_path', action='store', + help='Vault KV Mount Path (optional; defaults to "secret")') + class PamConfigurationEditMixin(RecordEditMixin): pam_record_types = None PAM_CONFIG_RECORD_TYPES = frozenset({ 'pamAwsConfiguration', 'pamAzureConfiguration', 'pamGcpConfiguration', 'pamDomainConfiguration', 'pamNetworkConfiguration', 'pamOciConfiguration', - 'pamGitHubConfiguration', + 'pamGitHubConfiguration', 'pamHashiCorpConfiguration', }) PAM_RESOURCE_RECORD_TYPES = frozenset({ 'pamDatabase', 'pamDirectory', 'pamMachine', 'pamRemoteBrowser', @@ -2859,6 +2870,22 @@ def parse_properties(self, params, record, **kwargs): # type: (KeeperParams, va oci_region = kwargs.get('oci_region') if oci_region: extra_properties.append(f'text.regionOci={oci_region}') + elif record.record_type == 'pamHashiCorpConfiguration': + hashicorp_id = kwargs.get('hashicorp_id') + if hashicorp_id: + extra_properties.append(f'text.pamHashiCorpId={hashicorp_id}') + vault_base_url = kwargs.get('vault_base_url') + if vault_base_url: + extra_properties.append(f'text.pamHashiCorpVaultBaseUrl={vault_base_url}') + vault_token = kwargs.get('vault_token') + if vault_token: + extra_properties.append(f'secret.pamHashiCorpVaultToken={vault_token}') + vault_namespace = kwargs.get('vault_namespace') + if vault_namespace: + extra_properties.append(f'text.pamHashiCorpVaultNamespace={vault_namespace}') + vault_mount_path = kwargs.get('vault_mount_path') + if vault_mount_path: + extra_properties.append(f'text.pamHashiCorpVaultMountPath={vault_mount_path}') if extra_properties: self.assign_typed_fields(record, [RecordEditMixin.parse_field(x) for x in extra_properties]) @@ -2936,9 +2963,11 @@ def execute(self, params, **kwargs): record_type = 'pamDomainConfiguration' elif config_type == 'oci': record_type = 'pamOciConfiguration' + elif config_type == 'hashicorp': + record_type = 'pamHashiCorpConfiguration' else: raise CommandError('pam-config-new', f'--environment {config_type} is not supported' - ' - supported options: local, aws, azure, gcp, domain, oci, github') + ' - supported options: local, aws, azure, gcp, domain, oci, github, hashicorp') title = kwargs.get('title') if not title: @@ -3754,6 +3783,28 @@ def _is_rotation_allowed_by_enforcement(params): return True +def ensure_gateway_management_allowed(params): + # type: (KeeperParams) -> bool + """Per-user enforcement gate on the 'allow_pam_gateway' role enforcement + (confirmed via live account_summary payload). Prints an error and returns + False when the user's enterprise enforcement disallows Gateway management. + Shared by both PAMCreateGatewayCommand and PAMGatewayRemoveCommand (and + their legacy discoveryrotation_v1 counterparts) to avoid duplicating the + check at every call site. + """ + try: + from .workflow.helpers import is_pam_action_allowed_by_enforcement + except ImportError as _e: + logging.debug('workflow.helpers not available; skipping gateway enforcement check: %s', _e) + return True + + if not is_pam_action_allowed_by_enforcement(params, 'allow_pam_gateway'): + print(f"{bcolors.FAIL}Gateway management is not allowed by your enterprise " + f"enforcement (allow_pam_gateway).{bcolors.ENDC}") + return False + return True + + class PAMGatewayActionRotateCommand(Command): parser = argparse.ArgumentParser(prog='pam action rotate') parser.add_argument('--record-uid', '-r', dest='record_uid', action='store', @@ -4393,6 +4444,9 @@ def get_parser(self): return PAMGatewayRemoveCommand.dr_remove_controller_parser def execute(self, params, **kwargs): + if not ensure_gateway_management_allowed(params): + return + gateway_name = kwargs.get('gateway') gateways = gateway_helper.get_all_gateways(params) @@ -4461,6 +4515,8 @@ def get_parser(self): return PAMCreateGatewayCommand.dr_create_controller_parser def execute(self, params, **kwargs): + if not ensure_gateway_management_allowed(params): + return gateway_name = kwargs.get('gateway_name') ksm_app = kwargs.get('ksm_app') diff --git a/keepercommander/commands/discoveryrotation_v1.py b/keepercommander/commands/discoveryrotation_v1.py index 8d2d21502..1468dcf62 100644 --- a/keepercommander/commands/discoveryrotation_v1.py +++ b/keepercommander/commands/discoveryrotation_v1.py @@ -22,7 +22,7 @@ from keeper_secrets_manager_core.utils import url_safe_str_to_bytes from .base import Command, GroupCommand, user_choice, dump_report_data, report_output_parser, json_output_parser, field_to_title, FolderMixin -from .discoveryrotation import PAMLegacyCommand +from .discoveryrotation import PAMLegacyCommand, ensure_gateway_management_allowed from .folder import FolderMoveCommand from .ksm import KSMCommand from .pam import gateway_helper, router_helper @@ -1710,6 +1710,9 @@ def get_parser(self): return PAMGatewayRemoveCommand.dr_remove_controller_parser def execute(self, params, **kwargs): + if not ensure_gateway_management_allowed(params): + return + gateway_name = kwargs.get('gateway') gateways = gateway_helper.get_all_gateways(params) @@ -1745,6 +1748,8 @@ def get_parser(self): return PAMCreateGatewayCommand.dr_create_controller_parser def execute(self, params, **kwargs): + if not ensure_gateway_management_allowed(params): + return gateway_name = kwargs.get('gateway_name') ksm_app = kwargs.get('ksm_app') diff --git a/keepercommander/commands/enterprise.py b/keepercommander/commands/enterprise.py index 57fffe7a6..6e5b1a362 100644 --- a/keepercommander/commands/enterprise.py +++ b/keepercommander/commands/enterprise.py @@ -51,6 +51,13 @@ from ..proto import record_pb2, APIRequest_pb2, enterprise_pb2, automator_pb2, pam_pb2 +# Privileges that require the current user to already hold them before they can be granted to, +# or removed from, another role. These control account transfer, team/company management, and +# financial operations. manage_nodes/sharing_administrator are excluded — restricting those would +# block delegated admins from routine management. +_PRIVILEGED_GRANTS = frozenset(('transfer_account', 'manage_companies', 'manage_teams')) + + def register_commands(commands): commands['enterprise-down'] = GetEnterpriseDataCommand() commands['enterprise-info'] = EnterpriseInfoCommand() @@ -397,7 +404,8 @@ def execute(self, params, **kwargs): nodes[node_id] = { 'node_id': node_id, 'parent_id': node.get('parent_id') or 0, - 'name': (node['data'].get('displayname') or '') if node.get('parent_id') else params.enterprise['enterprise_name'], + 'name': node['data'].get('displayname') or + (params.enterprise['enterprise_name'] if not node.get('parent_id') else ''), 'isolated': node.get('restrict_visibility') or False, 'users': [], 'teams': [], @@ -1554,7 +1562,9 @@ def is_in_chain(node_id, parent_id): 'node_id': node['node_id'], 'encrypted_data': encrypted_data } - rq['parent_id'] = parent_id if parent_id else node.get('parent_id') + effective_parent_id = parent_id if parent_id else node.get('parent_id') + if effective_parent_id: + rq['parent_id'] = effective_parent_id request_batch.append(rq) if request_batch: @@ -2260,6 +2270,72 @@ def is_node_managed_by_role(params, node_id, role_id): # type: (KeeperParams, i managed_nodes = params.enterprise.get('managed_nodes') return any(True for x in managed_nodes if x.get('managed_node_id') == node_id and x.get('role_id') == role_id) + @staticmethod + def get_effective_privileges_for_node(params, role_id, target_node_id): + # type: (KeeperParams, int, int) -> Set[str] + """Compute effective privileges for a role on a target node, honoring cascade. + + Walks up the ancestor chain to find privileges granted on parent nodes + that cascade down to the target node. + """ + effective_privs = set() # type: Set[str] + + # Build node lookup and parent map + nodes = {n['node_id']: n for n in params.enterprise.get('nodes', [])} + if target_node_id not in nodes: + return effective_privs + + # Build managed_nodes by role_id with cascade info + role_managed = {} # type: Dict[int, Dict[str, Any]] + for mn in params.enterprise.get('managed_nodes', []): + if mn['role_id'] == role_id: + role_managed[mn['managed_node_id']] = { + 'cascade': mn.get('cascade_node_management', False) + } + + # Walk up from target_node_id to root, collecting managed ancestor nodes + current_node_id = target_node_id + managed_ancestors = [] # type: List[int] + + while current_node_id: + if current_node_id in role_managed: + managed_ancestors.append(current_node_id) + node = nodes.get(current_node_id) + if node and node.get('parent_id'): + current_node_id = node['parent_id'] + else: + current_node_id = None + + # Collect privileges from managed ancestors that cascade or are exact matches + for rp in params.enterprise.get('role_privileges', []): + if rp['role_id'] != role_id: + continue + privilege = rp['privilege'].lower() + managed_node_id = rp['managed_node_id'] + + if managed_node_id == target_node_id: + # Exact match always applies + effective_privs.add(privilege) + elif managed_node_id in managed_ancestors: + # Ancestor privilege applies if cascade is enabled + if role_managed[managed_node_id]['cascade']: + effective_privs.add(privilege) + + return effective_privs + + @staticmethod + def get_current_enterprise_user_id(params): + # type: (KeeperParams) -> Optional[int] + """Find the current user's enterprise_user_id by username matching.""" + if 'users' not in params.enterprise: + return None + + username_lower = params.user.lower() if params.user else None + for user in params.enterprise['users']: + if user.get('username', '').lower() == username_lower: + return user.get('enterprise_user_id') + return None + def execute(self, params, **kwargs): if kwargs.get('add') and kwargs.get('remove'): raise CommandError('enterprise-role', "'add' and 'delete' parameters are mutually exclusive.") @@ -2655,6 +2731,19 @@ def execute(self, params, **kwargs): else: enforcement_value = None + # Restrict require_account_share enforcement to root admins only. + # Reuses EnterpriseCommand.get_user_root_nodes(), which fails open (treats an + # unresolvable current user as main admin) to match the rest of the class. + if key == 'require_account_share': + true_root_node_id = next((n['node_id'] for n in params.enterprise.get('nodes', []) + if n['node_id'] & 0xffffffff == 2), None) + is_root_admin = true_root_node_id is not None and \ + true_root_node_id in self.get_user_root_nodes(params) + if not is_root_admin: + logging.warning('Failed to modify enforcement \'%s\': Only enterprise root administrators can manage account transfer policies', + key) + continue + role_enforcements = params.enterprise.get('role_enforcements') or [] for role in matched_roles: role_id = role['role_id'] @@ -2776,9 +2865,23 @@ def execute(self, params, **kwargs): if not node: logging.warning('Role "%d" does not manage node "%d"', role_id, node_id) return - privileges = {x['privilege'] for x in params.enterprise.get('role_privileges', []) + privileges = {x['privilege'].lower() for x in params.enterprise.get('role_privileges', []) if x['role_id'] == role_id and x['managed_node_id'] == node_id} all_privileges = {x[1].lower() for x in constants.ROLE_PRIVILEGES} + + # Get current user's effective privileges for authorization check. + # Cascade-aware: walks ancestor chain to find privileges granted on parent nodes. + # Fails closed: an unresolvable current user yields no privileges, so the request is denied. + current_user_id = self.get_current_enterprise_user_id(params) + current_user_effective_privileges = set() # type: Set[str] + if current_user_id: + current_user_roles = {x['role_id'] for x in params.enterprise.get('role_users', []) + if x.get('enterprise_user_id') == current_user_id} + for user_role_id in current_user_roles: + # For each role the user has, get effective privileges on this node (honoring cascade) + priv_set = self.get_effective_privileges_for_node(params, user_role_id, node_id) + current_user_effective_privileges.update(priv_set) + for is_add in [True, False]: parameter = 'add_privilege' if is_add else 'remove_privilege' privilege_list = kwargs.get(parameter) @@ -2787,11 +2890,7 @@ def execute(self, params, **kwargs): privilege = privilege.lower() if privilege not in all_privileges: logging.warning('Add/Remove managed node privilege: invalid privilege: %s', privilege) - return - # if is_add: - # if privilege in ['transfer_account', 'manage_companies']: - # logging.warning('Add managed node privilege: Commander does not support \"%s\" privilege', privilege) - # return + continue if is_add and privilege in privileges: logging.info('Add privilege: Role "%d", Mode "%s" already contains privilege "%s" ', role_id, node_id, privilege) @@ -2801,6 +2900,17 @@ def execute(self, params, **kwargs): role_id, node_id, privilege) continue + # Verify caller holds the privilege before granting or revoking it on another + # role. Gating removal too, since a node admin without the privilege could + # otherwise strip transfer_account/manage_teams/manage_companies from a role + # that depends on it (e.g. an offboarding or security-owning role). + if privilege in _PRIVILEGED_GRANTS: + if privilege not in current_user_effective_privileges: + verb = 'assign' if is_add else 'remove' + logging.warning('Failed to %s \'%s\' privilege: You do not have the required privilege to modify \'%s\'', + verb, privilege, privilege) + continue + rq = { 'command': 'managed_node_privilege_add' if is_add else 'managed_node_privilege_remove', 'role_id': role_id, diff --git a/keepercommander/commands/enterprise_common.py b/keepercommander/commands/enterprise_common.py index eb0bba7e5..f967c93b6 100644 --- a/keepercommander/commands/enterprise_common.py +++ b/keepercommander/commands/enterprise_common.py @@ -27,7 +27,7 @@ def __init__(self): self.public_keys = {} self.team_keys = {} self._node_map = None - self._node_map_enterprise_name = None + self._node_map_signature = None def execute_args(self, params, args, **kwargs): if params.enterprise: @@ -296,11 +296,21 @@ def get_enterprise_ids(params, num_ids=1): return enterprise_ids def get_node_path(self, params, node_id, omit_root=False): - current_enterprise_name = params.enterprise.get('enterprise_name') - if self._node_map is None or self._node_map_enterprise_name != current_enterprise_name: - self._node_map_enterprise_name = current_enterprise_name + node_map_signature = ( + params.enterprise.get('enterprise_name'), + tuple( + (x['node_id'], x.get('parent_id', 0), x['data'].get('displayname'), x.get('name')) + for x in params.enterprise['nodes'] + ), + ) + if self._node_map is None or self._node_map_signature != node_map_signature: + self._node_map_signature = node_map_signature self._node_map = { - x['node_id']: (x['data'].get('displayname') or x.get('name') or str(x['node_id']) if x.get('parent_id', 0) > 0 else params.enterprise['enterprise_name'], x.get('parent_id', 0)) + x['node_id']: ( + x['data'].get('displayname') or x.get('name') or + (params.enterprise['enterprise_name'] if not x.get('parent_id') else str(x['node_id'])), + x.get('parent_id', 0), + ) for x in params.enterprise['nodes']} path = '' node = self._node_map.get(node_id) @@ -328,11 +338,10 @@ def resolve_nodes(params, name): # type: (KeeperParams, str) -> collections.It yield node continue if node_name: - if 'parent_id' in node: - display_name = node['data'].get('displayname') or '' - else: - display_name = params.enterprise['enterprise_name'] or '' - if display_name and display_name.lower() == node_name: + display_names = [node['data'].get('displayname') or ''] + if not node.get('parent_id'): + display_names.append(params.enterprise['enterprise_name'] or '') + if any(x and x.lower() == node_name for x in display_names): yield node else: if 'parent_id' not in node: diff --git a/keepercommander/commands/pam/vault_target.py b/keepercommander/commands/pam/vault_target.py index adfc63342..71c9c1fcf 100644 --- a/keepercommander/commands/pam/vault_target.py +++ b/keepercommander/commands/pam/vault_target.py @@ -325,6 +325,14 @@ def resolve_pam_record(params, identifier, rec_type=None): if not identifier: return None + try: + from ..pam_import.record_loader import load_pam_record + rec = load_pam_record(params, identifier) + if rec and _record_matches_type(rec, rec_type): + return rec + except Exception: + pass + if identifier in getattr(params, 'record_cache', {}): rec = vault.KeeperRecord.load(params, identifier) if rec and _record_matches_type(rec, rec_type): @@ -619,8 +627,13 @@ def update_pam_record(params, record, command='pam', force_nsf=False) -> bool: return False -def execute_record_add_in_folder(params, args, folder_uid, command='pam'): - """Add a record in *folder_uid*, using NSF-native creation when needed.""" +def execute_record_add_in_folder(params, args, folder_uid, command='pam', + sync_after=True): + """Add a record in *folder_uid*, using NSF-native creation when needed. + + When *sync_after* is False, NSF callers can defer sync_down to a batch + boundary (avoids one sync per record during large PAM imports). + """ from ..record_edit import RecordAddCommand from ..nested_share_folder.record_commands import NestedShareRecordAddCommand @@ -630,7 +643,7 @@ def execute_record_add_in_folder(params, args, folder_uid, command='pam'): nsf_args.pop('folder', None) nsf_args['folder_uid'] = folder_uid uid = NestedShareRecordAddCommand().execute(params, **nsf_args) - if uid: + if uid and sync_after: from ..pam_import.nsf_helpers import sync_down_preserving_nsf_keys sync_down_preserving_nsf_keys(params) return uid @@ -639,8 +652,13 @@ def execute_record_add_in_folder(params, args, folder_uid, command='pam'): return RecordAddCommand().execute(params, **record_args) -def execute_record_v3_add_in_folder(params, args, folder_uid, command='pam'): - """Add a v3 typed record in *folder_uid*, using NSF-native creation when needed.""" +def execute_record_v3_add_in_folder(params, args, folder_uid, command='pam', + sync_after=True): + """Add a v3 typed record in *folder_uid*, using NSF-native creation when needed. + + When *sync_after* is False, NSF callers can defer sync_down to a batch + boundary (avoids one sync per record during large PAM imports). + """ import json from ..recordv3 import RecordAddCommand @@ -663,8 +681,9 @@ def execute_record_v3_add_in_folder(params, args, folder_uid, command='pam'): if not result.get('success'): raise CommandError(command, normalize_nsf_user_message(result.get('message')) or 'Failed to create record in Nested Share Folder') - from ..pam_import.nsf_helpers import sync_down_preserving_nsf_keys - sync_down_preserving_nsf_keys(params) + if sync_after: + from ..pam_import.nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) return result['record_uid'] record_args['folder'] = folder_uid diff --git a/keepercommander/commands/pam_import/base.py b/keepercommander/commands/pam_import/base.py index 85cc852b4..d83db0885 100644 --- a/keepercommander/commands/pam_import/base.py +++ b/keepercommander/commands/pam_import/base.py @@ -982,6 +982,49 @@ def to_dag_dict(self, user_id: str) -> Optional[Dict[str, Any]]: } +def load_custom_text_fields(data) -> list: + """Load import JSON custom text fields for record-add field syntax.""" + if not isinstance(data, list): + return [] + fields = [] + for item in data: + if not isinstance(item, dict): + continue + field_type = str(item.get("type") or "text") + label = str(item.get("label") or "").strip() + raw_value = item.get("value") + if isinstance(raw_value, list): + value = raw_value[0] if raw_value else "" + else: + value = raw_value + if value in (None, ""): + continue + fields.append({ + "type": field_type, + "label": label, + "value": str(value), + }) + return fields + + +def append_custom_text_fields(fields: list, custom_fields: list) -> None: + """Append loaded custom fields using record-add's labelled custom syntax.""" + if not isinstance(custom_fields, list): + return + for field in custom_fields: + if not isinstance(field, dict): + continue + field_type = str(field.get("type") or "text") + label = str(field.get("label") or "").strip() + value = str(field.get("value") or "") + if not label or not value: + continue + safe_label = label.replace("=", "==") + if value.startswith("="): + value = f" {value}" + fields.append(f"c.{field_type}.{safe_label}={value}") + + class PamUserObject(): def __init__(self): self.folder_path = None # pam extend only @@ -997,6 +1040,7 @@ def __init__(self): self.connectDatabase = None self.managed = None self.oneTimeCode = None + self.custom = None self.attachments = None # fileRef self.scripts = None # script self.rotation_settings = None # DAG: rotation settings @@ -1024,6 +1068,7 @@ def load(cls, data: Union[str, dict], rotation_params: Optional[PamRotationParam obj.connectDatabase = str(data["connect_database"]) if "connect_database" in data else None obj.managed = utils.value_to_boolean(data["managed"]) if "managed" in data else None obj.oneTimeCode = str(data["otp"]) if "otp" in data else None + obj.custom = load_custom_text_fields(data.get("custom")) obj.attachments = PamAttachmentsObject.load(data.get("attachments", None)) obj.scripts = PamScriptsObject.load(data.get("scripts", None)) @@ -1062,6 +1107,7 @@ def create_record(self, params, folder_uid): if managed is not None: fields.append(f"f.checkbox.managed={str(managed).lower()}") if self.oneTimeCode: fields.append(f"f.oneTimeCode={self.oneTimeCode}") + append_custom_text_fields(fields, self.custom) files = self.attachments.attachments if self.attachments and isinstance(self.attachments, PamAttachmentsObject) else [] if files and isinstance(files, list): @@ -1070,7 +1116,8 @@ def create_record(self, params, folder_uid): fields.append(f"file=@{x.file}") if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid @@ -1165,7 +1212,8 @@ def create_record(self, params, folder_uid): fields.append(f"file=@{x.file}") if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid return uid @@ -1435,7 +1483,8 @@ def create_record(self, params, folder_uid): # switch to f.* once RT definition(s) update w/ pamSettings field if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid @@ -1628,7 +1677,8 @@ def create_record(self, params, folder_uid): # switch to f.* once RT definition(s) update w/ pamSettings field if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid @@ -1776,7 +1826,8 @@ def create_record(self, params, folder_uid): # switch to f.* once RT definition(s) update w/ pamSettings field if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid @@ -1881,7 +1932,8 @@ def create_record(self, params, folder_uid): # switch to f.* once RT definition(s) update w/ pamRemoteBrowserSettings field if fields: args["fields"] = fields - uid = execute_record_add_in_folder(params, args, folder_uid, command='pam-project-import') + uid = execute_record_add_in_folder( + params, args, folder_uid, command='pam-project-import', sync_after=False) if uid and isinstance(uid, str): self.uid = uid diff --git a/keepercommander/commands/pam_import/cyberark_import.py b/keepercommander/commands/pam_import/cyberark_import.py index 163b4c691..950404a48 100644 --- a/keepercommander/commands/pam_import/cyberark_import.py +++ b/keepercommander/commands/pam_import/cyberark_import.py @@ -30,6 +30,8 @@ from prompt_toolkit import HTML, print_formatted_text from ..base import Command +from ..pam.vault_target import is_nested_share_folder, is_pam_nsf_record +from ... import api, vault, vault_extensions from ...display import bcolors from ...error import CommandError from ...importer.cyberark.cyberark_pam import ( @@ -62,6 +64,7 @@ RECORD_TYPE_PAM_MACHINE, RECORD_TYPE_PAM_DATABASE, RECORD_TYPE_PAM_DIRECTORY, + RECORD_TYPE_PAM_REMOTE_BROWSER, SCHEDULE_ON_DEMAND, ROTATION_UNMAPPED, ) @@ -70,12 +73,23 @@ IdempotencyDecision, PartitionSummary, RecordDecision, - annotate_record_with_marker, build_existing_index, partition_records, strip_id_marker, summarize, ) +from .nsf_helpers import find_pam_configuration, get_folder_record_uids +from .record_loader import iter_accessible_record_uids, load_pam_record + + +def is_matching_title(title: str, base: str) -> bool: + """True when *title* equals *base* or is ``{base} #N`` (case-insensitive).""" + title = title.casefold() + if title == base: + return True + if not title.startswith(base): + return False + return bool(re.fullmatch(r" #\d+", title[len(base):])) class SecureTempFileStore: @@ -160,6 +174,7 @@ class ImportRunOptions: user_map_file: str sync_mode: str = "upsert" strict_policies: bool = False + use_nsf: bool = False raw_kwargs: dict = field(default_factory=dict) @@ -597,31 +612,11 @@ def _map_single_account( "reason": f"unmappable platformId: {platform_id}", }) return - # Embed the CyberArk identity marker on every mapped record so - # future re-imports can match incoming accounts to already-created - # Keeper records (see importer/cyberark/pam/idempotency.py). The - # marker is a single line in ``notes`` — cheap to carry and - # survives the pam project import path unchanged (unlike ``custom`` - # fields, which PamBaseMachineParser does not preserve). - account_id = str(account.get("id", "") or "").strip() - if account_id: - annotate_record_with_marker(record, account_id, safe_name) - for nested in record.get("users") or []: - if not isinstance(nested, dict): - continue - # Nested pamUsers share the parent account's CyberArk id - # because CyberArk models the credential+resource as a - # single account. The marker is idempotent so re-runs - # can update the pamUser independently from its parent. - annotate_record_with_marker(nested, account_id, safe_name) self._apply_folder_paths(record, safe_name, folder_mapper) if password_failed: reason = "password retrieval failed" is_incomplete = True if is_incomplete: - note = f"INCOMPLETE: {reason}" - existing = (record.get("notes") or "").strip() - record["notes"] = f"{existing}\n{note}".strip() if existing else note incomplete.append(record) dual_fields = detect_dual_account(account) if dual_fields: @@ -641,6 +636,10 @@ def _map_single_account( if record.get("type") == RECORD_TYPE_LOGIN: pam_users.append(record) else: + if record.get("type") == RECORD_TYPE_PAM_REMOTE_BROWSER: + for user in record.pop("users", []) or []: + if isinstance(user, dict): + pam_users.append(user) if opts.skip_users: record.pop("users", None) pam_resources.append(record) @@ -696,6 +695,9 @@ def _collect_dependents(self, account: dict, record: dict, if not isinstance(dep, dict): continue dep["machine_title"] = machine_title + refs = dep.setdefault("machine_refs", []) + if isinstance(refs, list) and machine_title: + refs.append(machine_title) if dep.get("service_type") is None: unmapped_items.append({ "category": "CyberArk dependent", @@ -939,6 +941,7 @@ def _execute_vault_import(self, import_data: dict, self.params, import_data, opts.project_name, opts.config_uid, opts.batch_size, opts.batch_delay, mapped.pam_resources, mapped.pam_users, + use_nsf=opts.use_nsf, ) except Exception as e: logging.error("Import failed: %s", type(e).__name__) @@ -988,28 +991,22 @@ def _populate_folder_info(self, project_result: dict, project_name: str) -> None seen_uids: set = set() for wrapper_uid in wrapper_uids: - wrapper = self.params.folder_cache.get(wrapper_uid) - if not wrapper: - continue - for child_uid in getattr(wrapper, "subfolders", []) or []: - child = self.params.folder_cache.get(child_uid) - if not child or getattr(child, "type", "") != "shared_folder": + for child_uid, name in CyberArkPAMCleanupCommand._iter_project_child_folders( + self.params, wrapper_uid): + if child_uid in seen_uids: continue - if child.uid in seen_uids: - continue - seen_uids.add(child.uid) - name = getattr(child, "name", "") or "" + seen_uids.add(child_uid) if name == config_suffix: - config_folder_uid = child.uid + config_folder_uid = child_uid config_folder_name = name elif name == resources_suffix: - legacy_resources_uid = child.uid + legacy_resources_uid = child_uid legacy_resources_name = name elif name == users_suffix: - legacy_users_uid = child.uid + legacy_users_uid = child_uid legacy_users_name = name else: - safe_folders.append({"name": name, "uid": child.uid}) + safe_folders.append({"name": name, "uid": child_uid}) if config_folder_uid: folders_info["config_folder"] = config_folder_name @@ -1100,6 +1097,8 @@ def _maybe_auto_extend(self) -> None: return if opts.config_uid: return + if opts.use_nsf: + return if opts.dry_run or opts.output_file or opts.list_safes or opts.estimate_only: return # sync_down here so folder_cache / record_cache reflect any @@ -1139,6 +1138,8 @@ def _prepare_idempotency(self, mapped: MappedImportResult) -> Optional[dict]: opts = self.options if (opts.sync_mode or "").lower() == "create": return None + if opts.use_nsf and not opts.config_uid: + return None if opts.dry_run or opts.output_file: # Both paths skip the vault write; there's nothing to # de-dupe against and the pre-scan would only slow the @@ -1157,26 +1158,19 @@ def _prepare_idempotency(self, mapped: MappedImportResult) -> Optional[dict]: # pay the full-vault scan cost for nothing. return None - # Collect all shared folders directly under the project - # wrapper user folder(s). Records live inside these (and - # inside their Resources / Users subfolders for the - # safe-per-folder layout). + # Collect all shared / NSF folders directly under the project + # wrapper folder(s). Records live inside these (and inside + # their Resources / Users subfolders for the safe-per-folder + # layout). shared_folder_uids: list[str] = [] seen: set = set() for wrapper_uid in wrapper_uids: - wrapper = self.params.folder_cache.get(wrapper_uid) - if not wrapper: - continue - for child_uid in getattr(wrapper, "subfolders", []) or []: - child = self.params.folder_cache.get(child_uid) - if child is None: - continue - if getattr(child, "type", "") != "shared_folder": + for child_uid, _name in CyberArkPAMCleanupCommand._iter_project_child_folders( + self.params, wrapper_uid): + if child_uid in seen: continue - if child.uid in seen: - continue - seen.add(child.uid) - shared_folder_uids.append(child.uid) + seen.add(child_uid) + shared_folder_uids.append(child_uid) existing = build_existing_index(self.params, shared_folder_uids) if not existing.by_account_id and not existing.by_title: @@ -1619,9 +1613,7 @@ def _apply_service_dependent_mappings( }) continue - machine_record = self._find_machine_record( - dep.get("machine_address", ""), machine_index, - ) + machine_record = self._find_dependent_machine_record(dep, machine_index) if machine_record is None: summary["skipped_missing_machine"] += 1 summary["details"].append({ @@ -1704,6 +1696,29 @@ def _apply_service_dependent_mappings( return summary + @classmethod + def _find_dependent_machine_record(cls, dep: dict, machine_index: dict[str, Any]) -> Any: + refs = [] + if isinstance(dep, dict): + raw_refs = dep.get("machine_refs") + if isinstance(raw_refs, list): + refs.extend(raw_refs) + refs.extend([ + dep.get("machine_address", ""), + dep.get("machine_title", ""), + ]) + seen = set() + for ref in refs: + ref = str(ref or "").strip() + key = ref.casefold() + if not ref or key in seen: + continue + seen.add(key) + machine_record = cls._find_machine_record(ref, machine_index) + if machine_record is not None: + return machine_record + return None + def _build_record_indexes(self, mapped: MappedImportResult, vault, vault_extensions ) -> tuple[dict[str, Any], dict[str, Any]]: """Build ``(machine, user)`` record lookups from freshly imported vault data. @@ -1726,16 +1741,18 @@ def _build_record_indexes(self, mapped: MappedImportResult, vault, vault_extensi t = (u.get("title") or "").casefold() if t: imported_user_titles.add(t) - for rec in vault_extensions.find_records(self.params, record_version=3): + from .record_loader import iter_accessible_record_uids, load_pam_record + + for record_uid in iter_accessible_record_uids(self.params): + rec = load_pam_record(self.params, record_uid) + if rec is None: + continue rtype = getattr(rec, "record_type", "") or "" title = (getattr(rec, "title", "") or "").casefold() if rtype == RECORD_TYPE_PAM_MACHINE and title in imported_machine_titles: - loaded = vault.KeeperRecord.load(self.params, rec.record_uid) - if loaded is None: - continue - machine_index[title] = loaded + machine_index[title] = rec host_field = next( - (f for f in loaded.fields + (f for f in getattr(rec, "fields", []) or [] if (getattr(f, "type", "") or "") == "pamHostname" or (getattr(f, "label", "") or "").lower() == "host"), None, @@ -1748,11 +1765,9 @@ def _build_record_indexes(self, mapped: MappedImportResult, vault, vault_extensi elif isinstance(raw, str): host_str = raw if host_str: - machine_index[host_str.casefold()] = loaded + machine_index[host_str.casefold()] = rec elif rtype == "pamUser" and title in imported_user_titles: - loaded = vault.KeeperRecord.load(self.params, rec.record_uid) - if loaded is not None: - user_index[title] = loaded + user_index[title] = rec return machine_index, user_index @staticmethod @@ -1922,6 +1937,8 @@ class CyberArkPAMImportCommand(Command): # Self-hosted with mutual TLS client cert (P12) pam project cyberark-import pvwa.internal.com --client-cert-p12 ./client.p12 --name "Internal" + # Import into Nested Share Folders (NSF) + pam project cyberark-import pvwa.example.com --name "PAM Migration" --nsf ''') parser.add_argument("server", action="store", help="CyberArk PVWA host (e.g. mycompany.cyberark.cloud or pvwa.example.com)") parser.add_argument("--name", "-n", required=False, dest="project_name", action="store", @@ -1938,7 +1955,12 @@ class CyberArkPAMImportCommand(Command): "control. 'ksm'/'exact' nest safe-named subfolders under the " "legacy Resources/Users shared folders. 'flat' puts every " "record into the two legacy shared folders with aggregated " - "permissions.") + "permissions. With --nsf, the same layout is created using " + "Nested Share Folders instead of classic shared folders.") + parser.add_argument("--nsf", required=False, dest="use_nsf", action="store_true", + default=False, + help="Create project folders and records in Nested Share Folders " + "(folders, records, rotation, PAM config, and permissions).") parser.add_argument("--safes", required=False, dest="safes", action="store", default="", help="Include only matching Safes (comma-separated, supports globs)") parser.add_argument("--exclude-safes", required=False, dest="exclude_safes", action="store", @@ -2126,6 +2148,7 @@ def execute(self, params, **kwargs): user_map_file=kwargs.get("user_map", ""), sync_mode=(kwargs.get("sync_mode") or "upsert").lower(), strict_policies=bool(kwargs.get("strict_policies", False)), + use_nsf=kwargs.get("use_nsf", False) is True, raw_kwargs=kwargs, ) CyberArkImportOrchestrator(self, params, client, options).run() @@ -2134,7 +2157,8 @@ def execute(self, params, **kwargs): def _execute_import(self, params, import_data: dict, project_name: str, config_uid: str, batch_size: int, batch_delay: float, - resources: list[dict], users: list[dict]) -> Optional[dict]: + resources: list[dict], users: list[dict], + use_nsf: bool = False) -> Optional[dict]: """Execute the vault import using pam project import/extend commands.""" from .edit import PAMProjectImportCommand from .extend import PAMProjectExtendCommand @@ -2145,17 +2169,18 @@ def _execute_import(self, params, import_data: dict, project_name: str, if total_records <= batch_size: # Single batch — use import or extend directly return self._single_batch_import( - params, import_data, project_name, config_uid + params, import_data, project_name, config_uid, use_nsf=use_nsf, ) else: # Multi-batch: first batch creates project, remaining extend return self._multi_batch_import( params, import_data, project_name, config_uid, - resources, users, batch_size, batch_delay, + resources, users, batch_size, batch_delay, use_nsf=use_nsf, ) def _single_batch_import(self, params, import_data: dict, - project_name: str, config_uid: str) -> dict: + project_name: str, config_uid: str, + use_nsf: bool = False) -> dict: """Import all records in a single batch.""" from .edit import PAMProjectImportCommand from .extend import PAMProjectExtendCommand @@ -2163,12 +2188,18 @@ def _single_batch_import(self, params, import_data: dict, tmp_path = _temp_store.write_json(import_data) try: if config_uid: + # Extend auto-detects NSF from the existing project tree. + if use_nsf: + logging.info( + "--nsf is ignored when extending an existing project " + "(--config); Nested Share Folders are detected automatically.") PAMProjectExtendCommand().execute( - params, config=config_uid, file_name=tmp_path, dry_run=False + params, config=config_uid, file_name=tmp_path, dry_run=False, ) else: PAMProjectImportCommand().execute( - params, project_name=project_name, file_name=tmp_path, dry_run=False + params, project_name=project_name, file_name=tmp_path, + dry_run=False, use_nsf=use_nsf, ) finally: _temp_store.remove(tmp_path) @@ -2179,7 +2210,8 @@ def _single_batch_import(self, params, import_data: dict, def _multi_batch_import(self, params, import_data: dict, project_name: str, config_uid: str, resources: list[dict], users: list[dict], - batch_size: int, batch_delay: float) -> dict: + batch_size: int, batch_delay: float, + use_nsf: bool = False) -> dict: """Import records in multiple batches with adaptive throttling.""" from .edit import PAMProjectImportCommand from .extend import PAMProjectExtendCommand @@ -2207,7 +2239,8 @@ def _multi_batch_import(self, params, import_data: dict, tmp_path = _temp_store.write_json(first_batch_data) try: PAMProjectImportCommand().execute( - params, project_name=project_name, file_name=tmp_path, dry_run=False + params, project_name=project_name, file_name=tmp_path, + dry_run=False, use_nsf=use_nsf, ) finally: _temp_store.remove(tmp_path) @@ -2217,13 +2250,13 @@ def _multi_batch_import(self, params, import_data: dict, if not config_uid: config_uid = self._find_config_uid(params, project_name) else: - # Subsequent batches: extend + # Subsequent batches: extend (NSF auto-detected from project) extend_data = build_extend_json(batch_resources, batch_users) tmp_path = _temp_store.write_json(extend_data) try: if config_uid: PAMProjectExtendCommand().execute( - params, config=config_uid, file_name=tmp_path, dry_run=False + params, config=config_uid, file_name=tmp_path, dry_run=False, ) else: logging.error("Cannot extend: PAM configuration UID not found after initial import") @@ -2240,17 +2273,33 @@ def _multi_batch_import(self, params, import_data: dict, def _find_config_uid(self, params, project_name: str) -> str: """Find PAM configuration UID by project name after initial import. - Handles #N suffix deduplication from PAMProjectImportCommand.""" - from ... import api, vault_extensions - + Handles #N suffix deduplication from PAMProjectImportCommand. + Searches classic vault records and Nested Share Folder caches. + """ api.sync_down(params) config_base = f"{project_name} Configuration".casefold() candidates = [] + + # Classic path (keeps existing unit-test mocks working) for c in vault_extensions.find_records(params, record_version=6): - t = c.title.casefold() - if t == config_base or (t.startswith(config_base) and re.match(r' #\d+$', t[len(config_base):])): + if is_matching_title(getattr(c, "title", "") or "", config_base): candidates.append(c) + + # NSF / full-access path — pick up configs that live only in NSF caches + seen_uids = {getattr(c, "record_uid", "") for c in candidates} + for uid in iter_accessible_record_uids(params): + if uid in seen_uids: + continue + rec = load_pam_record(params, uid) + if rec and getattr(rec, "version", None) == 6: + if is_matching_title(getattr(rec, "title", "") or "", config_base): + candidates.append(rec) + seen_uids.add(uid) + if not candidates: + exact = find_pam_configuration(params, f"{project_name} Configuration") + if exact: + return exact.record_uid logging.warning(f"PAM configuration not found for project '{project_name}' after import") return "" # Prefer highest suffix number (most recently created) @@ -2364,7 +2413,7 @@ def _interactive_safe_picker(safes: list[dict]) -> Optional[list[str]]: class CyberArkPAMCleanupCommand(Command): - """Remove a CyberArk-imported project: records, folders, gateway, KSM app.""" + """Remove a CyberArk-imported project: records, folders, and PAM config.""" parser = argparse.ArgumentParser( prog="pam project cyberark-cleanup", @@ -2402,16 +2451,13 @@ def execute(self, params, **kwargs): raise CommandError("pam project cyberark-cleanup", "Either --name or --config is required") - from ... import api, utils, vault, vault_extensions - from ..pam import gateway_helper - from ..pam.config_helper import configuration_controller_get - from ...loginv3 import CommonHelperMethods - api.sync_down(params) - # Find PAM config by name or UID + # Find PAM config by name or UID (classic + NSF) if config_uid: config_rec = vault.KeeperRecord.load(params, config_uid) + if not config_rec: + config_rec = load_pam_record(params, config_uid) if not config_rec: raise CommandError("pam project cyberark-cleanup", f"PAM config record '{config_uid}' not found") @@ -2420,51 +2466,39 @@ def execute(self, params, **kwargs): config_base = f"{project_name} Configuration".casefold() config_rec = None for c in vault_extensions.find_records(params, record_version=6): - if c.title.casefold().startswith(config_base): + if is_matching_title(getattr(c, "title", "") or "", config_base): config_rec = c config_uid = c.record_uid break + if not config_rec: + for uid in iter_accessible_record_uids(params): + rec = load_pam_record(params, uid) + if rec and getattr(rec, "version", None) == 6: + if is_matching_title(getattr(rec, "title", "") or "", config_base): + config_rec = rec + config_uid = uid + break + if not config_rec: + exact = find_pam_configuration(params, f"{project_name} Configuration") + if exact: + config_rec = exact + config_uid = exact.record_uid if not config_rec: raise CommandError("pam project cyberark-cleanup", f"PAM config for project '{project_name}' not found") - # Resolve gateway linked to this PAM config - gateway_uid = None - gateway_name = None - gw_match = None - try: - controller = configuration_controller_get( - params, CommonHelperMethods.url_safe_str_to_bytes( - config_rec.record_uid)) - if controller and controller.controllerUid: - gateway_uid = controller.controllerUid - all_gw = gateway_helper.get_all_gateways(params) - gw_match = next((g for g in all_gw - if g.controllerUid == gateway_uid), None) - if gw_match: - gateway_name = gw_match.controllerName - except Exception as e: - logging.debug("Could not resolve gateway: %s", e) - - # Resolve KSM application linked to the gateway - ksm_app_uid = None - ksm_app_name = None - if gw_match and gw_match.applicationUid: - ksm_app_uid = utils.base64_url_encode(gw_match.applicationUid) - app_rec = vault.KeeperRecord.load(params, ksm_app_uid) - if app_rec: - ksm_app_name = getattr(app_rec, "title", ksm_app_uid) - - # Find shared folders. The new safe-per-folder layout creates one - # shared folder per CyberArk safe under the project wrapper folder + # Find shared / NSF folders. The new safe-per-folder layout creates one + # folder per CyberArk safe under the project wrapper folder # plus an admin Config folder; each safe folder has two - # ``Resources``/``Users`` shared_folder_folder subfolders that + # ``Resources``/``Users`` subfolders that # hold the records. The legacy layout creates exactly two folders # ("{project} - Resources" and "{project} - Users") with safe-named # subfolders inside. Discover everything by walking the project - # wrapper user-folder under PAM Environments so cleanup handles - # both shapes (and any subset thereof) without hardcoding names. + # wrapper under PAM Environments so cleanup handles + # both classic shared folders and Nested Share Folders + # (and any subset thereof) without hardcoding names. sf_uids = [] + record_count = 0 sf_names: list = [] all_record_uids: set = set() res_name = f"{project_name} - Resources" @@ -2474,68 +2508,40 @@ def execute(self, params, **kwargs): def _collect_records_recursive(folder_uid: str): """Walk the folder subtree and accumulate every record UID (records living directly in this folder + records living in - any descendant ``shared_folder_folder``).""" + any descendant classic or NSF subfolder).""" stack = [folder_uid] + visited: set = set() while stack: fuid = stack.pop() - for ruid in params.subfolder_record_cache.get(fuid, set()) or set(): - all_record_uids.add(ruid) + if not fuid or fuid in visited: + continue + visited.add(fuid) + all_record_uids.update(get_folder_record_uids(params, fuid)) folder = params.folder_cache.get(fuid) if not folder: + # NSF-only parents may still appear in nested_share_folders + nsf = getattr(params, "nested_share_folders", None) or {} + for child_uid, info in nsf.items(): + if (info.get("parent_uid") or None) == fuid: + stack.append(child_uid) continue for sub_uid in getattr(folder, "subfolders", []) or []: stack.append(sub_uid) - def _delete_folder(folder_uid: str) -> bool: - folder = params.folder_cache.get(folder_uid) - if not folder: - return False - del_obj = { - "delete_resolution": "unlink", - "object_uid": folder.uid, - "object_type": folder.type, - } - parent = params.folder_cache.get(folder.parent_uid) - if parent: - del_obj["from_uid"] = parent.uid - del_obj["from_type"] = parent.type - else: - del_obj["from_type"] = "user_folder" - rq = {"command": "pre_delete", "objects": [del_obj]} - rs = api.communicate(params, rq) - if rs.get("result") != "success": - return False - pdr = rs.get("pre_delete_response", {}) - del_rq = { - "command": "delete", - "pre_delete_token": pdr.get("pre_delete_token", ""), - } - api.communicate(params, del_rq) - return True - project_folder_uids = self._find_project_wrapper_folder_uids( params, project_name, ) if project_folder_uids: sf_uids_seen: set = set() for project_uid in project_folder_uids: - project_folder = params.folder_cache.get(project_uid) - if not project_folder: - continue - for child_uid in getattr(project_folder, "subfolders", []) or []: - child = params.folder_cache.get(child_uid) - if not child: - continue - # Only collect shared folders that we created — i.e. - # the type is SharedFolderType. - if getattr(child, "type", "") != "shared_folder": + for child_uid, child_name in self._iter_project_child_folders( + params, project_uid): + if child_uid in sf_uids_seen: continue - if child.uid in sf_uids_seen: - continue - sf_uids_seen.add(child.uid) - sf_uids.append(child.uid) - sf_names.append(getattr(child, "name", "") or "") - _collect_records_recursive(child.uid) + sf_uids_seen.add(child_uid) + sf_uids.append(child_uid) + sf_names.append(child_name) + _collect_records_recursive(child_uid) else: # Fallback: scan the shared-folder cache by name. Catches the # legacy two-folder layout when the project wrapper folder was @@ -2546,25 +2552,24 @@ def _delete_folder(folder_uid: str) -> bool: sf_uids.append(sf_uid) sf_names.append(name) _collect_records_recursive(sf_uid) - - # Ensure PAM config is included even if it lives outside the - # discovered shared-folder tree. - if config_uid: - all_record_uids.add(config_uid) + # NSF fallback by name under nested_share_folders + for nsf_uid, info in (getattr(params, "nested_share_folders", None) or {}).items(): + name = info.get("name", "") or "" + if name in (res_name, usr_name, config_name) and nsf_uid not in sf_uids: + sf_uids.append(nsf_uid) + sf_names.append(name) + _collect_records_recursive(nsf_uid) + record_count = len(all_record_uids) print(f"\nCyberArk PAM Project Cleanup") print("=" * 50) - print(f" Project: {project_name}") - print(f" PAM Config: {config_uid}") - print(f" Gateway: {gateway_name or '(not found)'}") - print(f" KSM App: {ksm_app_name or ksm_app_uid or '(not found)'}") - print(f" Folders: {len(sf_uids)}") + print(f" Project: {project_name}") + print(f" Config: {config_uid}") + print(f" Folders: {len(sf_uids)}") for sf_name in sf_names: if sf_name: print(f" • {sf_name}") - if project_folder_uids: - print(f" Wrappers: {len(project_folder_uids)}") - print(f" Records: {len(all_record_uids)}") + print(f" Records: ~{record_count}") if dry_run: print(" (dry run — no changes made)") @@ -2572,131 +2577,205 @@ def _delete_folder(folder_uid: str) -> bool: return if not auto_confirm: - answer = input("\n Delete all of the above? [y/N]: ").strip().lower() + answer = input("\n Delete project folders (and all records inside)? [y/N]: ").strip().lower() if answer not in ("y", "yes"): print(" Cancelled.") return - deleted = 0 - failed = 0 - - # Delete records in batches (same API as api.delete_record) - if all_record_uids: - logging.warning("Deleting %d records...", len(all_record_uids)) - uid_list = list(all_record_uids) - batch_size = 50 - for i in range(0, len(uid_list), batch_size): - batch = uid_list[i:i + batch_size] - try: - rq = {"command": "record_update", "delete_records": batch} - api.communicate(params, rq) - deleted += len(batch) - except Exception as e: - failed += len(batch) - logging.warning("Failed to delete record batch: %s", e) - - # Delete shared folders (safe folders + Config folder) - if sf_uids: - logging.warning("Removing %d shared folder(s)...", len(sf_uids)) - for sf_uid in sf_uids: - try: - if not _delete_folder(sf_uid): - failed += 1 - logging.warning("Failed to remove shared folder %s", - sf_uid) - except Exception as e: - failed += 1 - logging.warning("Failed to remove shared folder %s: %s", - sf_uid, e) - - # Delete project wrapper user-folder(s) under PAM Environments - if project_folder_uids: - logging.warning("Removing %d project wrapper folder(s)...", - len(project_folder_uids)) - for wrapper_uid in project_folder_uids: - try: - if not _delete_folder(wrapper_uid): - failed += 1 - logging.warning("Failed to remove wrapper folder %s", - wrapper_uid) - except Exception as e: - failed += 1 - logging.warning("Failed to remove wrapper folder %s: %s", - wrapper_uid, e) - - # Remove gateway - if gateway_uid: - logging.warning("Removing gateway \"%s\"...", - gateway_name or gateway_uid) + from ..folder import FolderRemoveCommand + from ..record import RecordRemoveCommand + from ..nested_share_folder.folder_commands import NestedShareFolderRemoveCommand + from ..nested_share_folder.record_commands import NestedShareRecordRemoveCommand + + # Prefer folder deletion: records (and nested subfolders) are removed + # with the folder tree. Delete content folders first, then wrappers. + folders_deleted = 0 + folders_failed = 0 + folders_to_remove = list(sf_uids) + list(project_folder_uids) + for folder_uid in folders_to_remove: try: - gateway_helper.remove_gateway(params, gateway_uid) + if is_nested_share_folder(params, folder_uid): + NestedShareFolderRemoveCommand().execute( + params, folders=[folder_uid], force=True, + operation="folder-trash", quiet=True) + else: + FolderRemoveCommand().execute( + params, pattern=[folder_uid], force=True, quiet=True) + folders_deleted += 1 except Exception as e: - failed += 1 - logging.warning("Failed to remove gateway: %s", e) + folders_failed += 1 + logging.warning("Failed to delete folder %s: %s", + folder_uid, type(e).__name__) + + api.sync_down(params) - # Remove KSM application - if ksm_app_uid: - logging.warning("Removing KSM app \"%s\"...", - ksm_app_name or ksm_app_uid) + # PAM config may already be gone with its Config folder; remove only + # if it survived (e.g. legacy layout / config outside project folders). + config_deleted = False + if config_uid and ( + config_uid in (params.record_cache or {}) + or config_uid in (getattr(params, "nested_share_records", None) or {}) + or vault.KeeperRecord.load(params, config_uid) + or load_pam_record(params, config_uid)): try: - from ..ksm import KSMCommand - KSMCommand.remove_v5_app(params, ksm_app_uid, - purge=True, force=True) + if is_pam_nsf_record(params, config_uid): + NestedShareRecordRemoveCommand().execute( + params, records=[config_uid], force=True, + operation="owner-trash") + else: + RecordRemoveCommand().execute( + params, force=True, record=config_uid) + config_deleted = True + api.sync_down(params) except Exception as e: - failed += 1 - logging.warning("Failed to remove KSM app: %s", e) - - api.sync_down(params) - msg = f"\nCleanup complete: {deleted} records deleted" - if failed: - msg += f" ({failed} failed — see warnings above)" + folders_failed += 1 + logging.warning("Failed to delete PAM config record %s: %s", + config_uid, type(e).__name__) + + msg = f"\nCleanup complete: {folders_deleted} folders deleted" + if config_deleted: + msg += " (PAM config removed)" + if folders_failed: + msg += f" ({folders_failed} failed — see warnings above)" print(msg) print("=" * 50) PAM_ROOT_FOLDER_NAME = "PAM Environments" + @classmethod + def _is_project_content_folder(cls, params, folder_uid: str, folder=None) -> bool: + """True for classic shared folders or Nested Share Folders under a project.""" + from ...subfolder import BaseFolderNode + + if is_nested_share_folder(params, folder_uid): + return True + folder = folder or (params.folder_cache or {}).get(folder_uid) + if not folder: + return False + ftype = getattr(folder, "type", "") or "" + return ftype in ( + BaseFolderNode.SharedFolderType, + BaseFolderNode.NestedShareFolderType, + ) + + @classmethod + def _iter_project_child_folders(cls, params, wrapper_uid: str): + """Yield ``(uid, name)`` for content folders under a project wrapper. + + Includes classic shared folders and Nested Share Folders so + report / idempotency / cleanup share one discovery path. + """ + seen: set = set() + folder_cache = getattr(params, "folder_cache", None) or {} + wrapper = folder_cache.get(wrapper_uid) + if wrapper: + for child_uid in getattr(wrapper, "subfolders", []) or []: + if child_uid in seen: + continue + child = folder_cache.get(child_uid) + if not child or not cls._is_project_content_folder(params, child_uid, child): + continue + seen.add(child_uid) + yield child_uid, getattr(child, "name", "") or "" + + # NSF children may exist in nested_share_folders before folder_cache + # parent.subfolders is fully linked after a partial sync. + for child_uid, info in (getattr(params, "nested_share_folders", None) or {}).items(): + if child_uid in seen: + continue + if (info.get("parent_uid") or None) != wrapper_uid: + continue + if not cls._is_project_content_folder(params, child_uid): + continue + seen.add(child_uid) + yield child_uid, info.get("name", "") or "" + @classmethod def _find_project_wrapper_folder_uids(cls, params, project_name: str) -> list: - """Return UIDs of every project wrapper user-folder under + """Return UIDs of every project wrapper folder under ``PAM Environments`` whose name matches ``project_name`` (or ``project_name #N`` when the project was imported multiple times). - The wrapper folder is a *user folder* (not shared), and per-safe - shared folders are created as direct children of it. Returning a - list keeps cleanup correct in the rare case where two projects - share a name (PAMProjectImportCommand allows duplicates via the - ``#N`` suffix). + Supports classic user-folder wrappers and Nested Share Folder + wrappers created with ``--nsf``. Returning a list keeps cleanup + correct in the rare case where two projects share a name + (PAMProjectImportCommand allows duplicates via the ``#N`` suffix). """ + from ...subfolder import BaseFolderNode + wrapper_uids: list = [] folders = params.folder_cache if params and params.folder_cache else {} if not isinstance(folders, dict): - return wrapper_uids + folders = {} - # Locate root "PAM Environments" user folder(s). + # Locate root "PAM Environments" folder(s) — classic user folder + # and/or Nested Share Folder. root_uids: list = [] + seen_roots: set = set() for uid, f in folders.items(): if not f or getattr(f, "parent_uid", None): continue - if getattr(f, "type", "") != "user_folder": + ftype = getattr(f, "type", "") or "" + if ftype not in ( + BaseFolderNode.UserFolderType, + BaseFolderNode.NestedShareFolderType, + ): + continue + if getattr(f, "name", "") != cls.PAM_ROOT_FOLDER_NAME: + continue + root_uids.append(uid) + seen_roots.add(uid) + + for uid, info in (getattr(params, "nested_share_folders", None) or {}).items(): + if uid in seen_roots: + continue + if (info.get("parent_uid") or None) is not None: + continue + if info.get("name", "") != cls.PAM_ROOT_FOLDER_NAME: continue - if getattr(f, "name", "") == cls.PAM_ROOT_FOLDER_NAME: - root_uids.append(uid) + root_uids.append(uid) + seen_roots.add(uid) + if not root_uids: return wrapper_uids # PAMProjectImportCommand emits "{project_name}" or - # "{project_name} #N" for the wrapper user folder, so match both - # shapes here. + # "{project_name} #N" for the wrapper folder, so match both + # shapes here (classic user folder or NSF). base = project_name + name_re = re.compile(rf"^{re.escape(base)} #\d+$") + seen_wrappers: set = set() + + def _maybe_add_wrapper(uid: str, name: str, folder=None) -> None: + if uid in seen_wrappers: + return + if name != base and not name_re.match(name): + return + ftype = getattr(folder, "type", "") if folder else "" + if folder is not None: + if ftype not in ( + BaseFolderNode.UserFolderType, + BaseFolderNode.NestedShareFolderType, + ): + return + elif not is_nested_share_folder(params, uid): + return + wrapper_uids.append(uid) + seen_wrappers.add(uid) + for root_uid in root_uids: root_folder = folders.get(root_uid) - if not root_folder: - continue - for child_uid in getattr(root_folder, "subfolders", []) or []: - child = folders.get(child_uid) - if not child or getattr(child, "type", "") != "user_folder": + if root_folder: + for child_uid in getattr(root_folder, "subfolders", []) or []: + child = folders.get(child_uid) + if not child: + continue + name = getattr(child, "name", "") or "" + _maybe_add_wrapper(child_uid, name, child) + + for child_uid, info in (getattr(params, "nested_share_folders", None) or {}).items(): + if (info.get("parent_uid") or None) != root_uid: continue - name = getattr(child, "name", "") or "" - if name == base or re.match(rf"^{re.escape(base)} #\d+$", name): - wrapper_uids.append(child.uid) + _maybe_add_wrapper(child_uid, info.get("name", "") or "") + return wrapper_uids diff --git a/keepercommander/commands/pam_import/edit.py b/keepercommander/commands/pam_import/edit.py index e2c6420ef..2cd136053 100644 --- a/keepercommander/commands/pam_import/edit.py +++ b/keepercommander/commands/pam_import/edit.py @@ -445,10 +445,22 @@ def _create_safe_folders(self, params, project: dict, project_folder_uid: str, # record. Cannot live in any safe folder, or the safe's members # would gain access to the central config record. config_folder_name = f"""{res["project_folder"]} - Config""" + + # Verify principals once (avoids one round-trip per safe). + if all_user_perms: + self.verify_users_and_teams(params, all_user_perms) + + if use_nsf: + self._create_safe_folders_nsf_batch( + params, project_folder_uid, res, safe_folder_records, + config_folder_name, safe_folder_map, + ) + return + config_uid = self.create_subfolder( params, folder_name=config_folder_name, parent_uid=project_folder_uid, permissions=dict(default_fperm), - use_nsf=use_nsf, + use_nsf=False, ) res["resources_folder"] = config_folder_name res["users_folder"] = config_folder_name @@ -457,10 +469,6 @@ def _create_safe_folders(self, params, project: dict, project_folder_uid: str, res["config_folder_uid"] = config_uid res["config_folder"] = config_folder_name - # Verify principals once (avoids one round-trip per safe). - if all_user_perms: - self.verify_users_and_teams(params, all_user_perms) - # Create one shared folder per safe under the project wrapper and # apply its specific permission set. Inside each safe folder, # create two organizational subfolders named @@ -477,7 +485,7 @@ def _create_safe_folders(self, params, project: dict, project_folder_uid: str, folder_uid = self.create_subfolder( params, folder_name=record["name"], parent_uid=project_folder_uid, permissions=record["fperm"], - use_nsf=use_nsf, + use_nsf=False, ) # Top-level lookup key (no slash) maps to the safe folder # itself for callers that still emit ``folder_path = ""`` @@ -488,11 +496,11 @@ def _create_safe_folders(self, params, project: dict, project_folder_uid: str, usr_sub_name = f"{record['name']} - Users" res_sub_uid = self.create_subfolder( params, folder_name=res_sub_name, parent_uid=folder_uid, - use_nsf=use_nsf, + use_nsf=False, ) usr_sub_uid = self.create_subfolder( params, folder_name=usr_sub_name, parent_uid=folder_uid, - use_nsf=use_nsf, + use_nsf=False, ) safe_folder_map[f"{record['name']}/{res_sub_name}"] = res_sub_uid safe_folder_map[f"{record['name']}/{usr_sub_name}"] = usr_sub_uid @@ -511,6 +519,101 @@ def _create_safe_folders(self, params, project: dict, project_folder_uid: str, res["safe_folder_map"] = safe_folder_map + def _create_safe_folders_nsf_batch(self, params, project_folder_uid: str, + res: dict, safe_folder_records: list, + config_folder_name: str, + safe_folder_map: dict) -> None: + """Create Config + per-safe NSF folders via batched folder_add_v3. + + Two layers (parents must exist before children): + 1. Config folder + each safe folder under the project + 2. ``{safe} - Resources`` / ``{safe} - Users`` under each safe + """ + from .nsf_helpers import create_nsf_folders_batch + + # Layer 1: config + safe wrappers (same parent) + layer1_specs = [ + {"name": config_folder_name, "parent_uid": project_folder_uid}, + ] + for record in safe_folder_records: + layer1_specs.append({ + "name": record["name"], + "parent_uid": project_folder_uid, + }) + + layer1 = create_nsf_folders_batch( + params, layer1_specs, sync=False, command='pam-project-import', + ) + config_uid = layer1[0]["folder_uid"] + res["resources_folder"] = config_folder_name + res["users_folder"] = config_folder_name + res["resources_folder_uid"] = config_uid + res["users_folder_uid"] = config_uid + res["config_folder_uid"] = config_uid + res["config_folder"] = config_folder_name + + safe_uids: list = [] + for i, record in enumerate(safe_folder_records): + folder_uid = layer1[i + 1]["folder_uid"] + safe_uids.append(folder_uid) + safe_folder_map[record["name"]] = folder_uid + + # Layer 2: Resources / Users children under each safe + layer2_specs: list = [] + layer2_meta: list = [] # (safe_index, kind, sub_name) + for i, record in enumerate(safe_folder_records): + res_sub_name = f"{record['name']} - Resources" + usr_sub_name = f"{record['name']} - Users" + parent_uid = safe_uids[i] + layer2_specs.append({"name": res_sub_name, "parent_uid": parent_uid}) + layer2_meta.append((i, "resources", res_sub_name)) + layer2_specs.append({"name": usr_sub_name, "parent_uid": parent_uid}) + layer2_meta.append((i, "users", usr_sub_name)) + + layer2 = [] + if layer2_specs: + layer2 = create_nsf_folders_batch( + params, layer2_specs, sync=True, command='pam-project-import', + ) + else: + # Config-only project: still need one sync after layer 1. + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) + + # Assemble maps / safe_folders entries + by_safe: dict = { + i: {"resources_uid": None, "users_uid": None, + "resources_name": None, "users_name": None} + for i in range(len(safe_folder_records)) + } + for result, (safe_i, kind, sub_name) in zip(layer2, layer2_meta): + uid = result["folder_uid"] + parent_name = safe_folder_records[safe_i]["name"] + safe_folder_map[f"{parent_name}/{sub_name}"] = uid + if kind == "resources": + by_safe[safe_i]["resources_uid"] = uid + by_safe[safe_i]["resources_name"] = sub_name + else: + by_safe[safe_i]["users_uid"] = uid + by_safe[safe_i]["users_name"] = sub_name + + for i, record in enumerate(safe_folder_records): + folder_uid = safe_uids[i] + info = by_safe[i] + res["safe_folders"].append({ + "name": record["name"], + "safe_name": record["safe_name"], + "uid": folder_uid, + "resources_subfolder": info["resources_name"], + "resources_subfolder_uid": info["resources_uid"], + "users_subfolder": info["users_name"], + "users_subfolder_uid": info["users_uid"], + }) + if record["uperm"]: + self.add_folder_permissions(params, folder_uid, record["uperm"]) + + res["safe_folder_map"] = safe_folder_map + def process_ksm_app(self, params, project: dict) -> dict: res = { "app_name_target": "", @@ -1039,17 +1142,14 @@ def create_subfolder(self, params, folder_name:str, parent_uid:str="", permissio name = str(folder_name or "").strip() if use_nsf or is_nested_share_folder(params, parent_uid): - from .nsf_helpers import seed_nsf_folder_cache, sync_down_preserving_nsf_keys - from ...nested_share_folder.folder_api import create_folder_v3 - result = create_folder_v3(params, name, parent_uid=parent_uid or None) - if isinstance(result, dict) and result.get('success') is False: - raise CommandError("pam", result.get('message') or 'Failed to create Nested Share Folder') - folder_uid = result.get('folder_uid') if isinstance(result, dict) else None - if not folder_uid: - raise CommandError("pam", f'Nested Share Folder creation did not return UID: {name}') - folder_key = result.get('folder_key_unencrypted') if isinstance(result, dict) else None - seed_nsf_folder_cache(params, folder_uid, name, parent_uid or None, folder_key) - sync_down_preserving_nsf_keys(params) + from .nsf_helpers import create_nsf_folders_batch + results = create_nsf_folders_batch( + params, + [{"name": name, "parent_uid": parent_uid or None}], + sync=True, + command='pam', + ) + folder_uid = results[0]["folder_uid"] params.environment_variables[LAST_FOLDER_UID] = folder_uid return folder_uid @@ -1536,6 +1636,9 @@ def _resolve_folder_uid(obj, default_uid: str) -> str: logging.warning(f"Processing external users: {len(users)}") for n, user in enumerate(users): # standalone users user.create_record(params, _resolve_folder_uid(user, shfusr)) + if user.uid and user.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) if n % pdelta == 0: print(f"{n}/{len(users)}") print(f"{len(users)}/{len(users)}\n") @@ -1549,6 +1652,11 @@ def _resolve_folder_uid(obj, default_uid: str) -> str: admin_uid = get_admin_credential(mach, True) mach_folder_uid = _resolve_folder_uid(mach, shfres) mach.create_record(params, mach_folder_uid) + # NSF creates with sync_after=False; pam tunnel edit resolves from + # record_cache, so sync once when the new UID is not loaded yet. + if mach.uid and mach.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) tdag.link_resource_to_config(mach.uid) if isinstance(mach, PamRemoteBrowserObject): # RBI args = parse_command_options(mach, True) @@ -1621,6 +1729,9 @@ def _resolve_folder_uid(obj, default_uid: str) -> str: # to every record originating from that CyberArk safe. user_folder_uid = _resolve_folder_uid(user, mach_folder_uid) user.create_record(params, user_folder_uid) + if user.uid and user.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) if isinstance(user, PamUserObject): # rotation setup tdag.link_user_to_resource(user.uid, mach.uid, admin_uid==user.uid, True) if user.rotation_settings: @@ -1712,4 +1823,10 @@ def _resolve_folder_uid(obj, default_uid: str) -> str: else: logging.debug(f"Unable to resolve domain admin '{pce.dom_administrative_credential}' for PAM Domain configuration.") + # One sync after bulk NSF record creates (create_record uses sync_after=False). + use_nsf = project["options"].get("use_nsf", False) is True + if use_nsf or is_nested_share_folder(params, shfres) or is_nested_share_folder(params, shfusr): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) + logging.debug("Done processing project data.") diff --git a/keepercommander/commands/pam_import/extend.py b/keepercommander/commands/pam_import/extend.py index ecc846e04..f2b1a208b 100644 --- a/keepercommander/commands/pam_import/extend.py +++ b/keepercommander/commands/pam_import/extend.py @@ -1426,6 +1426,9 @@ def process_data(self, params, project): for n, user in enumerate(new_users): folder_uid = getattr(user, "resolved_folder_uid", None) or shfusr extend_create_record(params, user, folder_uid) + if user.uid and user.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) if n % pdelta == 0: print(f"{n}/{len(new_users)}") print(f"{len(new_users)}/{len(new_users)}\n") @@ -1441,6 +1444,11 @@ def process_data(self, params, project): folder_uid = getattr(mach, "resolved_folder_uid", None) or shfres admin_uid = get_admin_credential(mach, True) extend_create_record(params, mach, folder_uid) + # NSF creates with sync_after=False; pam tunnel edit resolves from + # record_cache / NSF caches, so sync when the new UID is not loaded yet. + if mach.uid and mach.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) tdag.link_resource_to_config(mach.uid) if isinstance(mach, PamRemoteBrowserObject): args = parse_command_options(mach, True) @@ -1508,6 +1516,9 @@ def process_data(self, params, project): rs.resourceUid = mach.uid ufolder = getattr(user, "resolved_folder_uid", None) or shfusr extend_create_record(params, user, ufolder) + if user.uid and user.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) if isinstance(user, PamUserObject): tdag.link_user_to_resource(user.uid, mach.uid, admin_uid == user.uid, True) if rs: @@ -1546,6 +1557,9 @@ def process_data(self, params, project): rs.resourceUid = mach.uid ufolder = getattr(user, "resolved_folder_uid", None) or shfusr extend_create_record(params, user, ufolder) + if user.uid and user.uid not in (getattr(params, 'record_cache', None) or {}): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) if isinstance(user, PamUserObject): tdag.link_user_to_resource(user.uid, mach.uid, admin_uid == user.uid, True) if rs: @@ -1628,5 +1642,11 @@ def process_data(self, params, project): if refs: api.sync_down(params) add_pam_scripts(params, pam_cfg_uid, refs) + + # One sync after bulk NSF record creates (create_record uses sync_after=False). + if is_nested_share_folder(params, shfres) or is_nested_share_folder(params, shfusr): + from .nsf_helpers import sync_down_preserving_nsf_keys + sync_down_preserving_nsf_keys(params) + logging.debug("Done processing project data.") return diff --git a/keepercommander/commands/pam_import/nsf_helpers.py b/keepercommander/commands/pam_import/nsf_helpers.py index 4088daa93..6cc823857 100644 --- a/keepercommander/commands/pam_import/nsf_helpers.py +++ b/keepercommander/commands/pam_import/nsf_helpers.py @@ -395,6 +395,65 @@ def create_nsf_subfolder(params, folder_name: str, parent_uid: str = '', return folder_uid +_NSF_FOLDER_BATCH_LIMIT = 100 + + +def create_nsf_folders_batch(params, folder_specs: List[dict], *, + sync: bool = True, + command: str = 'pam') -> List[dict]: + """Create NSF folders via ``vault/folders/v3/add`` in chunks of 100. + + Each *folder_specs* entry is ``{'name': str, 'parent_uid': str|None}``. + Returns the batch API result list (same order as *folder_specs*). + Seeds local NSF caches after each successful create; optionally syncs once + at the end so callers avoid one round-trip per folder. + """ + from ...nested_share_folder.folder_api import create_folders_batch_v3 + + if not folder_specs: + return [] + + results: List[dict] = [] + for start in range(0, len(folder_specs), _NSF_FOLDER_BATCH_LIMIT): + chunk = folder_specs[start:start + _NSF_FOLDER_BATCH_LIMIT] + try: + chunk_results = create_folders_batch_v3(params, chunk) + except Exception as exc: + raise CommandError(command, f'NSF folder batch create failed: {exc}') from exc + + if len(chunk_results) != len(chunk): + raise CommandError( + command, + f'NSF folder batch returned {len(chunk_results)} results for {len(chunk)} folders', + ) + + for spec, result in zip(chunk, chunk_results): + if not result.get('success'): + name = spec.get('name') or '?' + raise CommandError( + command, + result.get('message') or f'Failed to create Nested Share Folder: {name}', + ) + folder_uid = result.get('folder_uid') + if not folder_uid: + raise CommandError( + command, + f"Nested Share Folder creation did not return UID: {spec.get('name')}", + ) + seed_nsf_folder_cache( + params, + folder_uid, + spec.get('name') or '', + spec.get('parent_uid') or None, + result.get('folder_key_unencrypted'), + ) + results.append(result) + + if sync: + sync_down_preserving_nsf_keys(params) + return results + + def extend_create_record(params, obj, folder_uid: str) -> Optional[str]: """Create a PAM import record in a classic or NSF folder.""" return obj.create_record(params, folder_uid) diff --git a/keepercommander/commands/tunnel/port_forward/tunnel_helpers.py b/keepercommander/commands/tunnel/port_forward/tunnel_helpers.py index 902939437..e3384c78b 100644 --- a/keepercommander/commands/tunnel/port_forward/tunnel_helpers.py +++ b/keepercommander/commands/tunnel/port_forward/tunnel_helpers.py @@ -331,6 +331,12 @@ def __init__(self, tube_id, conversation_id, gateway_uid, symmetric_key, self.websocket_thread = None self.websocket_ready_event = None self.websocket_stop_event = None + # Late-arrival channel for Gateway error payloads (is_ok=False). + # The -nti HTTP offer path waits on `gateway_error_event` briefly + # after an RRC_TIMEOUT, so a delayed "Unable to establish connection + # to remote host …" message can replace the generic timeout text. + self.gateway_error_message = None + self.gateway_error_event = threading.Event() # Optional attributes (set dynamically) # Note: signal_handler is set after TunnelSignalHandler is created self.signal_handler = None # type: ignore[assignment] @@ -1542,9 +1548,22 @@ def route_message_to_rust(response_item, tube_registry): elif not payload_data.get('is_ok', True): # Gateway returned an explicit error (is_ok=False) — log the message and move on. # This includes auth failures (401 on get_leafs), overload responses, etc. + gateway_error_text = payload_data.get('data', 'unknown error') logging.error( - f"Gateway error for {conversation_id}: {payload_data.get('data', 'unknown error')}" + f"Gateway error for {conversation_id}: {gateway_error_text}" ) + try: + err_tube_id = tube_registry.tube_id_from_connection_id(conversation_id) + if not err_tube_id: + url_safe_cid = conversation_id.replace('+', '-').replace('/', '_').rstrip('=') + err_tube_id = tube_registry.tube_id_from_connection_id(url_safe_cid) + if err_tube_id: + err_session = get_tunnel_session(err_tube_id) + if err_session is not None: + err_session.gateway_error_message = gateway_error_text + err_session.gateway_error_event.set() + except Exception as stash_exc: + logging.debug(f"Failed to stash gateway error on session: {stash_exc}") elif payload_data.get('data', '') == '': logging.debug("Empty data field an acknowledgment, no action needed") elif payload_data.get('data') and "ice candidate added" in payload_data.get('data').lower(): @@ -2636,6 +2655,13 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, } logging.debug(f"Including user-supplied host in payload: {target_host}:{target_port}") + # KeeperRDP Proxy and KeeperDB Proxy both auto-route on the + # gateway side from the record's `allowKeeper{DB,RDP}Proxy` + # setting alone, so no client-side opt-in flag is included in + # the WebRTC payload here. The Commander-side `--proxy` flag is + # used for validators (record-type check, `allowKeeperXxxProxy` + # presence check, launch-credential preflight) and to print the + # right banner; it does not need to round-trip to the gateway. string_data = json.dumps(data) bytes_data = string_to_bytes(string_data) encrypted_data = tunnel_encrypt(symmetric_key, bytes_data) @@ -2699,6 +2725,7 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, except Exception as e: error_msg = str(e) is_bad_state = "RRC_BAD_STATE" in error_msg + is_timeout = "RRC_TIMEOUT" in error_msg is_last_attempt = (attempt == max_retries) if is_bad_state and not is_last_attempt: @@ -2712,7 +2739,21 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, logging.error(f"RRC_BAD_STATE persists after {max_retries} retries") logging.error("This may indicate network issues or backend problems") - logging.error(f"Failed to send offer via HTTP: {error_msg}") + # On RRC_TIMEOUT, give the Gateway a brief grace period to + # deliver a real error payload via WebSocket (e.g. "Unable + # to establish connection to remote host …"). Without this + # the user only sees the generic HTTP timeout even when the + # gateway already knows why upstream is unreachable. + final_error = error_msg + if is_timeout and not tunnel_session.gateway_error_event.is_set(): + logging.debug( + "RRC_TIMEOUT — waiting up to 8s for delayed Gateway error payload" + ) + tunnel_session.gateway_error_event.wait(timeout=8.0) + if tunnel_session.gateway_error_message: + final_error = tunnel_session.gateway_error_message + + logging.error(f"Failed to send offer via HTTP: {final_error}") # Cleanup on final failure logging.debug(f"Cleaning up failed tunnel {commander_tube_id}") @@ -2734,7 +2775,7 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, signal_handler.cleanup() unregister_tunnel_session(commander_tube_id) - return {"success": False, "error": f"Failed to send offer via HTTP: {e}"} + return {"success": False, "error": f"Failed to start tunnel: {final_error}"} # Continue with the rest of the flow after successful offer send # Trickle ICE: Response comes via WebSocket (HTTP response is empty) @@ -2753,6 +2794,26 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, payload_json = json.loads(payload_str) logging.debug(f"Non-trickle ICE: Parsed payload JSON, keys: {payload_json.keys()}") + # Gateway-side error: the HTTP response carries an + # is_ok=False envelope whose 'data' field is a plain- + # text reason rather than an encrypted SDP answer. + # Surface it with the same one-line wording the + # trickle path uses (route_message_to_rust) and bail + # out — otherwise the code below decrypts garbage, + # raises a TypeError in json.loads(None), and the + # banner prints anyway. + if not payload_json.get('is_ok', True): + gateway_error_text = payload_json.get('data', 'unknown error') + logging.error( + f"Gateway error for {conversation_id_original}: {gateway_error_text}" + ) + if tunnel_session.websocket_stop_event and tunnel_session.websocket_thread: + tunnel_session.websocket_stop_event.set() + tunnel_session.websocket_thread.join(timeout=2.0) + signal_handler.cleanup() + unregister_tunnel_session(commander_tube_id) + return {"success": False, "error": f"Failed to start tunnel: {gateway_error_text}"} + encrypted_answer = payload_json.get('data') if encrypted_answer: # Decrypt the answer using the tunnel's symmetric key @@ -2773,8 +2834,6 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, logging.error(f"Non-trickle ICE: No 'payload' field in gateway response: {gateway_payload}") except Exception as e: logging.error(f"Non-trickle ICE: Failed to process HTTP response: {e}") - import traceback - logging.error(f"Traceback: {traceback.format_exc()}") # Send any buffered ICE candidates that arrived before offer was sent (trickle ICE only) if trickle_ice and tunnel_session.buffered_ice_candidates: @@ -2811,6 +2870,7 @@ def start_rust_tunnel(params, record_uid, gateway_uid, host, port, "conversation_id": conversation_id_original, # Use original, not base64 encoded "tube_registry": tube_registry, "status": "connecting", # Indicates async connection in progress + "local_host": tunnel_session.host, # Actual bound host (usually 127.0.0.1) "local_port": tunnel_session.port, # Actual bound port (may differ from requested) } diff --git a/keepercommander/commands/tunnel_and_connections.py b/keepercommander/commands/tunnel_and_connections.py index f818ef8f7..3a530118b 100644 --- a/keepercommander/commands/tunnel_and_connections.py +++ b/keepercommander/commands/tunnel_and_connections.py @@ -29,8 +29,8 @@ from .base import Command, GroupCommand, dump_report_data, RecordMixin from .tunnel.port_forward.TunnelGraph import TunnelDAG -from .tunnel.port_forward.tunnel_helpers import find_open_port, get_config_uid, get_config_uid_via_pam_link, \ - get_keeper_tokens, \ +from .tunnel.port_forward.tunnel_helpers import find_open_port, is_port_open, get_config_uid, \ + get_config_uid_via_pam_link, get_keeper_tokens, \ get_or_create_tube_registry, get_gateway_uid_from_record, resolve_record, resolve_pam_config, resolve_folder, \ remove_field, start_rust_tunnel, get_tunnel_session, unregister_tunnel_session, CloseConnectionReasons, \ wait_for_tunnel_connection, create_rust_webrtc_settings, \ @@ -391,6 +391,13 @@ class PAMTunnelEditCommand(Command): pam_cmd_parser.add_argument('--keeper-db-proxy', '-kdbp', required=False, dest='keeper_db_proxy', choices=['on', 'off', 'default'], help='Enable/disable Keeper Database Proxy for pamDatabase records (on/off/default)') + pam_cmd_parser.add_argument('--keeper-proxy', '-kp', required=False, dest='keeper_proxy', + choices=['on', 'off', 'default'], + help='Enable/disable Keeper Proxy, auto-selecting the backend from the record\'s ' + 'connection protocol: KeeperDB Proxy for database protocols ' + '(pamDatabase, or pamMachine set to a database protocol), KeeperRDP Proxy ' + 'for RDP (pamMachine). SSH is not yet supported. Records/protocols that ' + 'don\'t map to a proxy backend are ignored with a warning (on/off/default)') def get_parser(self): return PAMTunnelEditCommand.pam_cmd_parser @@ -591,6 +598,71 @@ def execute(self, params, **kwargs): pam_settings.value[0]["connection"].pop('allowKeeperDBProxy', None) dirty = True + # Handle --keeper-proxy: generalized on/off/default that auto-selects the + # backend from the record's connection protocol instead of requiring the + # protocol-specific flag (--keeper-db-proxy today; --keeper-rdp-proxy / + # --keeper-ssh-proxy would be the alternative). Missing option -> no-op, + # no validation at all. + keeper_proxy = kwargs.get('keeper_proxy') + if keeper_proxy: + protocol = '' + if pam_settings and pam_settings.value and isinstance(pam_settings.value[0], dict): + protocol = (pam_settings.value[0].get('connection') or {}).get('protocol') or '' + protocol = str(protocol).strip().lower() + + proxy_key = None + proxy_bucket = None + if record_type == 'pamDatabase' or protocol in PAMConnectionEditCommand.db_protocols: + proxy_key = 'allowKeeperDBProxy' + proxy_bucket = 'connection' + elif protocol == 'rdp': + proxy_key = 'allowKeeperRDPProxy' + proxy_bucket = 'portForward' + elif protocol == 'ssh': + # TODO(ssh-proxy): once KeeperSSH Proxy support lands on the gateway, + # set pamSettings..allowKeeperSSHProxy here, mirroring the + # KeeperDB/KeeperRDP branches above. + print(f"{bcolors.WARNING}KeeperSSH Proxy is not implemented yet; " + f"--keeper-proxy has no effect for record {record_uid}.{bcolors.ENDC}") + else: + print(f"{bcolors.WARNING}--keeper-proxy has no effect for record {record_uid} " + f"(type=\"{record_type}\", protocol=\"{protocol}\"). KeeperDB Proxy applies " + f"only to database records/protocols, KeeperRDP Proxy only to RDP records, " + f"and KeeperSSH Proxy (planned) only to SSH records.{bcolors.ENDC}") + + if proxy_key: + if keeper_proxy == 'default': + # Pure deletion: never create pamSettings or its connection/portForward + # buckets just to immediately leave them empty. No-op (no validation, + # no error) if pamSettings, the bucket, or the key isn't there at all. + if pam_settings and pam_settings.value and isinstance(pam_settings.value[0], dict): + bucket = pam_settings.value[0].get(proxy_bucket) + if isinstance(bucket, dict) and proxy_key in bucket: + bucket.pop(proxy_key, None) + dirty = True + else: + if keeper_proxy == 'on' and not tmp_dag.check_if_resource_has_launch_credential(record_uid): + raise CommandError('', + f'{bcolors.FAIL}No Launch Credentials assigned to record "{record_uid}". ' + f'Please assign launch credentials to the record before enabling ' + f'the proxy.\n' + f'Use: {bcolors.OKBLUE}pam connection edit ' + f'--launch-user (-lu) {bcolors.ENDC}') + if not pam_settings: + pam_settings = vault.TypedField.new_field('pamSettings', {"connection": {}, "portForward": {}}, "") + record.custom.append(pam_settings) + if not pam_settings.value: + pam_settings.value.append({"connection": {}, "portForward": {}}) + if not isinstance(pam_settings.value[0], dict): + pam_settings.value[0] = {"connection": {}, "portForward": {}} + if not isinstance(pam_settings.value[0].get(proxy_bucket), dict): + pam_settings.value[0][proxy_bucket] = {} + current_value = pam_settings.value[0][proxy_bucket].get(proxy_key) + target = (keeper_proxy == 'on') + if current_value is not target: + pam_settings.value[0][proxy_bucket][proxy_key] = target + dirty = True + if dirty: tmp_dag.set_resource_allowed(resource_uid=record_uid, tunneling=_tunneling, allowed_settings_name=allowed_settings_name) was_nsf = update_pam_record(params, record, command='pam tunnel edit') @@ -629,10 +701,9 @@ class PAMTunnelStartCommand(Command): help='Disable trickle ICE for WebRTC connections. By default, trickle ICE is enabled ' 'for real-time candidate exchange.') pam_cmd_parser.add_argument('--proxy', '-px', required=False, dest='proxy', action='store_true', - help='Activate KeeperDB Proxy: the gateway substitutes credentials ' - 'from your Keeper vault when the local client connects to the tunnel.') - # TODO(rdp-proxy): once RDP Proxy support lands on pamMachine, generalize --proxy - # to or auto-detect from the record type). For now, --proxy is KeeperDB-only. + help='Activate Keeper Proxy (KeeperDB for pamDatabase, KeeperRDP for ' + 'pamMachine + RDP): the gateway substitutes credentials from your ' + 'Keeper vault when the local client connects to the tunnel.') pam_cmd_parser.add_argument('--reason', '-r', required=False, dest='workflow_reason', type=str, help='Justification text for workflow access request. Used when the record\'s ' 'workflow requires a reason; non-interactive equivalent of the inline prompt.') @@ -704,6 +775,26 @@ def _resolve_database_type(record, pam_settings_value): return 'mssql' return None + @staticmethod + def _resolve_connection_protocol(pam_settings_value): + # Lower-cased connection protocol from the record's pamSettings + # ('rdp' | 'ssh' | 'vnc' | 'telnet' | ''). pamMachine uses this to + # pick the right Gateway proxy backend. + if isinstance(pam_settings_value, dict): + return ((pam_settings_value.get('connection') or {}).get('protocol') or '').strip().lower() + return '' + + @staticmethod + def _print_keeperrdp_proxy_banner(host, port): + print(f"\n{bcolors.OKGREEN}KeeperRDP Proxy ready{bcolors.ENDC}") + print(f" Listening: {host}:{port}") + print(f" Connect: mstsc /v:{host}:{port} " + f"(or wfreerdp /v:{host}:{port} /sec:tls /cert:ignore)") + print(f"{bcolors.OKBLUE} Note: when your RDP client prompts for credentials supply any " + f"value (or the per-session NLA token if NLA is enabled). The proxy substitutes the " + f"credentials configured in your Keeper vault before forwarding to the target." + f"{bcolors.ENDC}") + @staticmethod def _print_keeperdb_proxy_banner(host, port, db_type): suffix = f' ({db_type})' if db_type else '' @@ -746,6 +837,27 @@ def execute(self, params, **kwargs): port = kwargs.get('port') no_trickle_ice = kwargs.get('no_trickle_ice', False) + # Port selection policy (matches the keeper-pam-webrtc-rs local + # listener's SO_EXCLUSIVEADDRUSE behavior on Windows): + # * user passed `--port N` (any N, including 49152) → strict: + # ask Rust for exactly N; if the bind fails (port already + # in use, permission denied, etc.) surface the error and + # don't silently pick a different port. The intent of + # `--port` is "I want exactly this port; if you can't give + # it to me, tell me." + # * no `--port` given → prefer `LEGACY_DEFAULT_TUNNEL_PORT` + # (49152) for backward-compatibility with docs / muscle + # memory that hard-code the connect string. Pre-probe with + # `is_port_open` (which is authoritative on Windows now + # that Rust uses SO_EXCLUSIVEADDRUSE) to avoid the noisy + # "address in use" error path in the common case where + # 49152 is already taken. If the probe says it's free we + # still keep `allow_ephemeral_fallback=True` so the rare + # TOCTOU race (something grabs 49152 between the probe and + # the Rust bind) falls back silently to an ephemeral port + # instead of failing the whole start. + LEGACY_DEFAULT_TUNNEL_PORT = 49152 + allow_ephemeral_fallback = False if port is not None and port > 0: try: port = find_open_port(tried_ports=[], preferred_port=port, host=host) @@ -753,10 +865,12 @@ def execute(self, params, **kwargs): print(f"{bcolors.FAIL}{e}{bcolors.ENDC}") return else: - port = find_open_port(tried_ports=[], host=host) - if port is None: - print(f"{bcolors.FAIL}Could not find open port to use for tunnel{bcolors.ENDC}") - return + probe_host = host or '127.0.0.1' + if is_port_open(probe_host, LEGACY_DEFAULT_TUNNEL_PORT): + port = LEGACY_DEFAULT_TUNNEL_PORT + allow_ephemeral_fallback = True + else: + port = 0 # Sync and validate record api.sync_down(params) @@ -831,48 +945,92 @@ def execute(self, params, **kwargs): pam_settings_value = pam_settings.get_default_value() if pam_settings else {} allow_supply_host = pam_settings_value.get('allowSupplyHost', False) if isinstance(pam_settings_value, dict) else False - # --proxy: KeeperDB Proxy mode (gateway substitutes credentials from vault). - # This is a Commander-side validator/declaration; the gateway currently - # auto-routes pamDatabase + allowKeeperDBProxy to the proxy regardless of - # any client-side flag (see is_keeperdb_proxy_tunnel in dr-controller's - # tunnel_helpers.py and _build_protocol_settings in WebRTCSessionAction.py). - # Requiring no-`--proxy` to mean "raw TCP tunnel to remote host" depends on - # a future gateway change to honor a client-side opt-in flag; until that - # lands, omitting --proxy will still proxy if the record allows it. - is_keeperdb_proxy = bool(kwargs.get('proxy')) + # --proxy: Keeper Proxy mode. Backend is auto-selected from the record + # type (and protocol for pamMachine): + # * pamDatabase -> KeeperDB Proxy (DB credential injection) + # * pamMachine + protocol=rdp -> KeeperRDP Proxy (RDP credential injection) + # Both backends require the corresponding `allowKeeper{DB,RDP}Proxy` + # flag on the record; the gateway auto-routes from that flag alone, + # so the Commander side does *not* round-trip an opt-in to the + # gateway. The `--proxy` flag here is purely a client-side + # declaration that drives our validators (record-type + flag- + # presence + launch-credential preflight) and the post-start + # banner. + is_keeper_proxy = bool(kwargs.get('proxy')) + is_keeperdb_proxy = False + is_keeperrdp_proxy = False db_type_for_banner = None - if is_keeperdb_proxy: + if is_keeper_proxy: record_type = record.record_type - # TODO(rdp-proxy): once RDP Proxy support lands, also accept - # 'pamMachine' here and dispatch by record type. - if record_type != 'pamDatabase': - print(f"{bcolors.FAIL}--proxy is only supported on pamDatabase records. " - f"Record {record_uid} is of type \"{record_type}\".{bcolors.ENDC}") - return - allow_kdb = isinstance(pam_settings_value, dict) and bool( - (pam_settings_value.get('portForward') or {}).get('allowKeeperDBProxy') - or (pam_settings_value.get('connection') or {}).get('allowKeeperDBProxy') - ) - if not allow_kdb: - print(f"{bcolors.FAIL}KeeperDB Proxy is not enabled for record {record_uid}.{bcolors.ENDC}") - print(f"{bcolors.WARNING}Enable it with " - f"{bcolors.OKBLUE}'pam tunnel edit {record_uid} --keeper-db-proxy on'" - f"{bcolors.ENDC}") - return - # Mirror the launch-credential pre-flight from PAMTunnelEditCommand - # (--keeper-db-proxy on path) so the failure message and timing are - # identical between edit and start. - _est, _ett, _tk = get_keeper_tokens(params) - _existing_cfg = get_config_uid(params, _est, _ett, record_uid) - _proxy_dag = TunnelDAG(params, _est, _ett, _existing_cfg, transmission_key=_tk) - if not _proxy_dag.check_if_resource_has_launch_credential(record_uid): - print(f"{bcolors.FAIL}No Launch Credentials assigned to record \"{record_uid}\". " - f"Please assign launch credentials before using --proxy.{bcolors.ENDC}") - print(f"{bcolors.WARNING}Use: " - f"{bcolors.OKBLUE}pam connection edit --launch-user (-lu) " - f"{bcolors.ENDC}") + protocol = self._resolve_connection_protocol(pam_settings_value) + + if record_type == 'pamDatabase': + is_keeperdb_proxy = True + elif record_type == 'pamMachine' and protocol == 'rdp': + is_keeperrdp_proxy = True + else: + detail = f' (protocol="{protocol}")' if record_type == 'pamMachine' else '' + print(f"{bcolors.FAIL}--proxy is supported on pamDatabase or pamMachine+RDP records. " + f"Record {record_uid} is of type \"{record_type}\"{detail}.{bcolors.ENDC}") return - db_type_for_banner = self._resolve_database_type(record, pam_settings_value) + + if is_keeperdb_proxy: + allow_kdb = isinstance(pam_settings_value, dict) and bool( + (pam_settings_value.get('portForward') or {}).get('allowKeeperDBProxy') + or (pam_settings_value.get('connection') or {}).get('allowKeeperDBProxy') + ) + if not allow_kdb: + print(f"{bcolors.FAIL}KeeperDB Proxy is not enabled for record {record_uid}.{bcolors.ENDC}") + print(f"{bcolors.WARNING}Enable it with " + f"{bcolors.OKBLUE}'pam tunnel edit {record_uid} --keeper-db-proxy on'" + f"{bcolors.ENDC}") + return + # Mirror the launch-credential pre-flight from PAMTunnelEditCommand + # (--keeper-db-proxy on path) so the failure message and timing are + # identical between edit and start. + _est, _ett, _tk = get_keeper_tokens(params) + _existing_cfg = get_config_uid(params, _est, _ett, record_uid) + _proxy_dag = TunnelDAG(params, _est, _ett, _existing_cfg, transmission_key=_tk) + if not _proxy_dag.check_if_resource_has_launch_credential(record_uid): + print(f"{bcolors.FAIL}No Launch Credentials assigned to record \"{record_uid}\". " + f"Please assign launch credentials before using --proxy.{bcolors.ENDC}") + print(f"{bcolors.WARNING}Use: " + f"{bcolors.OKBLUE}pam connection edit --launch-user (-lu) " + f"{bcolors.ENDC}") + return + db_type_for_banner = self._resolve_database_type(record, pam_settings_value) + else: + # KeeperRDP Proxy: validate the record flag set via + # record-update -r pamSettings=$JSON:{"portForward":{"allowKeeperRDPProxy":true}} + # Unlike KeeperDB-Proxy's writer/reader bucket mismatch, + # RDP only ever stores this flag in `portForward` — no + # `connection` fallback. + allow_krdp = isinstance(pam_settings_value, dict) and bool( + (pam_settings_value.get('portForward') or {}).get('allowKeeperRDPProxy') + ) + if not allow_krdp: + print(f"{bcolors.FAIL}KeeperRDP Proxy is not enabled for record {record_uid}.{bcolors.ENDC}") + print(f"{bcolors.WARNING}Enable it on the record's pamSettings.portForward bucket: " + f"set {bcolors.OKBLUE}allowKeeperRDPProxy=true{bcolors.WARNING} (e.g. via " + f"{bcolors.OKBLUE}record-update -r {record_uid} --force " + f"pamSettings=$JSON:{{\"portForward\":{{\"allowKeeperRDPProxy\":true}}}}" + f"{bcolors.ENDC})") + return + # Launch-credential pre-flight (same gate KeeperDB uses). + # The gateway resolves the linked pamUser's login/password + # into tunnel_params via _process_user_record, so failing + # without one yields the cryptic + # `gateway_webrtcaction_missing_rdp_credentials`. + _est, _ett, _tk = get_keeper_tokens(params) + _existing_cfg = get_config_uid(params, _est, _ett, record_uid) + _proxy_dag = TunnelDAG(params, _est, _ett, _existing_cfg, transmission_key=_tk) + if not _proxy_dag.check_if_resource_has_launch_credential(record_uid): + print(f"{bcolors.FAIL}No Launch Credentials assigned to record \"{record_uid}\". " + f"Please assign launch credentials before using --proxy.{bcolors.ENDC}") + print(f"{bcolors.WARNING}Use: " + f"{bcolors.OKBLUE}pam connection edit --launch-user (-lu) " + f"{bcolors.ENDC}") + return # Get target host and port if allow_supply_host: @@ -1074,7 +1232,74 @@ def execute(self, params, **kwargs): f"WebRTC cleanup is best-effort.{bcolors.ENDC}") return - result = start_rust_tunnel(params, record_uid, gateway_uid, host, port, seed, target_host, target_port, socks, trickle_ice, record.title, allow_supply_host=allow_supply_host, two_factor_value=two_factor_value) + # When `allow_ephemeral_fallback` is on we're about to attempt + # the legacy default port and the pre-probe said it was free. + # In the rare TOCTOU window the bind can still race and lose, + # in which case Rust (via pyo3-log) and Commander both emit + # noisy ERROR-level lines ("Failed to bind … os error 10048", + # the pyo3 PyBind wrapper, and Commander's "Error in + # start_rust_tunnel"). Suppress ERROR-level logging during + # this first attempt so the user only sees those if the retry + # also fails. Reset before the retry so any second-attempt + # errors surface normally. + if allow_ephemeral_fallback: + logging.disable(logging.ERROR) + try: + result = start_rust_tunnel( + params, record_uid, gateway_uid, host, port, seed, + target_host, target_port, socks, trickle_ice, + record.title, allow_supply_host=allow_supply_host, + two_factor_value=two_factor_value, + ) + finally: + logging.disable(logging.NOTSET) + else: + result = start_rust_tunnel( + params, record_uid, gateway_uid, host, port, seed, + target_host, target_port, socks, trickle_ice, + record.title, allow_supply_host=allow_supply_host, + two_factor_value=two_factor_value, + ) + + # No-`--port` legacy default fallback. If Rust failed to bind the + # preferred LEGACY_DEFAULT_TUNNEL_PORT because something else + # already owns it (concurrent `pam tunnel start`, an unrelated + # service, etc.), retry with port=0 so the OS picks an + # ephemeral port. We let the actual bind decide instead of + # trusting the (now stale) pre-probe. Only triggers when the + # user did not pass `--port`. + if ( + allow_ephemeral_fallback + and (not result or not result.get("success")) + ): + err_text = ((result or {}).get("error") or "").lower() + # Windows reports SO_EXCLUSIVEADDRUSE collisions as + # WSAEACCES (os error 10013, "An attempt was made to access + # a socket in a way forbidden by its access permissions") + # rather than the more familiar WSAEADDRINUSE (10048). + # Unix uses EADDRINUSE / "Address already in use". + looks_like_bind_taken = ( + "address already in use" in err_text + or "wsaeaddrinuse" in err_text + or "wsaeaccess" in err_text + or "access permissions" in err_text + or ("bind" in err_text and "10013" in err_text) + or ("bind" in err_text and "10048" in err_text) + or ("bind" in err_text and "in use" in err_text) + or ("bind" in err_text and "forbidden" in err_text) + ) + if looks_like_bind_taken: + logging.debug( + f"start_rust_tunnel: legacy default port {port} unavailable " + f"({err_text!r}); retrying with ephemeral (port=0)" + ) + port = 0 + result = start_rust_tunnel( + params, record_uid, gateway_uid, host, port, seed, + target_host, target_port, socks, trickle_ice, + record.title, allow_supply_host=allow_supply_host, + two_factor_value=two_factor_value, + ) if result and result.get("success"): # When --proxy was used, print the KeeperDB Proxy info banner once. @@ -1084,8 +1309,18 @@ def execute(self, params, **kwargs): # Single call covers interactive, foreground, run, and background- # child modes — the background parent returns earlier and never # reaches this branch. + # + # Use the actually-bound `local_host`/`local_port` from Rust + # (may differ from the requested CLI args when the requested + # port was already in use and Rust fell back to ephemeral — + # otherwise repeat `pam tunnel start` invocations print the + # same `:port` even though only the first one is reachable). + banner_host = result.get("local_host") or host + banner_port = result.get("local_port") or port if is_keeperdb_proxy: - self._print_keeperdb_proxy_banner(host, port, db_type_for_banner) + self._print_keeperdb_proxy_banner(banner_host, banner_port, db_type_for_banner) + elif is_keeperrdp_proxy: + self._print_keeperrdp_proxy_banner(banner_host, banner_port) # Workflow lease expiry handling. # # At expiresOn we close the tube (stops new channels, sends diff --git a/keepercommander/commands/universalsecretsync.py b/keepercommander/commands/universalsecretsync.py index 5bbda109e..157e84b04 100644 --- a/keepercommander/commands/universalsecretsync.py +++ b/keepercommander/commands/universalsecretsync.py @@ -79,7 +79,7 @@ def print_uss_configurations_list(params, format_type='table'): # Only process these specific configuration types uss_supported_types = ('pamGcpConfiguration', 'pamAzureConfiguration', 'pamAwsConfiguration', - 'pamGitHubConfiguration') + 'pamGitHubConfiguration', 'pamHashiCorpConfiguration') configs_data = [] for record in configurations: @@ -115,6 +115,21 @@ def print_uss_configurations_list(params, format_type='table'): # GitHub-specific fields are nested under the 'github' key. github_data = config_data.get('github') or {} + # HashiCorp-specific fields are nested under the 'hashicorp' key. + hashicorp_data = config_data.get('hashicorp') or {} + + # Decrypt HashiCorp vault_base_url if present + vault_base_url = 'N/A' + vault_base_url_encrypted = hashicorp_data.get('vaultBaseUrl') + if vault_base_url_encrypted: + try: + vault_base_url_bytes = crypto.decrypt_aes_v2( + utils.base64_url_decode(vault_base_url_encrypted), record.record_key) + vault_base_url = vault_base_url_bytes.decode('utf-8') + except Exception as e: + logging.debug(f"Failed to decrypt vault_base_url for record {record.record_uid}: {e}") + vault_base_url = 'N/A' + # Decrypt vault_name if present. The router stores it under the # 'vaultName' key as a base64-url string of the encrypted bytes. vault_name = 'N/A' @@ -128,6 +143,19 @@ def print_uss_configurations_list(params, format_type='table'): logging.debug(f"Failed to decrypt vault_name for record {record.record_uid}: {e}") vault_name = 'N/A' + # Decrypt object_storage_bucket if present. The router stores it under + # the 'objectStorageBucket' key as a base64-url string of the encrypted bytes. + object_storage_bucket = 'N/A' + object_storage_bucket_encrypted = config_data.get('objectStorageBucket') + if object_storage_bucket_encrypted: + try: + object_storage_bucket_bytes = crypto.decrypt_aes_v2( + utils.base64_url_decode(object_storage_bucket_encrypted), record.record_key) + object_storage_bucket = object_storage_bucket_bytes.decode('utf-8') + except Exception as e: + logging.debug(f"Failed to decrypt object_storage_bucket for record {record.record_uid}: {e}") + object_storage_bucket = 'N/A' + # Decrypt sync_identity if present. The router stores it under the # 'syncIdentity' key as a base64-url string of the encrypted bytes; the # decrypted value is the UID of the Identity record used for syncing. @@ -201,11 +229,13 @@ def print_uss_configurations_list(params, format_type='table'): 'dry_run_enabled': config_data.get('dryRunEnabled', False), 'folder_count': folder_count, 'vault_name': vault_name, + 'object_storage_bucket': object_storage_bucket, 'sync_identity': sync_identity, 'scope': scope_str, 'owner': owner, 'organization_visibility': org_visibility_str, 'repos': repo_names, + 'vault_base_url': vault_base_url, }) except Exception as e: # Skip records that fail to load or don't have USS config @@ -226,8 +256,9 @@ def print_uss_configurations_list(params, format_type='table'): # Display as simple summary table table = [] - headers = ['Network UID', 'Title', 'Type', 'Enabled', 'Dry Run', 'Folders', 'Vault Name', 'Sync Identity', - 'Scope', 'Owner', 'Org Visibility', 'Repos'] + headers = ['Network UID', 'Title', 'Type', 'Enabled', 'Dry Run', 'Folders', 'Vault Name', + 'Object Storage Bucket', 'Sync Identity', 'Scope', 'Owner', 'Org Visibility', 'Repos', + 'Vault Base URL'] for config in configs_data: enabled_str = f"{bcolors.OKGREEN}Yes{bcolors.ENDC}" if config['enabled'] else f"{bcolors.FAIL}No{bcolors.ENDC}" @@ -245,11 +276,13 @@ def print_uss_configurations_list(params, format_type='table'): dry_run_str, folders_str, config['vault_name'], + config.get('object_storage_bucket', 'N/A'), config['sync_identity'], config.get('scope', 'N/A'), config.get('owner', 'N/A'), config.get('organization_visibility', 'N/A'), - repos_str + repos_str, + config.get('vault_base_url', 'N/A') ] table.append(row) @@ -273,7 +306,7 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): # Check if it's a supported USS configuration type uss_supported_types = ('pamGcpConfiguration', 'pamAzureConfiguration', 'pamAwsConfiguration', - 'pamGitHubConfiguration') + 'pamGitHubConfiguration', 'pamHashiCorpConfiguration') if not isinstance(network, vault.TypedRecord) or network.record_type not in uss_supported_types: if format_type == 'json': return json.dumps({"error": f'Record "{network_uid}" is not a USS configuration'}) @@ -310,6 +343,21 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): # GitHub-specific fields are nested under the 'github' key. github_data = config_data.get('github') or {} + # HashiCorp-specific fields are nested under the 'hashicorp' key. + hashicorp_data = config_data.get('hashicorp') or {} + + # Decrypt HashiCorp vault_base_url if present + vault_base_url = 'N/A' + vault_base_url_encrypted = hashicorp_data.get('vaultBaseUrl') + if vault_base_url_encrypted: + try: + vault_base_url_bytes = crypto.decrypt_aes_v2( + utils.base64_url_decode(vault_base_url_encrypted), network.record_key) + vault_base_url = vault_base_url_bytes.decode('utf-8') + except Exception as e: + logging.debug(f"Failed to decrypt vault_base_url for network {network.record_uid}: {e}") + vault_base_url = 'N/A' + # Decrypt vault_name if present. The router stores it under the # 'vaultName' key as a base64-url string of the encrypted bytes. vault_name = 'N/A' @@ -323,6 +371,19 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): logging.debug(f"Failed to decrypt vault_name for network {network.record_uid}: {e}") vault_name = 'N/A' + # Decrypt object_storage_bucket if present. The router stores it under + # the 'objectStorageBucket' key as a base64-url string of the encrypted bytes. + object_storage_bucket = 'N/A' + object_storage_bucket_encrypted = config_data.get('objectStorageBucket') + if object_storage_bucket_encrypted: + try: + object_storage_bucket_bytes = crypto.decrypt_aes_v2( + utils.base64_url_decode(object_storage_bucket_encrypted), network.record_key) + object_storage_bucket = object_storage_bucket_bytes.decode('utf-8') + except Exception as e: + logging.debug(f"Failed to decrypt object_storage_bucket for network {network.record_uid}: {e}") + object_storage_bucket = 'N/A' + # Decrypt sync_identity if present. The router stores it under the # 'syncIdentity' key as a base64-url string of the encrypted bytes; the # decrypted value is the UID of the Identity record used for syncing. @@ -433,11 +494,13 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): 'enabled': config_data.get('enabled', False), 'dry_run_enabled': config_data.get('dryRunEnabled', False), 'vault_name': vault_name, + 'object_storage_bucket': object_storage_bucket, 'sync_identity': sync_identity, 'scope': scope_str, 'owner': owner, 'organization_visibility': org_visibility_str, 'repos': repo_names, + 'vault_base_url': vault_base_url, 'folders': [] } @@ -468,11 +531,13 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): table.append(['Enabled', 'Yes' if config_data.get('enabled', False) else 'No']) table.append(['Dry Run', 'Yes' if config_data.get('dryRunEnabled', False) else 'No']) table.append(['Vault Name', vault_name]) + table.append(['Object Storage Bucket', object_storage_bucket]) table.append(['Sync Identity', sync_identity]) table.append(['Scope', scope_str]) table.append(['Owner', owner]) table.append(['Org Visibility', org_visibility_str]) table.append(['Repos', ', '.join(repo_names) if repo_names else 'None']) + table.append(['Vault Base URL', vault_base_url]) table.append(['', '']) # Blank row separator # Display folder sync details @@ -521,6 +586,7 @@ def print_uss_configuration_details(params, network_uid, format_type='table'): class PAMUniversalSyncConfigAddCommand(Command): parser = argparse.ArgumentParser(prog='pam universal-sync-config add') + parser.add_argument('--network', '-n', required=True, dest='network', action='store', help='Network UID or name to configure universal sync') parser.add_argument('--enabled', '-e', dest='enabled', action='store', @@ -533,6 +599,8 @@ class PAMUniversalSyncConfigAddCommand(Command): help='Identity record UID to use for syncing') parser.add_argument('--vault-name', '-vn', dest='vault_name', action='store', help='Vault name for universal sync') + parser.add_argument('--object-storage-bucket', '-osb', dest='object_storage_bucket', action='store', + help='Object storage bucket for storing file reference content') parser.add_argument('--scope', '-sc', dest='scope', action='store', choices=['repository', 'organization'], help='GitHub sync scope: a single repository or an entire organization') @@ -543,6 +611,14 @@ class PAMUniversalSyncConfigAddCommand(Command): help='Repository visibility to sync when scope is organization') parser.add_argument('--repo', '-r', dest='repo', action='append', help='GitHub repository name to sync (can be specified multiple times; scope must be repository)') + parser.add_argument('--vault-base-url', '-vbu', dest='vault_base_url', action='store', + help='HashiCorp Vault Base URL (e.g., https://vault.company.com:8200)') + parser.add_argument('--vault-token', '-vt', dest='vault_token', action='store', + help='HashiCorp Vault Token (optional; syncIdentity takes precedence)') + parser.add_argument('--vault-namespace', '-vns', dest='vault_namespace', action='store', + help='HashiCorp Vault Namespace (optional; leave blank for Community Edition)') + parser.add_argument('--vault-mount-path', '-vmp', dest='vault_mount_path', action='store', + help='HashiCorp Vault KV Mount Path (optional; defaults to "secret")') def get_parser(self): return PAMUniversalSyncConfigAddCommand.parser @@ -593,6 +669,12 @@ def execute(self, params, **kwargs): encrypted_vault_name = crypto.encrypt_aes_v2(vault_name_bytes, network.record_key) rq.vaultName = encrypted_vault_name + object_storage_bucket = kwargs.get('object_storage_bucket') + if object_storage_bucket: + object_storage_bucket_bytes = string_to_bytes(object_storage_bucket) + encrypted_object_storage_bucket = crypto.encrypt_aes_v2(object_storage_bucket_bytes, network.record_key) + rq.objectStorageBucket = encrypted_object_storage_bucket + scope = kwargs.get('scope') if scope is not None: rq.github.scope = pam_pb2.ORGANIZATION if scope == 'organization' else pam_pb2.REPOSITORY @@ -616,6 +698,30 @@ def execute(self, params, **kwargs): repo_obj.name = crypto.encrypt_aes_v2(repo_bytes, network.record_key) rq.github.repos.append(repo_obj) + vault_base_url = kwargs.get('vault_base_url') + if vault_base_url: + vault_base_url_bytes = string_to_bytes(vault_base_url) + encrypted_vault_base_url = crypto.encrypt_aes_v2(vault_base_url_bytes, network.record_key) + rq.hashicorp.vaultBaseUrl = encrypted_vault_base_url + + vault_token = kwargs.get('vault_token') + if vault_token: + vault_token_bytes = string_to_bytes(vault_token) + encrypted_vault_token = crypto.encrypt_aes_v2(vault_token_bytes, network.record_key) + rq.hashicorp.vaultToken = encrypted_vault_token + + vault_namespace = kwargs.get('vault_namespace') + if vault_namespace: + vault_namespace_bytes = string_to_bytes(vault_namespace) + encrypted_vault_namespace = crypto.encrypt_aes_v2(vault_namespace_bytes, network.record_key) + rq.hashicorp.vaultNamespace = encrypted_vault_namespace + + vault_mount_path = kwargs.get('vault_mount_path') + if vault_mount_path: + vault_mount_path_bytes = string_to_bytes(vault_mount_path) + encrypted_vault_mount_path = crypto.encrypt_aes_v2(vault_mount_path_bytes, network.record_key) + rq.hashicorp.vaultMountPath = encrypted_vault_mount_path + encrypted_session_token, encrypted_transmission_key, transmission_key = get_keeper_tokens(params) try: @@ -641,6 +747,8 @@ class PAMUniversalSyncConfigEditCommand(Command): help='Identity record UID to use for syncing') parser.add_argument('--vault-name', '-vn', dest='vault_name', action='store', help='Vault name for universal sync') + parser.add_argument('--object-storage-bucket', '-osb', dest='object_storage_bucket', action='store', + help='Object storage bucket for storing file reference content') parser.add_argument('--scope', '-sc', dest='scope', action='store', choices=['repository', 'organization'], help='GitHub sync scope: a single repository or an entire organization') @@ -651,6 +759,14 @@ class PAMUniversalSyncConfigEditCommand(Command): help='Repository visibility to sync when scope is organization') parser.add_argument('--repo', '-r', dest='repo', action='append', help='GitHub repository name to sync (can be specified multiple times; scope must be repository)') + parser.add_argument('--vault-base-url', '-vbu', dest='vault_base_url', action='store', + help='HashiCorp Vault Base URL (e.g., https://vault.company.com:8200)') + parser.add_argument('--vault-token', '-vt', dest='vault_token', action='store', + help='HashiCorp Vault Token (optional; syncIdentity takes precedence)') + parser.add_argument('--vault-namespace', '-vns', dest='vault_namespace', action='store', + help='HashiCorp Vault Namespace (optional; leave blank for Community Edition)') + parser.add_argument('--vault-mount-path', '-vmp', dest='vault_mount_path', action='store', + help='HashiCorp Vault KV Mount Path (optional; defaults to "secret")') def get_parser(self): return PAMUniversalSyncConfigEditCommand.parser @@ -747,6 +863,14 @@ def execute(self, params, **kwargs): elif existing_config.get('vaultName'): rq.vaultName = utils.base64_url_decode(existing_config['vaultName']) + object_storage_bucket = kwargs.get('object_storage_bucket') + if object_storage_bucket: + object_storage_bucket_bytes = string_to_bytes(object_storage_bucket) + encrypted_object_storage_bucket = crypto.encrypt_aes_v2(object_storage_bucket_bytes, network.record_key) + rq.objectStorageBucket = encrypted_object_storage_bucket + elif existing_config.get('objectStorageBucket'): + rq.objectStorageBucket = utils.base64_url_decode(existing_config['objectStorageBucket']) + # GitHub-specific fields live under the nested 'github' object, both in the # request message (rq.github) and in the existing DAG edge content. existing_github = existing_config.get('github') or {} @@ -786,6 +910,41 @@ def execute(self, params, **kwargs): repo_obj.name = utils.base64_url_decode(existing_repo) rq.github.repos.append(repo_obj) + # HashiCorp-specific fields live under the nested 'hashicorp' object + existing_hashicorp = existing_config.get('hashicorp') or {} + + vault_base_url = kwargs.get('vault_base_url') + if vault_base_url: + vault_base_url_bytes = string_to_bytes(vault_base_url) + encrypted_vault_base_url = crypto.encrypt_aes_v2(vault_base_url_bytes, network.record_key) + rq.hashicorp.vaultBaseUrl = encrypted_vault_base_url + elif existing_hashicorp.get('vaultBaseUrl'): + rq.hashicorp.vaultBaseUrl = utils.base64_url_decode(existing_hashicorp['vaultBaseUrl']) + + vault_token = kwargs.get('vault_token') + if vault_token: + vault_token_bytes = string_to_bytes(vault_token) + encrypted_vault_token = crypto.encrypt_aes_v2(vault_token_bytes, network.record_key) + rq.hashicorp.vaultToken = encrypted_vault_token + elif existing_hashicorp.get('vaultToken'): + rq.hashicorp.vaultToken = utils.base64_url_decode(existing_hashicorp['vaultToken']) + + vault_namespace = kwargs.get('vault_namespace') + if vault_namespace: + vault_namespace_bytes = string_to_bytes(vault_namespace) + encrypted_vault_namespace = crypto.encrypt_aes_v2(vault_namespace_bytes, network.record_key) + rq.hashicorp.vaultNamespace = encrypted_vault_namespace + elif existing_hashicorp.get('vaultNamespace'): + rq.hashicorp.vaultNamespace = utils.base64_url_decode(existing_hashicorp['vaultNamespace']) + + vault_mount_path = kwargs.get('vault_mount_path') + if vault_mount_path: + vault_mount_path_bytes = string_to_bytes(vault_mount_path) + encrypted_vault_mount_path = crypto.encrypt_aes_v2(vault_mount_path_bytes, network.record_key) + rq.hashicorp.vaultMountPath = encrypted_vault_mount_path + elif existing_hashicorp.get('vaultMountPath'): + rq.hashicorp.vaultMountPath = utils.base64_url_decode(existing_hashicorp['vaultMountPath']) + encrypted_session_token, encrypted_transmission_key, transmission_key = get_keeper_tokens(params) try: diff --git a/keepercommander/commands/utils.py b/keepercommander/commands/utils.py index 0cfb4a0e4..3be980c95 100644 --- a/keepercommander/commands/utils.py +++ b/keepercommander/commands/utils.py @@ -45,7 +45,7 @@ from ..error import CommandError from ..generator import ( KeeperPasswordGenerator, DicewarePasswordGenerator, CryptoPassphraseGenerator, - KeeperPassphraseGenerator, PASSPHRASE_SEPARATOR_HELP, + KeeperPassphraseGenerator, AESKeyGenerator, PASSPHRASE_SEPARATOR_HELP, _parse_passphrase_separator_token ) from ..params import KeeperParams, LAST_RECORD_UID, LAST_FOLDER_UID, LAST_SHARED_FOLDER_UID from ..proto import ssocloud_pb2, enterprise_pb2, APIRequest_pb2 @@ -81,7 +81,6 @@ def add(self, other): self.failed += other.failed self.vault_changed = self.vault_changed or other.vault_changed - class DeletionValidator: """Centralized validation for deletion operations.""" @@ -441,7 +440,10 @@ def register_command_info(aliases, command_info): help='Optional. File path to word list') crypto_group = generate_parser.add_argument_group('Crypto') -crypto_group.add_argument('--crypto', dest='crypto', action='store_true', help='Generate crypto wallet passphrase') +crypto_group.add_argument('--crypto', dest='crypto', action='store_true', help='Generate crypto wallet private key') + +aes_key_group = generate_parser.add_argument_group('AES 256-bit Key') +aes_key_group.add_argument('--aes-key', dest='aes_key', action='store_true', help='AES 256 bit encryption key. Base64 URL safe encoded') reset_password_parser = argparse.ArgumentParser(prog='reset-password', description='Reset Master Password') reset_password_parser.add_argument('--delete-sso', dest='delete_alternate', action='store_true', @@ -521,6 +523,8 @@ def execute(self, params, **kwargs): class ThisDeviceCommand(Command): + skip_sync_on_auth = True + def get_parser(self): return this_device_parser @@ -2237,13 +2241,15 @@ def execute(self, params, number=None, no_breachwatch=None, if kwargs.get('crypto') is True: kpg = CryptoPassphraseGenerator() + elif kwargs.get('aes_key') is True: + kpg = AESKeyGenerator() elif kwargs.get('passphrase') is True: from ..enforcement import PasswordComplexityEnforcer policy = PasswordComplexityEnforcer.get_policy(params) word_count = length if length != 20 else None pp_separator = kwargs.get('pp_separator') if isinstance(pp_separator, str) and pp_separator.strip(): - pp_separator, pp_sep_error = generator._parse_passphrase_separator_token( + pp_separator, pp_sep_error = _parse_passphrase_separator_token( pp_separator.strip()) if pp_sep_error: logging.error(pp_sep_error) diff --git a/keepercommander/generator.py b/keepercommander/generator.py index 14955376e..ad744b177 100644 --- a/keepercommander/generator.py +++ b/keepercommander/generator.py @@ -19,7 +19,7 @@ from typing import Optional, List, Iterator, Sequence, Tuple from collections import namedtuple -from . import crypto +from . import crypto, utils from Cryptodome.Random.random import shuffle @@ -478,6 +478,11 @@ def create_from_policy(cls, policy, length_override=None, separator_override=Non ) +class AESKeyGenerator(PasswordGenerator): + def generate(self): + return utils.base64_url_encode(utils.generate_aes_key()) + + class CryptoPassphraseGenerator(PasswordGenerator): def __init__(self): self._vocabulary = None # type: Optional[List[str]] diff --git a/keepercommander/importer/commands.py b/keepercommander/importer/commands.py index 223f5525b..83f2d75ed 100644 --- a/keepercommander/importer/commands.py +++ b/keepercommander/importer/commands.py @@ -155,6 +155,7 @@ def _cyberark_skip_arg(value): apply_membership_parser = argparse.ArgumentParser(prog='apply-membership', description='Loads shared folder membership from JSON file') apply_membership_parser.add_argument('--full-sync', dest='full_sync', action='store_true', help='Update and remove membership also.') +apply_membership_parser.add_argument('--unsafe', dest='unsafe', action='store_true', help='Combined with --full-sync. Will remove yourself from folders where you have no permission.') apply_membership_parser.add_argument('name', type=str, nargs='?', help='Input file name. "shared_folder_membership.json" if omitted.') apply_membership_parser.error = raise_parse_exception apply_membership_parser.exit = suppress_exit @@ -547,8 +548,9 @@ def execute(self, params, **kwargs): teams.append(obj) full_sync = kwargs.get('full_sync') is True + unsafe = kwargs.get('unsafe') is True if len(shared_folders) > 0: - imp_exp.import_user_permissions(params, shared_folders, full_sync) + imp_exp.import_user_permissions(params, shared_folders, full_sync, unsafe) if len(teams) > 0: imp_exp.import_teams(params, teams, full_sync) diff --git a/keepercommander/importer/cyberark/pam/__init__.py b/keepercommander/importer/cyberark/pam/__init__.py index 21bbfdf4e..abb4c951e 100644 --- a/keepercommander/importer/cyberark/pam/__init__.py +++ b/keepercommander/importer/cyberark/pam/__init__.py @@ -20,6 +20,7 @@ RECORD_TYPE_PAM_DATABASE, RECORD_TYPE_PAM_DIRECTORY, RECORD_TYPE_PAM_MACHINE, + RECORD_TYPE_PAM_REMOTE_BROWSER, ROTATION_UNMAPPED, SCHEDULE_ON_DEMAND, SYSTEM_SAFES, @@ -101,6 +102,7 @@ "RECORD_TYPE_PAM_DATABASE", "RECORD_TYPE_PAM_DIRECTORY", "RECORD_TYPE_PAM_MACHINE", + "RECORD_TYPE_PAM_REMOTE_BROWSER", "ROTATION_UNMAPPED", "SCHEDULE_ON_DEMAND", "IDENTITY_LOGIN_SUCCESS", diff --git a/keepercommander/importer/cyberark/pam/account_mapper.py b/keepercommander/importer/cyberark/pam/account_mapper.py index 68de82b8a..7eeadaa0f 100644 --- a/keepercommander/importer/cyberark/pam/account_mapper.py +++ b/keepercommander/importer/cyberark/pam/account_mapper.py @@ -1,4 +1,4 @@ -# _ __ +# _ __ # | |/ /___ ___ _ __ ___ _ _ ® # | ' resolved schedule (or # ``None`` once we've decided the platform inherits the master # policy default). One CyberArk API call per unique platformId. self._platform_schedule_cache: Dict[str, Optional[dict]] = {} - # Diagnostic counter — how many accounts received a platform-level + # Diagnostic counter - how many accounts received a platform-level # schedule vs. inherited the master default. Surfaced in the # cyberark_import.py orchestrator output so operators can verify # the override was applied. self.platform_schedule_overrides: Dict[str, int] = {} - # Per-platform session-recording overrides. Maps platformId → + # Per-platform session-recording overrides. Maps platformId -> # ("on"|"off"|None) for graphical session recording. ``None`` - # means we couldn't determine the policy — caller falls back to + # means we couldn't determine the policy - caller falls back to # whatever the master policy / Keeper default dictates. Only one # API call per unique platformId thanks to caching. self._platform_session_cache: Dict[str, Optional[Tuple[str, str]]] = {} @@ -141,7 +145,7 @@ def __init__(self, platform_map_override: Optional[dict] = None, # on every Keeper PAM resource record from the same platform so # the source-system context is preserved post-migration. self._platform_metadata_cache: Dict[str, List[dict]] = {} - # Per-platform CyberArk passwordGenRules → Keeper + # Per-platform CyberArk passwordGenRules -> Keeper # ``rotation_settings.password_complexity`` translation cache. # Populated lazily on first hit of each platformId. self._platform_complexity_cache: Dict[str, Optional[str]] = {} @@ -161,6 +165,164 @@ def __init__(self, platform_map_override: Optional[dict] = None, "PerformPeriodicChange", "rotatePasswordsAutomatically", ) + # These values are already represented by canonical login fields. + _LOGIN_METADATA_TOP_LEVEL_EXCLUSIONS = frozenset({ + "id", "name", "platformid", "address", "username", "safename", + "platformaccountproperties", + }) + _LOGIN_METADATA_PROPERTY_EXCLUSIONS = frozenset({ + "url", "itemname", "logondomain", + "consoleurl", "portalurl", "loginurl", + }) + # Retrieved credentials belong only in the Keeper password field. These + # exact key names guard against an API variant embedding secret material + # in the account payload while allowing safe metadata such as secretType + # and secretManagement to be retained. + _LOGIN_METADATA_SECRET_KEYS = frozenset({ + "password", "secret", "content", "credential", "credentials", + "privatekey", "privatepemkey", "sshkey", + }) + _CLOUD_RBI_DEFAULTS = { + "aws": { + "url": "https://signin.aws.amazon.com/console", + "page": "*.signin.aws.amazon.com", + "username": "#username", + "password": "#password", + "submit": "#signin_button", + }, + "azure": { + "url": "https://portal.azure.com/", + "page": "*.login.microsoftonline.com", + "username": "input[type=email], input[name=loginfmt]", + "password": "input[type=password], input[name=passwd]", + "submit": "input[type=submit], button[type=submit]", + }, + "gcp": { + "url": "https://console.cloud.google.com/", + "page": "accounts.google.com", + "username": "input[type=email], input[name=identifier]", + "password": "input[type=password], input[name=password]", + "submit": "button[type=submit]", + }, + } + + @classmethod + def _cloud_rbi_defaults(cls, cloud: str) -> dict: + return dict(cls._CLOUD_RBI_DEFAULTS.get((cloud or "").lower(), {})) + + @classmethod + def _build_cloud_autofill_targets(cls, cloud: str, url: str) -> str: + defaults = cls._cloud_rbi_defaults(cloud) + page = defaults.get("page") or url or "*" + step = {"page": page} + if defaults.get("username"): + step["username-field"] = defaults["username"] + if defaults.get("password"): + step["password-field"] = defaults["password"] + if defaults.get("submit"): + step["submit"] = defaults["submit"] + return json.dumps([step]) + + @staticmethod + def _first_prop(props: dict, *names: str) -> str: + if not isinstance(props, dict): + return "" + lowered = {str(k).casefold(): k for k in props.keys()} + for name in names: + key = lowered.get(str(name).casefold()) + if key and props.get(key) not in (None, ""): + return str(props[key]).strip() + return "" + + @classmethod + def _metadata_label_to_text(cls, label: str) -> str: + parts = [] + for part in str(label).replace("_", " ").split("."): + part = re.sub(r"(?<=[A-Z])(?=[A-Z][a-z])", " ", part) + part = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", part) + parts.append(" ".join(part.split())) + return ".".join(parts) + + @staticmethod + def _metadata_value_to_text(value) -> str: + if isinstance(value, str): + return value + return json.dumps( + value, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), + ) + + @classmethod + def _build_account_metadata_custom_fields(cls, account: dict) -> List[dict]: + """Flatten CyberArk metadata in the classic importer custom-field style.""" + out: List[dict] = [] + if not isinstance(account, dict): + return out + + properties = account.get("platformAccountProperties") + properties = dict(properties) if isinstance(properties, dict) else {} + for key in ("platformName", "platformId"): + value = account.get(key) + if value not in (None, ""): + properties.setdefault("Platform Name", value) + break + for key in ("deviceType", "device type"): + value = account.get(key) + if value not in (None, ""): + properties.setdefault("Device Type", value) + break + + existing_labels = set() + + def _add(path: str, value) -> None: + if len(out) >= MAX_ACCOUNT_METADATA_FIELDS or value in (None, "", [], {}): + return + if isinstance(value, dict): + for child_key, child_value in value.items(): + child_name = str(child_key) + if child_name.casefold() in cls._LOGIN_METADATA_SECRET_KEYS: + continue + _add(f"{path}.{child_name}", child_value) + return + if isinstance(value, (list, tuple)): + text = cls._metadata_value_to_text(list(value)) + elif isinstance(value, bool): + text = "true" if value else "false" + else: + text = cls._metadata_value_to_text(value) + text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', text) + if len(text) > MAX_ACCOUNT_METADATA_VALUE_LEN: + text = text[:MAX_ACCOUNT_METADATA_VALUE_LEN] + label = cls._metadata_label_to_text(path) + label_key = label.casefold() + if not label or label_key in existing_labels: + return + out.append({ + "type": "text", + "label": label[:120], + "value": [text], + }) + existing_labels.add(label_key) + + for key, value in properties.items(): + key_name = str(key) + folded = key_name.casefold() + if (folded in cls._LOGIN_METADATA_PROPERTY_EXCLUSIONS + or folded in cls._LOGIN_METADATA_SECRET_KEYS): + continue + _add(key_name, value) + return out + + @classmethod + def _build_unmapped_login_custom_fields(cls, account: dict) -> List[dict]: + return cls._build_account_metadata_custom_fields(account) + + def _add_source_metadata_to_user(self, user_record: dict, account: dict, platform_id: str) -> None: + """Copy CyberArk source metadata onto the generated pamUser record.""" + account_metadata_fields = self._build_account_metadata_custom_fields(account) + if account_metadata_fields: + user_record.setdefault("custom", []).extend(account_metadata_fields) + @staticmethod def _flatten_rotation_policy(data: dict) -> Dict[str, Any]: """Flatten the platform rotation-policy payload to a single dict. @@ -212,10 +374,10 @@ def _flatten_rotation_policy(data: dict) -> Dict[str, Any]: @staticmethod def _schedule_from_change_block(change: dict) -> Optional[dict]: - """Translate ISPSS ``change`` group → Keeper ``schedule`` dict. + """Translate ISPSS ``change`` group -> Keeper ``schedule`` dict. Mirrors how the Master Policy itself is converted - (``MasterPolicyMapper.days_to_cron``) — the interval value alone + (``MasterPolicyMapper.days_to_cron``) - the interval value alone drives the cadence. CyberArk's ``allowedPeriodic`` / ``performPeriodicChange`` bookkeeping flag is informational (it reflects whether *CyberArk's* CPM auto-triggers the change) and is @@ -247,29 +409,29 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: Calls ``/api/platforms/{platformId}/rotation-policy/`` once per platform and translates the response into a Quartz CRON via ``MasterPolicyMapper.days_to_cron``. Returns ``None`` when the - platform has no custom policy — callers fall back to the master + platform has no custom policy - callers fall back to the master policy default schedule. Honors CyberArk's cascade: - 1. **Per-platform with override** — a recognized exception flag + 1. **Per-platform with override** - a recognized exception flag (``overridesMasterPolicy``, ``isException``, etc.) is true, OR no flag is present but the interval differs from the Master - Policy's own interval → use ``change.interval`` verbatim, + Policy's own interval -> use ``change.interval`` verbatim, converted to CRON exactly like the Master Policy default is - (``allowedPeriodic`` does NOT force on-demand — see + (``allowedPeriodic`` does NOT force on-demand - see ``_schedule_from_change_block``). - 2. **Master-policy exception** — platform listed in + 2. **Master-policy exception** - platform listed in ``/api/platforms/master-rotation-policy/exceptions/`` with a custom change interval (common for Win Local Admins and other platforms that inherit master but have a Master Policy - exception) — used when the per-platform flag says "no + exception) - used when the per-platform flag says "no override" but this separate bulk endpoint disagrees. - 3. **Per-platform without override** — + 3. **Per-platform without override** - no exception detected by (1) or (2): inherit the master - policy's own schedule (``None`` → caller applies + policy's own schedule (``None`` -> caller applies ``default_rotation_schedule``). - 4. **Legacy flat shapes** — older PVWA responses with + 4. **Legacy flat shapes** - older PVWA responses with ``rotateEveryXDays`` / ``passwordChangeDays`` at top level. """ if not platform_id or not self._client: @@ -280,7 +442,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: raw = self._client.fetch_platform_rotation_policy(platform_id) if not raw: msg = ( - f"Platform '{platform_id}' rotation-policy not accessible — " + f"Platform '{platform_id}' rotation-policy not accessible - " "inheriting master policy default." ) if self.strict_policies: @@ -290,7 +452,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: self._platform_schedule_cache[platform_id] = None return None - # ── New ISPSS shape ──────────────────────────────────── + # -- New ISPSS shape ------------------------------------ change = raw.get("change") if isinstance(raw.get("change"), dict) else None if change is not None: try: @@ -302,7 +464,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: # platform's own interval differs from the Master Policy's # configured interval, CyberArk is evidently applying a # platform-specific value regardless of what this tenant calls - # the flag — honor the number it actually returned. + # the flag - honor the number it actually returned. interval_mismatch = ( overrides is None and self._master_change_days > 0 @@ -338,7 +500,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: platform_id, schedule, ) return schedule - # No override, no separate master exception on record — + # No override, no separate master exception on record - # this platform's cadence matches the Master Policy default. # Inherit it (the caller applies ``default_rotation_schedule``, # itself a CRON built from the same master interval), rather @@ -351,7 +513,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: self._platform_schedule_cache[platform_id] = None return None - # overrides is None — this tenant's rotation-policy response + # overrides is None - this tenant's rotation-policy response # doesn't carry any recognized exception flag at all. When we # know the Master Policy's own interval, use it as the source # of truth: a matching interval means no exception exists (the @@ -360,13 +522,13 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: if self._master_change_days > 0: logging.info( "Platform '%s' rotation policy: interval matches master " - "(%d days) — inheriting master policy", platform_id, + "(%d days) - inheriting master policy", platform_id, self._master_change_days, ) self._platform_schedule_cache[platform_id] = None return None - # No flag and no master baseline to compare against — fall back + # No flag and no master baseline to compare against - fall back # to treating the platform's own ``change`` block as authoritative # (legacy behavior for tenants where this is the only info we get). schedule = self._schedule_from_change_block(change) @@ -377,7 +539,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: ) return schedule - # No ``change`` block — check master-policy exceptions before legacy. + # No ``change`` block - check master-policy exceptions before legacy. exc = self._master_exception_schedules.get(platform_id) if exc is not None: schedule = copy.deepcopy(exc) @@ -388,7 +550,7 @@ def _resolve_platform_schedule(self, platform_id: str) -> Optional[dict]: ) return schedule - # ── Legacy flat shapes (PVWA Self-Hosted, Get-Platforms) ─── + # -- Legacy flat shapes (PVWA Self-Hosted, Get-Platforms) --- flat = self._flatten_rotation_policy(raw) days = 0 for key in self._PLATFORM_ROTATE_DAYS_KEYS: @@ -518,7 +680,7 @@ def _add(label: str, val): change = rotation_raw.get("change") if isinstance(rotation_raw.get("change"), dict) else {} verify = rotation_raw.get("verify") if isinstance(rotation_raw.get("verify"), dict) else {} - # Rotation cadence operational settings — preserved on the + # Rotation cadence operational settings - preserved on the # Keeper record so a user can reproduce them in CyberArk. _add("CyberArk Rotation Interval (days)", change.get("interval")) if change.get("allowedPeriodic") is not None: @@ -562,12 +724,12 @@ def _resolve_platform_session_recording( self, platform_id: str) -> Optional[Tuple[str, str]]: """Return ``("on"|"off", "on"|"off")`` for (graphical, text) recording. - Probes the per-platform endpoints in order (secrets-policy → - platform details → legacy .asmx). When the platform exposes a + Probes the per-platform endpoints in order (secrets-policy -> + platform details -> legacy .asmx). When the platform exposes a ``recordAndSaveSessionActivity`` rule we honor it; that single boolean drives both Keeper recording flags because CyberArk does not split graphical vs. text. Returns ``None`` when no endpoint - answered with usable data — the resource then inherits whatever + answered with usable data - the resource then inherits whatever the master policy / Keeper default decides. """ if not platform_id or not self._client: @@ -576,10 +738,10 @@ def _resolve_platform_session_recording( return self._platform_session_cache[platform_id] # Session-recording flags can live in any of three CyberArk - # surfaces depending on tenant type — none of them is a single + # surfaces depending on tenant type - none of them is a single # source of truth, so we query and merge: # - # 1. ``/PasswordVault/API/Platforms/{id}`` — Get-Platform-Details. + # 1. ``/PasswordVault/API/Platforms/{id}`` - Get-Platform-Details. # On older PVWA Self-Hosted this carries the # ``sessionManagement.recordAndSaveSessionActivity`` boolean. # On Privilege Cloud / ISPSS this DOES NOT include the @@ -588,11 +750,11 @@ def _resolve_platform_session_recording( # ``PSMServerId``/``PSMServerName`` from here as a fallback # indicator (PSM-attached platforms record by default). # 2. ``/PasswordVault/services/PoliciesMgt.asmx/ - # GetPolicyRulesSessionMonitoring`` — the PVWA admin-UI grid + # GetPolicyRulesSessionMonitoring`` - the PVWA admin-UI grid # service. On Privilege Cloud this is the *canonical* place # where ``RecordSession`` / ``MonitorSession`` rules are # surfaced (the user explicitly pointed us here). - # 3. PSMServer presence — heuristic fallback when neither (1) + # 3. PSMServer presence - heuristic fallback when neither (1) # nor (2) has the explicit flag: a platform that has a PSM # ConnectionComponent (``PSM-RDP``, ``PSM-SSH``, ...) # records sessions by default in CyberArk. @@ -607,7 +769,7 @@ def _resolve_platform_session_recording( self._platform_session_cache[platform_id] = None return None - # ── Build a flat ``rules`` dict from every envelope we got ──── + # -- Build a flat ``rules`` dict from every envelope we got ---- rules: Dict[str, Any] = {} for raw in (details, asmx): if not isinstance(raw, dict) or not raw: @@ -656,7 +818,7 @@ def _resolve_platform_session_recording( if k in rules and truthy(rules[k]): record_val = True break - # 3) Heuristic: PSMServer reference present → PSM is engaged + # 3) Heuristic: PSMServer reference present -> PSM is engaged # and CyberArk records by default. Used only when neither # (1) nor (2) provided an answer. if record_val is None: @@ -665,7 +827,7 @@ def _resolve_platform_session_recording( v = rules.get(k) if isinstance(v, str) and v.strip(): logging.debug( - "Platform '%s' has PSMServer=%s — assuming " + "Platform '%s' has PSMServer=%s - assuming " "session recording is on (PSM default)", platform_id, v, ) @@ -769,9 +931,9 @@ def _resolve_from_platform_metadata(self, platform_id: str) -> Optional[dict]: """Map a custom platformId via PVWA's PlatformBaseID / SystemType. Order: - 1. PlatformBaseID matches a built-in (e.g. WinDomain) — use that mapping. - 2. SystemType matches our SYSTEM_TYPE_MAP (e.g. Windows → RDP). - 3. None — caller falls through to keyword guessing. + 1. PlatformBaseID matches a built-in (e.g. WinDomain) - use that mapping. + 2. SystemType matches our SYSTEM_TYPE_MAP (e.g. Windows -> RDP). + 3. None - caller falls through to keyword guessing. """ meta = self._platform_index.get(platform_id) if not meta: @@ -791,8 +953,8 @@ def _infer_operating_system(platform_id: str, protocol: Optional[str]) -> Optional[str]: """Derive ``operating_system`` for a pamMachine from platformId/protocol. - Keyword scan on platformId (Win→windows, Unix/Linux→linux), then - protocol fallback (rdp→windows, ssh→linux). Returns ``None`` when + Keyword scan on platformId (Win->windows, Unix/Linux->linux), then + protocol fallback (rdp->windows, ssh->linux). Returns ``None`` when ambiguous so the field is left unset. """ pid = (platform_id or "").lower() @@ -834,9 +996,10 @@ def _enrich_port_from_details(self, platform_id: str, mapping: dict) -> dict: def map_account(self, account: dict, password: Optional[str] = None, safe_name: str = "") -> Optional[dict]: - """Convert a CyberArk account dict → pam_data record dict. + """Convert a CyberArk account dict -> pam_data record dict. - Returns None if the platformId is completely unknown and has no default. + Accounts with no resolvable platform mapping are returned as standalone + login records instead of speculative PAM resources. """ platform_id = account.get("platformId", "") mapping = self.platform_map.get(platform_id) if platform_id else None @@ -844,10 +1007,10 @@ def map_account(self, account: dict, password: Optional[str] = None, if mapping is None: # Resolution order for unknown / customer-renamed platforms: - # 1. PVWA platform metadata — PlatformBaseID → DEFAULT_PLATFORM_MAP, - # or SystemType → _SYSTEM_TYPE_MAP. Authoritative. + # 1. PVWA platform metadata - PlatformBaseID -> DEFAULT_PLATFORM_MAP, + # or SystemType -> _SYSTEM_TYPE_MAP. Authoritative. # 2. Substring keyword match on platformId / name. - # 3. pamMachine/SSH fallback. + # 3. Standalone login fallback (no PAM resource or nested user). label = platform_id if platform_id else "(empty)" self.unmapped_platforms[label] = self.unmapped_platforms.get(label, 0) + 1 @@ -856,7 +1019,7 @@ def map_account(self, account: dict, password: Optional[str] = None, mapping = via_pvwa mapping_source = "pvwa-platform" logging.warning( - "Unknown platformId '%s' for account '%s' — resolved via PVWA " + "Unknown platformId '%s' for account '%s' - resolved via PVWA " "platform metadata to %s/%s (port %s). Add it to --platform-map " "to lock in.", platform_id, account.get("name", ""), @@ -870,7 +1033,7 @@ def map_account(self, account: dict, password: Optional[str] = None, mapping_source = "keyword-guess" if platform_id: logging.warning( - "Unknown platformId '%s' for account '%s' — pattern-matched " + "Unknown platformId '%s' for account '%s' - pattern-matched " "to %s/%s (port %s). Add it to --platform-map to lock in.", platform_id, account.get("name", ""), mapping.get("record_type"), mapping.get("protocol") or "n/a", @@ -878,26 +1041,26 @@ def map_account(self, account: dict, password: Optional[str] = None, ) else: logging.debug( - "Empty platformId for account '%s' — pattern-matched to %s/%s.", + "Empty platformId for account '%s' - pattern-matched to %s/%s.", account.get("name", ""), mapping.get("record_type"), mapping.get("protocol") or "n/a", ) else: mapping = dict(FALLBACK_PLATFORM_MAP) - mapping_source = "fallback-ssh" + mapping_source = "fallback-login" if platform_id: logging.warning( - "Unknown platformId '%s' for account '%s' — defaulting to " - "pamMachine/SSH. Use --platform-map to override.", + "Unknown platformId '%s' for account '%s' - importing as " + "a standalone login. Use --platform-map to create a PAM resource.", platform_id, account.get("name", "")) else: - logging.debug("Empty platformId for account '%s' — defaulting to pamMachine/SSH.", + logging.debug("Empty platformId for account '%s' - importing as a standalone login.", account.get("name", "")) - # When the mapping came from a fallback path we trust PVWA's + # When the mapping came from an inferred resolution path we trust PVWA's # per-platform Details endpoint over our static defaults for the - # port. Cheap with caching — one call per unique custom platform. - if mapping_source in ("pvwa-platform", "keyword-guess", "fallback-ssh"): + # port. Cheap with caching - one call per unique custom platform. + if mapping_source in ("pvwa-platform", "keyword-guess"): mapping = self._enrich_port_from_details(platform_id, mapping) record_type = mapping.get("record_type", RECORD_TYPE_PAM_MACHINE) @@ -937,7 +1100,8 @@ def map_account(self, account: dict, password: Optional[str] = None, ) if record_type == RECORD_TYPE_LOGIN: - # BusinessWebsite → login record (not pamMachine) + # Explicit website mappings and unresolved platforms become + # standalone login records, never PAM resources with nested users. record = { "type": RECORD_TYPE_LOGIN, "title": item_name or title, @@ -946,26 +1110,102 @@ def map_account(self, account: dict, password: Optional[str] = None, } if url: record["url"] = url + if mapping_source == "fallback-login": + custom = self._build_unmapped_login_custom_fields(account) + if custom: + record["custom"] = custom return record + if record_type == RECORD_TYPE_PAM_REMOTE_BROWSER: + cloud = (mapping.get("cloud") or "").lower() + defaults = self._cloud_rbi_defaults(cloud) + rbi_url = ( + self._first_prop( + props, "URL", "Url", "url", "ConsoleURL", "ConsoleUrl", + "PortalURL", "PortalUrl", "LoginURL", "LoginUrl", + ) + or address + or defaults.get("url") + or url + ) + if rbi_url and not re.match(r"^https?://", rbi_url, re.IGNORECASE): + rbi_url = f"https://{rbi_url}" + + user_title = f"{login}@{title}" if login else f"user@{title or raw_name}" + user_record = { + "type": "pamUser", + "title": user_title, + "login": login, + "password": password or "", + } + secret_mgmt = account.get("secretManagement", {}) + cpm_enabled = secret_mgmt.get("automaticManagementEnabled", True) + if mapping.get("rotation"): + if cpm_enabled: + platform_sched = self._resolve_platform_schedule(platform_id) + if platform_sched: + schedule = copy.deepcopy(platform_sched) + self.platform_schedule_overrides[platform_id] = ( + self.platform_schedule_overrides.get(platform_id, 0) + 1 + ) + else: + schedule = copy.deepcopy(self._default_rotation_schedule) + else: + schedule = {"type": SCHEDULE_ON_DEMAND} + user_record["rotation_settings"] = { + "rotation": mapping["rotation"], + "enabled": "on" if cpm_enabled else "off", + "schedule": schedule, + } + complexity = self._resolve_platform_password_complexity(platform_id) + if complexity: + user_record["rotation_settings"]["password_complexity"] = complexity + self.platform_complexity_overrides[platform_id] = ( + self.platform_complexity_overrides.get(platform_id, 0) + 1 + ) + if password: + user_record["managed"] = True + self._add_source_metadata_to_user(user_record, account, platform_id) + + resource = { + "type": RECORD_TYPE_PAM_REMOTE_BROWSER, + "title": title or raw_name or f"{cloud.upper()} Console", + "url": rbi_url, + "users": [user_record], + "pam_settings": { + "options": { + "connections": "on", + "remote_browser_isolation": "on", + "graphical_session_recording": "on", + }, + "connection": { + "protocol": "http", + "autofill_credentials": user_title, + "autofill_targets": self._build_cloud_autofill_targets(cloud, rbi_url), + }, + }, + } + metadata_fields = self._resolve_platform_metadata(platform_id) + if metadata_fields: + resource.setdefault("custom", []).extend(copy.deepcopy(metadata_fields)) + account_metadata_fields = self._build_account_metadata_custom_fields(account) + if account_metadata_fields: + resource.setdefault("custom", []).extend(account_metadata_fields) + return resource + if record_type in (RECORD_TYPE_PAM_MACHINE, RECORD_TYPE_PAM_DATABASE): secret_type_check = account.get("secretType", "password").lower() - # No target host and no SSH key material → route to login. A + # No target host and no SSH key material -> route to login. A # pamMachine without a host can never be reached by the gateway, # so the credential is more useful as a standalone login record. # SSH keys keep pamMachine semantics even without an address so # the private_pem_key field is preserved. if not address and secret_type_check != "key": - note = (f"CyberArk platform: {platform_id}\n" - "No address — imported as login (not pamMachine)" - if platform_id else - "CyberArk account had no address — imported as login") return { "type": RECORD_TYPE_LOGIN, "title": title or raw_name, "login": login, "password": password or "", - "notes": note, } # Build pamUser nested inside the resource user_record = { @@ -993,7 +1233,7 @@ def map_account(self, account: dict, password: Optional[str] = None, dn = props.get("DistinguishedName", "") or props.get("distinguishedName", "") if dn: user_record["distinguished_name"] = dn - # Rotation settings — derive from CyberArk secretManagement state + # Rotation settings - derive from CyberArk secretManagement state secret_mgmt = account.get("secretManagement", {}) cpm_enabled = secret_mgmt.get("automaticManagementEnabled", True) if mapping.get("rotation"): @@ -1041,18 +1281,9 @@ def map_account(self, account: dict, password: Optional[str] = None, self.platform_complexity_overrides.get( platform_id_for_sched, 0) + 1 ) - reason = secret_mgmt.get("manualManagementReason", "") - if not cpm_enabled: - existing = user_record.get("notes", "") - line = f"CyberArk CPM disabled: {reason}" - user_record["notes"] = f"{existing}\n{line}".strip() - cpm_status = secret_mgmt.get("status", "") - if cpm_status and cpm_status.lower() == "failure": - existing = user_record.get("notes", "") - line = f"CyberArk CPM status: FAILURE ({reason})" - user_record["notes"] = f"{existing}\n{line}".strip() if password: user_record["managed"] = True + self._add_source_metadata_to_user(user_record, account, platform_id) resource_title = title or address or raw_name resource = { @@ -1062,13 +1293,13 @@ def map_account(self, account: dict, password: Optional[str] = None, "port": str(port) if port else "", "users": [user_record], } - # Map LogonDomain → domain_name on resource (Windows AD domain) + # Map LogonDomain -> domain_name on resource (Windows AD domain) if logon_domain and record_type == RECORD_TYPE_PAM_MACHINE: resource["domain_name"] = logon_domain # Derive operating_system on pamMachine so downstream consumers # (notably ``pam action service add``, which only mounts on Windows # hosts) can dispatch on it. Inferred from platformId keywords, - # fallback to protocol (rdp ⇒ windows, ssh ⇒ linux). + # fallback to protocol (rdp => windows, ssh => linux). if record_type == RECORD_TYPE_PAM_MACHINE: inferred_os = self._infer_operating_system( platform_id, mapping.get("protocol"), @@ -1091,7 +1322,7 @@ def map_account(self, account: dict, password: Optional[str] = None, recording = self._resolve_platform_session_recording( platform_id_for_settings) # Trigger workflows resolution (records unmapped items in - # self.platform_workflow_unmapped — return value not used + # self.platform_workflow_unmapped - return value not used # here because Keeper has no per-resource workflow toggles). self._resolve_platform_workflows(platform_id_for_settings) @@ -1135,14 +1366,25 @@ def map_account(self, account: dict, password: Optional[str] = None, copy.deepcopy(metadata_fields)) return resource - logging.warning('Unsupported record_type "%s" for platform "%s" — account skipped', + logging.warning('Unsupported record_type "%s" for platform "%s" - account skipped', record_type, account.get("platformId", "Unknown")) return None def is_incomplete(self, account: dict) -> Tuple[bool, str]: """Check if a CyberArk account is missing required fields for PAM import.""" + platform_id = account.get("platformId", "") + mapping = self.platform_map.get(platform_id) if platform_id else None + if mapping is None: + mapping = self._resolve_from_platform_metadata(platform_id) if platform_id else None + if mapping is None: + mapping = _guess_platform_mapping(platform_id, account.get("name", "")) + + is_cloud_rbi = ( + isinstance(mapping, dict) + and mapping.get("record_type") == RECORD_TYPE_PAM_REMOTE_BROWSER + ) reasons = [] - if not account.get("address"): + if not account.get("address") and not is_cloud_rbi: reasons.append("missing address/host") if not account.get("userName"): reasons.append("missing userName") @@ -1150,3 +1392,4 @@ def is_incomplete(self, account: dict) -> Tuple[bool, str]: return True, "; ".join(reasons) return False, "" + diff --git a/keepercommander/importer/cyberark/pam/constants.py b/keepercommander/importer/cyberark/pam/constants.py index b366f7a66..2d02fac73 100644 --- a/keepercommander/importer/cyberark/pam/constants.py +++ b/keepercommander/importer/cyberark/pam/constants.py @@ -61,6 +61,11 @@ def reset_system_safes() -> None: # Maximum character length for a single custom metadata field value. MAX_PLATFORM_METADATA_VALUE_LEN = 500 +# Equivalent limits for source-account metadata preserved on standalone login +# records when no CyberArk platform mapping can be resolved. +MAX_ACCOUNT_METADATA_FIELDS = 50 +MAX_ACCOUNT_METADATA_VALUE_LEN = 500 + # Maximum safe name length for Keeper shared folder names MAX_SAFE_NAME_LENGTH = 28 @@ -72,6 +77,7 @@ def reset_system_safes() -> None: RECORD_TYPE_PAM_MACHINE = "pamMachine" RECORD_TYPE_PAM_DATABASE = "pamDatabase" RECORD_TYPE_PAM_DIRECTORY = "pamDirectory" +RECORD_TYPE_PAM_REMOTE_BROWSER = "pamRemoteBrowser" # Rotation schedule type emitted in import JSON / PAM settings SCHEDULE_ON_DEMAND = "on-demand" @@ -122,11 +128,29 @@ def reset_system_safes() -> None: "CheckPointGAIA": {"record_type": RECORD_TYPE_PAM_MACHINE, "rotation": "general", "protocol": "ssh", "port": "22"}, # CyberArk internal — service accounts, import as pamMachine/SSH "CyberArk": {"record_type": RECORD_TYPE_PAM_MACHINE, "rotation": "general", "protocol": "ssh", "port": "22"}, + # Cloud consoles: create an RBI resource plus a companion pamUser. + "AWS": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}, + "AWSAccessKeys": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}, + "AWSAccessKey": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}, + "AmazonWebServices": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}, + "Azure": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "azure"}, + "AzureAccessKeys": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "azure"}, + "AzureAccessKey": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "azure"}, + "MicrosoftAzure": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "azure"}, + "GCP": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}, + "GCPAccessKeys": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}, + "GCPServiceAccount": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}, + "GoogleCloud": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}, + "GoogleCloudPlatform": {"record_type": RECORD_TYPE_PAM_REMOTE_BROWSER, "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}, # Web — login record, NOT pamMachine "BusinessWebsite": {"record_type": RECORD_TYPE_LOGIN, "rotation": None, "protocol": None, "port": None}, } -# Fallback mapping for accounts with empty or unknown platformId +# Fallback mapping for accounts that cannot be resolved through the explicit +# platform map, CyberArk platform metadata, or keyword matching. Without a +# reliable platform mapping there is not enough information to create a PAM +# resource and its nested pamUser safely, so preserve the credential as a +# standalone login record. FALLBACK_PLATFORM_MAP = { - "record_type": RECORD_TYPE_PAM_MACHINE, "rotation": "general", "protocol": "ssh", "port": "22", + "record_type": RECORD_TYPE_LOGIN, "rotation": None, "protocol": None, "port": None, } diff --git a/keepercommander/importer/cyberark/pam/dependents.py b/keepercommander/importer/cyberark/pam/dependents.py index 7259308f2..9f143e051 100644 --- a/keepercommander/importer/cyberark/pam/dependents.py +++ b/keepercommander/importer/cyberark/pam/dependents.py @@ -120,11 +120,28 @@ def resolve_account_dependents(client: 'CyberArkPVWAClient', dep, ("Address", "address", "Host", "host", "MachineAddress", "machineAddress", "TargetAddress", "targetAddress", - "ComputerName", "computerName"), + "ComputerName", "computerName", "LogonDevice", "logonDevice", + "Logon Device", "logon device", "Device", "device", + "Server", "server"), ) or _first_nonempty( props, ("address", "Address", "host", "Host", "machineAddress", - "MachineAddress"), + "MachineAddress", "LogonDevice", "logonDevice", "Logon Device", + "logon device", "Device", "device", "Server", "server"), + )) + machine_refs = _nonempty_values( + dep, + ("Address", "address", "Host", "host", "MachineAddress", + "machineAddress", "TargetAddress", "targetAddress", + "ComputerName", "computerName", "LogonDevice", "logonDevice", + "Logon Device", "logon device", "Device", "device", + "Server", "server"), + ) + machine_refs.extend(_nonempty_values( + props, + ("address", "Address", "host", "Host", "machineAddress", + "MachineAddress", "LogonDevice", "logonDevice", "Logon Device", + "logon device", "Device", "device", "Server", "server"), )) # ``platformId`` (Privilege Cloud) is the most reliable type signal — # it returns concise category codes like ``WinService`` / ``SchedTask`` @@ -152,6 +169,7 @@ def resolve_account_dependents(client: 'CyberArkPVWAClient', continue results.append({ "machine_address": address, + "machine_refs": _dedupe(machine_refs), "service_type": _normalize_dependent_type(raw_type), "raw_type": raw_type, "service_name": name, @@ -198,3 +216,29 @@ def _first_nonempty(source: dict, keys: tuple) -> str: if text: return text return "" + + +def _nonempty_values(source: dict, keys: tuple) -> List[str]: + values: List[str] = [] + if not isinstance(source, dict): + return values + for key in keys: + val = source.get(key) + if val is None: + continue + text = str(val).strip() + if text: + values.append(text) + return values + + +def _dedupe(values: List[str]) -> List[str]: + seen = set() + out: List[str] = [] + for value in values: + key = value.casefold() + if key in seen: + continue + seen.add(key) + out.append(value) + return out diff --git a/keepercommander/importer/cyberark/pam/idempotency.py b/keepercommander/importer/cyberark/pam/idempotency.py index 4851c2188..5097e8232 100644 --- a/keepercommander/importer/cyberark/pam/idempotency.py +++ b/keepercommander/importer/cyberark/pam/idempotency.py @@ -44,6 +44,9 @@ from enum import Enum from typing import Any, Dict, List, Optional, Set, Tuple +from keepercommander.commands.pam_import.nsf_helpers import get_folder_record_uids +from keepercommander.commands.pam_import.record_loader import load_pam_record + # --------------------------------------------------------------------------- # Notes marker @@ -203,9 +206,8 @@ def build_existing_index(params, folder_uids) -> ExistingRecordIndex: # are populated by ``api.sync_down``, but a fresh session without a # sync will have them as ``None`` or empty dicts. Bail early so the # importer falls back to always-create mode instead of crashing. - subfolder_record_cache = getattr(params, "subfolder_record_cache", None) or {} folder_cache = getattr(params, "folder_cache", None) or {} - if not folder_cache: + if not folder_cache and not getattr(params, "nested_share_folders", None): return index # Recursively collect record UIDs from every subfolder. @@ -218,15 +220,19 @@ def build_existing_index(params, folder_uids) -> ExistingRecordIndex: continue visited.add(fuid) index.scanned_folder_uids.add(fuid) - for ruid in (subfolder_record_cache.get(fuid) or set()): + record_uids = get_folder_record_uids(params, fuid) + for ruid in record_uids: all_record_uids.add(ruid) index.folder_by_record[ruid] = fuid folder = folder_cache.get(fuid) for sub_uid in (getattr(folder, "subfolders", []) or []) if folder else []: stack.append(sub_uid) + for child_uid, info in (getattr(params, "nested_share_folders", None) or {}).items(): + if (info.get("parent_uid") or None) == fuid and child_uid not in visited: + stack.append(child_uid) for ruid in all_record_uids: - rec = vault.KeeperRecord.load(params, ruid) + rec = load_pam_record(params, ruid) or vault.KeeperRecord.load(params, ruid) if rec is None: continue rtype = getattr(rec, "record_type", "") or "" diff --git a/keepercommander/importer/cyberark/pam/linked_accounts.py b/keepercommander/importer/cyberark/pam/linked_accounts.py index 93e4d3720..d4e040662 100644 --- a/keepercommander/importer/cyberark/pam/linked_accounts.py +++ b/keepercommander/importer/cyberark/pam/linked_accounts.py @@ -59,23 +59,15 @@ def resolve_linked_accounts(client: 'CyberArkPVWAClient', "title": user_title, "login": link_data.get("userName", linked_name), "password": password or "", - "notes": f"CyberArk role: {role_name} account\n" - f"Linked to: {account.get('name', account_id)}\n" - f"Source safe: {linked_safe}", + "custom": [ + {"type": "text", "label": "CyberArk Role", "value": [f"{role_name} account"]}, + {"type": "text", "label": "CyberArk Linked To", "value": [account.get("name", account_id)]}, + {"type": "text", "label": "CyberArk Source Safe", "value": [linked_safe]}, + ], "_ca_role": role_name, # Internal: logon, reconcile, or enable "_ca_id": str(linked_id), # Internal: used by idempotency layer "_ca_safe": linked_safe, # Internal: used by idempotency layer } - # Embed CyberArk identity marker so re-imports can match this - # linked account to the existing Keeper record. The linked - # account has its own CyberArk id distinct from the master - # account it decorates, so we tag with ``linked_id`` (not the - # outer ``account_id``). - try: - from .idempotency import annotate_record_with_marker - annotate_record_with_marker(linked_user, str(linked_id), linked_safe) - except Exception: # noqa: BLE001 — never block linked-account resolution on annotation failure - logging.debug("Failed to annotate linked account %s with CyberArk-ID marker", linked_id) result.append(linked_user) logging.info('Resolved linked %s account: %s', role_name, user_title) diff --git a/keepercommander/importer/cyberark/pam/platform_mapping.py b/keepercommander/importer/cyberark/pam/platform_mapping.py index 0920c16b7..5125684b6 100644 --- a/keepercommander/importer/cyberark/pam/platform_mapping.py +++ b/keepercommander/importer/cyberark/pam/platform_mapping.py @@ -9,8 +9,6 @@ import re from typing import Dict, Optional, Tuple -from .constants import FALLBACK_PLATFORM_MAP - # Keyword → mapping for custom/renamed CyberArk platforms (e.g. "Custom-WinDomain" → RDP). # Matched against platformId and account name; first match wins. Database patterns # are checked before generic Windows/Unix to avoid false positives. @@ -24,6 +22,12 @@ ("mssql", {"record_type": "pamDatabase", "rotation": "general", "protocol": "sql-server", "port": "1433", "database_type": "mssql"}), ("sqlserver", {"record_type": "pamDatabase", "rotation": "general", "protocol": "sql-server", "port": "1433", "database_type": "mssql"}), ("mongo", {"record_type": "pamDatabase", "rotation": "general", "protocol": "mongodb", "port": "27017", "database_type": "mongodb"}), + # Cloud consoles (RBI) + ("aws", {"record_type": "pamRemoteBrowser", "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}), + ("amazon", {"record_type": "pamRemoteBrowser", "rotation": "general", "protocol": None, "port": None, "cloud": "aws"}), + ("azure", {"record_type": "pamRemoteBrowser", "rotation": "general", "protocol": None, "port": None, "cloud": "azure"}), + ("gcp", {"record_type": "pamRemoteBrowser", "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}), + ("googlecloud", {"record_type": "pamRemoteBrowser", "rotation": "general", "protocol": None, "port": None, "cloud": "gcp"}), # Windows (RDP) ("windomain", {"record_type": "pamMachine", "rotation": "general", "protocol": "rdp", "port": "3389"}), ("windows", {"record_type": "pamMachine", "rotation": "general", "protocol": "rdp", "port": "3389"}), @@ -47,7 +51,7 @@ def _guess_platform_mapping(platform_id: str, raw_name: str) -> Optional[dict]: """Map unknown/custom platformIds to a record type via keyword scan. - Returns a mapping dict on match, or None (caller uses FALLBACK_PLATFORM_MAP). + Returns a mapping dict on match, or None (caller imports a standalone login). """ haystack = f"{platform_id or ''}\n{raw_name or ''}".lower() if not haystack.strip(): diff --git a/keepercommander/importer/imp_exp.py b/keepercommander/importer/imp_exp.py index 75914c0df..66e1d3075 100644 --- a/keepercommander/importer/imp_exp.py +++ b/keepercommander/importer/imp_exp.py @@ -617,7 +617,7 @@ def import_teams(params, teams, full_sync=False): # type: (KeeperParams, List[ def import_user_permissions(params, shared_folders, - full_sync=False): # type: (KeeperParams, List[ImportSharedFolder], bool) -> None + full_sync=False, unsafe=False): # type: (KeeperParams, List[ImportSharedFolder], bool) -> None if not shared_folders: return @@ -654,7 +654,7 @@ def import_user_permissions(params, folders = [x for x in folders if x.uid in params.shared_folder_cache] if folders: - permissions = prepare_folder_permission(params, folders, full_sync) + permissions = prepare_folder_permission(params, folders, full_sync, unsafe) if permissions: teams_added = 0 users_added = 0 @@ -1889,32 +1889,46 @@ def prepare_folder_add(params, folders, records, manage_users, manage_records, c is_last = True if digest not in folder_hash: - folder_uid = api.generate_record_uid() - folder_type = 'shared_folder' if is_last else 'user_folder' + existing_by_uid = None + if is_last: + sf_uid = getattr(fol, 'uid', None) + if sf_uid and sf_uid in params.shared_folder_cache and sf_uid in params.folder_cache: + # The import file names an existing shared folder by uid. + # Reuse it in place (no new folder, no move) so any + # records that reference this path resolve to it. + shared_folder_key = params.shared_folder_cache[sf_uid]['shared_folder_key_unencrypted'] + existing_by_uid = sf_uid, 'shared_folder', shared_folder_key + + if existing_by_uid is not None: + folder_uid, folder_type, folder_key = existing_by_uid + folder_hash[digest] = existing_by_uid + else: + folder_uid = api.generate_record_uid() + folder_type = 'shared_folder' if is_last else 'user_folder' - fol_req = folder_pb2.FolderRequest() - fol_req.folderUid = base64.urlsafe_b64decode(folder_uid + '==') - fol_req.folderType = 2 if folder_type == 'shared_folder' else 1 + fol_req = folder_pb2.FolderRequest() + fol_req.folderUid = base64.urlsafe_b64decode(folder_uid + '==') + fol_req.folderType = 2 if folder_type == 'shared_folder' else 1 - if parent_uid: - fol_req.parentFolderUid = base64.urlsafe_b64decode(parent_uid + '==') + if parent_uid: + fol_req.parentFolderUid = base64.urlsafe_b64decode(parent_uid + '==') - folder_key = utils.generate_aes_key() - fol_req.encryptedFolderKey = crypto.encrypt_aes_v1(folder_key, params.data_key) + folder_key = utils.generate_aes_key() + fol_req.encryptedFolderKey = crypto.encrypt_aes_v1(folder_key, params.data_key) - data = {'name': comp} - fol_req.folderData = crypto.encrypt_aes_v1(json.dumps(data).encode('utf-8'), folder_key) + data = {'name': comp} + fol_req.folderData = crypto.encrypt_aes_v1(json.dumps(data).encode('utf-8'), folder_key) - if folder_type == 'shared_folder': - fol_req.sharedFolderFields.encryptedFolderName = \ - crypto.encrypt_aes_v1(comp.encode('utf-8'), folder_key) - fol_req.sharedFolderFields.manageUsers = fol.manage_users or manage_users - fol_req.sharedFolderFields.manageRecords = fol.manage_records or manage_records - fol_req.sharedFolderFields.canEdit = fol.can_edit or can_edit - fol_req.sharedFolderFields.canShare = fol.can_share or can_share + if folder_type == 'shared_folder': + fol_req.sharedFolderFields.encryptedFolderName = \ + crypto.encrypt_aes_v1(comp.encode('utf-8'), folder_key) + fol_req.sharedFolderFields.manageUsers = fol.manage_users or manage_users + fol_req.sharedFolderFields.manageRecords = fol.manage_records or manage_records + fol_req.sharedFolderFields.canEdit = fol.can_edit or can_edit + fol_req.sharedFolderFields.canShare = fol.can_share or can_share - folder_add.append(fol_req) - folder_hash[digest] = folder_uid, folder_type, folder_key if folder_type == 'shared_folder' else None + folder_add.append(fol_req) + folder_hash[digest] = folder_uid, folder_type, folder_key if folder_type == 'shared_folder' else None else: folder_uid, folder_type, folder_key = folder_hash[digest] if is_last: @@ -2572,7 +2586,7 @@ def prepare_record_link(params, records): return record_links -def prepare_folder_permission(params, folders, full_sync): +def prepare_folder_permission(params, folders, full_sync, unsafe=False): # type: (KeeperParams, List[ImportSharedFolder], bool) -> list """Prepare a list of API interactions for changes to folder permissions.""" shared_folder_lookup = {} @@ -2668,6 +2682,9 @@ def prepare_folder_permission(params, folders, full_sync): existing_users.update((x['username'] for x in shared_folder['users'])) if params.user in existing_users: existing_users.remove(params.user) + if unsafe: + # Set user to end of array to be removed last + existing_users.add(params.user) keep_teams = set() keep_users = set() diff --git a/keepercommander/proto/pam_pb2.py b/keepercommander/proto/pam_pb2.py index b773f6c8e..6c0f68645 100644 --- a/keepercommander/proto/pam_pb2.py +++ b/keepercommander/proto/pam_pb2.py @@ -26,7 +26,7 @@ from . import record_pb2 as record__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tpam.proto\x12\x03PAM\x1a\x10\x65nterprise.proto\x1a\x0crecord.proto\"\x83\x01\n\x13PAMRotationSchedule\x12\x11\n\trecordUid\x18\x01 \x01(\x0c\x12\x18\n\x10\x63onfigurationUid\x18\x02 \x01(\x0c\x12\x15\n\rcontrollerUid\x18\x03 \x01(\x0c\x12\x14\n\x0cscheduleData\x18\x04 \x01(\t\x12\x12\n\nnoSchedule\x18\x05 \x01(\x08\"K\n\x1cPAMRotationSchedulesResponse\x12+\n\tschedules\x18\x01 \x03(\x0b\x32\x18.PAM.PAMRotationSchedule\"\x94\x01\n\x13PAMOnlineController\x12\x15\n\rcontrollerUid\x18\x01 \x01(\x0c\x12\x13\n\x0b\x63onnectedOn\x18\x02 \x01(\x03\x12\x11\n\tipAddress\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\t\x12-\n\x0b\x63onnections\x18\x05 \x03(\x0b\x32\x18.PAM.PAMWebRtcConnection\"\xa7\x01\n\x13PAMWebRtcConnection\x12\x15\n\rconnectionUid\x18\x01 \x01(\x0c\x12\'\n\x04type\x18\x02 \x01(\x0e\x32\x19.PAM.WebRtcConnectionType\x12\x11\n\trecordUid\x18\x03 \x01(\x0c\x12\x10\n\x08userName\x18\x04 \x01(\t\x12\x11\n\tstartedOn\x18\x05 \x01(\x03\x12\x18\n\x10\x63onfigurationUid\x18\x06 \x01(\x0c\"Y\n\x14PAMOnlineControllers\x12\x12\n\ndeprecated\x18\x01 \x03(\x0c\x12-\n\x0b\x63ontrollers\x18\x02 \x03(\x0b\x32\x18.PAM.PAMOnlineController\"9\n\x10PAMRotateRequest\x12\x12\n\nrequestUid\x18\x01 \x01(\x0c\x12\x11\n\trecordUid\x18\x02 \x01(\x0c\"A\n\x16PAMControllersResponse\x12\'\n\x0b\x63ontrollers\x18\x01 \x03(\x0b\x32\x12.PAM.PAMController\"=\n\x13PAMRemoveController\x12\x15\n\rcontrollerUid\x18\x01 \x01(\x0c\x12\x0f\n\x07message\x18\x02 \x01(\t\"L\n\x1bPAMRemoveControllerResponse\x12-\n\x0b\x63ontrollers\x18\x01 \x03(\x0b\x32\x18.PAM.PAMRemoveController\"=\n\x10PAMModifyRequest\x12)\n\noperations\x18\x01 \x03(\x0b\x32\x15.PAM.PAMDataOperation\"\x98\x01\n\x10PAMDataOperation\x12,\n\roperationType\x18\x01 \x01(\x0e\x32\x15.PAM.PAMOperationType\x12\x30\n\rconfiguration\x18\x02 \x01(\x0b\x32\x19.PAM.PAMConfigurationData\x12$\n\x07\x65lement\x18\x03 \x01(\x0b\x32\x13.PAM.PAMElementData\"e\n\x14PAMConfigurationData\x12\x18\n\x10\x63onfigurationUid\x18\x01 \x01(\x0c\x12\x0e\n\x06nodeId\x18\x02 \x01(\x03\x12\x15\n\rcontrollerUid\x18\x03 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\"E\n\x0ePAMElementData\x12\x12\n\nelementUid\x18\x01 \x01(\x0c\x12\x11\n\tparentUid\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"m\n\x19PAMElementOperationResult\x12\x12\n\nelementUid\x18\x01 \x01(\x0c\x12+\n\x06result\x18\x02 \x01(\x0e\x32\x1b.PAM.PAMOperationResultType\x12\x0f\n\x07message\x18\x03 \x01(\t\"B\n\x0fPAMModifyResult\x12/\n\x07results\x18\x01 \x03(\x0b\x32\x1e.PAM.PAMElementOperationResult\"x\n\nPAMElement\x12\x12\n\nelementUid\x18\x01 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\x0f\n\x07\x63reated\x18\x03 \x01(\x03\x12\x14\n\x0clastModified\x18\x04 \x01(\x03\x12!\n\x08\x63hildren\x18\x05 \x03(\x0b\x32\x0f.PAM.PAMElement\"#\n\x14PAMGenericUidRequest\x12\x0b\n\x03uid\x18\x01 \x01(\x0c\"%\n\x15PAMGenericUidsRequest\x12\x0c\n\x04uids\x18\x01 \x03(\x0c\"\xab\x01\n\x10PAMConfiguration\x12\x18\n\x10\x63onfigurationUid\x18\x01 \x01(\x0c\x12\x0e\n\x06nodeId\x18\x02 \x01(\x03\x12\x15\n\rcontrollerUid\x18\x03 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x12\x0f\n\x07\x63reated\x18\x05 \x01(\x03\x12\x14\n\x0clastModified\x18\x06 \x01(\x03\x12!\n\x08\x63hildren\x18\x07 \x03(\x0b\x32\x0f.PAM.PAMElement\"B\n\x11PAMConfigurations\x12-\n\x0e\x63onfigurations\x18\x01 \x03(\x0b\x32\x15.PAM.PAMConfiguration\"\xab\x02\n\rPAMController\x12\x15\n\rcontrollerUid\x18\x01 \x01(\x0c\x12\x16\n\x0e\x63ontrollerName\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65viceToken\x18\x03 \x01(\t\x12\x12\n\ndeviceName\x18\x04 \x01(\t\x12\x0e\n\x06nodeId\x18\x05 \x01(\x03\x12\x0f\n\x07\x63reated\x18\x06 \x01(\x03\x12\x14\n\x0clastModified\x18\x07 \x01(\x03\x12\x16\n\x0e\x61pplicationUid\x18\x08 \x01(\x0c\x12\x30\n\rappClientType\x18\t \x01(\x0e\x32\x19.Enterprise.AppClientType\x12\x15\n\risInitialized\x18\n \x01(\x08\x12\x18\n\x10maxInstanceCount\x18\x0b \x01(\x05\x12\x10\n\x08lastSeen\x18\x0c \x01(\x03\"P\n\x1dPAMSetMaxInstanceCountRequest\x12\x15\n\rcontrollerUid\x18\x01 \x01(\x0c\x12\x18\n\x10maxInstanceCount\x18\x02 \x01(\x05\"%\n\x12\x43ontrollerResponse\x12\x0f\n\x07payload\x18\x01 \x01(\t\"M\n\x1aPAMConfigurationController\x12\x18\n\x10\x63onfigurationUid\x18\x01 \x01(\x0c\x12\x15\n\rcontrollerUid\x18\x02 \x01(\x0c\"\xa3\x01\n\x17\x43onfigurationAddRequest\x12\x18\n\x10\x63onfigurationUid\x18\x01 \x01(\x0c\x12\x11\n\trecordKey\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12(\n\x0brecordLinks\x18\x04 \x03(\x0b\x32\x13.Records.RecordLink\x12#\n\x05\x61udit\x18\x05 \x01(\x0b\x32\x14.Records.RecordAudit\"J\n\x10RelayAccessCreds\x12\x10\n\x08username\x18\x01 \x01(\t\x12\x10\n\x08password\x18\x02 \x01(\t\x12\x12\n\nserverTime\x18\x03 \x01(\x03\"\\\n\x10KturnAccessCreds\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\r\n\x05token\x18\x02 \x01(\x0c\x12\x18\n\x10\x63\x65rt_fingerprint\x18\x03 \x01(\t\x12\x12\n\nexpires_at\x18\x04 \x01(\x04\"\x81\x02\n\x14PAMRecordingsRequest\x12\x11\n\trecordUid\x18\x01 \x01(\x0c\x12\x10\n\x08maxCount\x18\x02 \x01(\x05\x12\x17\n\nrangeStart\x18\x03 \x01(\x03H\x00\x88\x01\x01\x12\x15\n\x08rangeEnd\x18\x04 \x01(\x03H\x01\x88\x01\x01\x12$\n\x05types\x18\x05 \x03(\x0e\x32\x15.PAM.PAMRecordingType\x12)\n\x05risks\x18\x06 \x03(\x0e\x32\x1a.PAM.PAMRecordingRiskLevel\x12\x11\n\tprotocols\x18\x07 \x03(\t\x12\x14\n\x0c\x63loseReasons\x18\x08 \x03(\x05\x42\r\n\x0b_rangeStartB\x0b\n\t_rangeEnd\"\x89\x02\n\x1cPAMRecordingsForUsersRequest\x12\x11\n\tusernames\x18\x01 \x03(\t\x12\x10\n\x08maxCount\x18\x02 \x01(\x05\x12\x17\n\nrangeStart\x18\x03 \x01(\x03H\x00\x88\x01\x01\x12\x15\n\x08rangeEnd\x18\x04 \x01(\x03H\x01\x88\x01\x01\x12$\n\x05types\x18\x05 \x03(\x0e\x32\x15.PAM.PAMRecordingType\x12)\n\x05risks\x18\x06 \x03(\x0e\x32\x1a.PAM.PAMRecordingRiskLevel\x12\x11\n\tprotocols\x18\x07 \x03(\t\x12\x14\n\x0c\x63loseReasons\x18\x08 \x03(\x05\x42\r\n\x0b_rangeStartB\x0b\n\t_rangeEnd\"\xd4\x02\n\x0cPAMRecording\x12\x15\n\rconnectionUid\x18\x01 \x01(\x0c\x12,\n\rrecordingType\x18\x02 \x01(\x0e\x32\x15.PAM.PAMRecordingType\x12\x11\n\trecordUid\x18\x03 \x01(\x0c\x12\x10\n\x08userName\x18\x04 \x01(\t\x12\x11\n\tstartedOn\x18\x05 \x01(\x03\x12\x0e\n\x06length\x18\x06 \x01(\x05\x12\x10\n\x08\x66ileSize\x18\x07 \x01(\x03\x12\x11\n\tcreatedOn\x18\x08 \x01(\x03\x12\x10\n\x08protocol\x18\t \x01(\t\x12\x13\n\x0b\x63loseReason\x18\n \x01(\x05\x12\x19\n\x11recordingDuration\x18\x0b \x01(\x05\x12\x36\n\x12\x61iOverallRiskLevel\x18\x0c \x01(\x0e\x32\x1a.PAM.PAMRecordingRiskLevel\x12\x18\n\x10\x61iOverallSummary\x18\r \x01(\x0c\"O\n\x15PAMRecordingsResponse\x12%\n\nrecordings\x18\x01 \x03(\x0b\x32\x11.PAM.PAMRecording\x12\x0f\n\x07hasMore\x18\x02 \x01(\x08\"*\n\x07PAMData\x12\x0e\n\x06vertex\x18\x01 \x01(\x0c\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\"\x17\n\x07UidList\x12\x0c\n\x04uids\x18\x01 \x03(\x0c\"5\n\x0fPAMServiceNames\x12\x13\n\x0bresourceUid\x18\x01 \x01(\x0c\x12\r\n\x05names\x18\x02 \x01(\x0c\"\xb4\x03\n\x11PAMResourceConfig\x12\x11\n\trecordUid\x18\x01 \x01(\x0c\x12\x17\n\nnetworkUid\x18\x02 \x01(\x0cH\x00\x88\x01\x01\x12\x15\n\x08\x61\x64minUid\x18\x03 \x01(\x0cH\x01\x88\x01\x01\x12\x11\n\x04meta\x18\x04 \x01(\x0cH\x02\x88\x01\x01\x12\x1f\n\x12\x63onnectionSettings\x18\x05 \x01(\x0cH\x03\x88\x01\x01\x12\'\n\x0c\x63onnectUsers\x18\x06 \x01(\x0b\x32\x0c.PAM.UidListH\x04\x88\x01\x01\x12\x16\n\tdomainUid\x18\x07 \x01(\x0cH\x05\x88\x01\x01\x12\x18\n\x0bjitSettings\x18\x08 \x01(\x0cH\x06\x88\x01\x01\x12\x1d\n\x10keeperAiSettings\x18\t \x01(\x0cH\x07\x88\x01\x01\x12\x1b\n\x0eupdateServices\x18\n \x01(\x08H\x08\x88\x01\x01\x42\r\n\x0b_networkUidB\x0b\n\t_adminUidB\x07\n\x05_metaB\x15\n\x13_connectionSettingsB\x0f\n\r_connectUsersB\x0c\n\n_domainUidB\x0e\n\x0c_jitSettingsB\x13\n\x11_keeperAiSettingsB\x11\n\x0f_updateServices\"%\n\x16PAMUniversalSyncFolder\x12\x0b\n\x03uid\x18\x01 \x01(\x0c\" \n\x10GitHubRepository\x12\x0c\n\x04name\x18\x01 \x01(\x0c\"\xd6\x01\n\x0cGitHubConfig\x12$\n\x05scope\x18\x01 \x01(\x0e\x32\x10.PAM.GitHubScopeH\x00\x88\x01\x01\x12\r\n\x05owner\x18\x02 \x01(\x0c\x12\x46\n\x16organizationVisibility\x18\x03 \x01(\x0e\x32!.PAM.GitHubOrganizationVisibilityH\x01\x88\x01\x01\x12$\n\x05repos\x18\x04 \x03(\x0b\x32\x15.PAM.GitHubRepositoryB\x08\n\x06_scopeB\x19\n\x17_organizationVisibility\"\xaf\x02\n\x16PAMUniversalSyncConfig\x12\x12\n\nnetworkUid\x18\x01 \x01(\x0c\x12\x14\n\x07\x65nabled\x18\x02 \x01(\x08H\x00\x88\x01\x01\x12\x1a\n\rdryRunEnabled\x18\x03 \x01(\x08H\x01\x88\x01\x01\x12,\n\x07\x66olders\x18\x04 \x03(\x0b\x32\x1b.PAM.PAMUniversalSyncFolder\x12\x19\n\x0csyncIdentity\x18\x05 \x01(\x0cH\x02\x88\x01\x01\x12\x16\n\tvaultName\x18\x06 \x01(\x0cH\x03\x88\x01\x01\x12&\n\x06github\x18\x07 \x01(\x0b\x32\x11.PAM.GitHubConfigH\x04\x88\x01\x01\x42\n\n\x08_enabledB\x10\n\x0e_dryRunEnabledB\x0f\n\r_syncIdentityB\x0c\n\n_vaultNameB\t\n\x07_github\"7\n\x11NhiMetricsRequest\x12\x11\n\tstartTime\x18\x01 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x02 \x01(\x03\"\x9c\x02\n\x0ePamUsageByUser\x12\x0e\n\x06userId\x18\x01 \x01(\x05\x12!\n\x19recordRotationScheduledOk\x18\x02 \x01(\x05\x12\x1c\n\x14pamConnectionStarted\x18\x03 \x01(\x05\x12\x18\n\x10pamTunnelStarted\x18\x04 \x01(\x05\x12\x1b\n\x13\x64iscoveryJobStarted\x18\x05 \x01(\x05\x12 \n\x18recordRotationOnDemandOk\x18\x06 \x01(\x05\x12\"\n\x1apamSessionRecordingStarted\x18\x07 \x01(\x05\x12\x15\n\rpamRbiStarted\x18\x08 \x01(\x05\x12%\n\x1dpamSessionRbiRecordingStarted\x18\t \x01(\x05\"p\n\x0eNhiUsageByUser\x12\x0e\n\x06userId\x18\x01 \x01(\x05\x12\x11\n\trotations\x18\x02 \x01(\x05\x12\x0f\n\x07tunnels\x18\x03 \x01(\x05\x12\x13\n\x0b\x63onnections\x18\x04 \x01(\x05\x12\x15\n\rdiscoveryJobs\x18\x05 \x01(\x05\"\x84\x02\n\x12NhiMetricsResponse\x12\x14\n\x0c\x65nterpriseId\x18\x01 \x01(\x03\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x18\n\x10uniqueKsmDevices\x18\x04 \x01(\x05\x12\x18\n\x10pamGatewayOnline\x18\x05 \x01(\x05\x12/\n\x0epamUsageByUser\x18\x06 \x03(\x0b\x32\x13.PAM.PamUsageByUserB\x02\x18\x01\x12\x10\n\x08nhiCount\x18\x07 \x01(\x05\x12\x13\n\x0bksmNhiCount\x18\x08 \x01(\x05\x12(\n\x0busageByUser\x18\t \x03(\x0b\x32\x13.PAM.NhiUsageByUser\"D\n\x16NhiBulkMetricsResponse\x12*\n\tresponses\x18\x01 \x03(\x0b\x32\x17.PAM.NhiMetricsResponse\"^\n\x0bNhiUidEntry\x12\x0b\n\x03uid\x18\x01 \x01(\t\x12\"\n\x08\x63\x61tegory\x18\x02 \x01(\x0e\x32\x10.PAM.NhiCategory\x12\x0e\n\x06ksmNhi\x18\x03 \x01(\x08\x12\x0e\n\x06\x61ppUid\x18\x04 \x01(\t\"7\n\x11GetNhiUidsRequest\x12\x11\n\tstartTime\x18\x01 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x02 \x01(\x03\"4\n\x12GetNhiUidsResponse\x12\x1e\n\x04uids\x18\x01 \x03(\x0b\x32\x10.PAM.NhiUidEntry\"6\n\x1dSetNhiKsmEffectiveDateRequest\x12\x15\n\reffectiveDate\x18\x01 \x01(\x03\"L\n\x1eGetNhiKsmEffectiveDateResponse\x12\x15\n\reffectiveDate\x18\x01 \x01(\x03\x12\x13\n\x0b\x64\x65\x66\x61ultDate\x18\x02 \x01(\x03\"I\n\x1fPAMUniversalSyncPreCheckRequest\x12\x12\n\nnetworkUid\x18\x01 \x01(\x0c\x12\x12\n\nfolderUids\x18\x02 \x03(\x0c\"C\n\x1ePAMUniversalSyncPreCheckResult\x12\x11\n\tfolderUid\x18\x01 \x01(\x0c\x12\x0e\n\x06isUsed\x18\x02 \x01(\x08\"X\n PAMUniversalSyncPreCheckResponse\x12\x34\n\x07results\x18\x01 \x03(\x0b\x32#.PAM.PAMUniversalSyncPreCheckResult*\x9e\x01\n\x14WebRtcConnectionType\x12\x0e\n\nCONNECTION\x10\x00\x12\n\n\x06TUNNEL\x10\x01\x12\x07\n\x03SSH\x10\x02\x12\x07\n\x03RDP\x10\x03\x12\x08\n\x04HTTP\x10\x04\x12\x07\n\x03VNC\x10\x05\x12\n\n\x06TELNET\x10\x06\x12\t\n\x05MYSQL\x10\x07\x12\x0e\n\nSQL_SERVER\x10\x08\x12\x0e\n\nPOSTGRESQL\x10\t\x12\x0e\n\nKUBERNETES\x10\n*@\n\x10PAMOperationType\x12\x07\n\x03\x41\x44\x44\x10\x00\x12\n\n\x06UPDATE\x10\x01\x12\x0b\n\x07REPLACE\x10\x02\x12\n\n\x06\x44\x45LETE\x10\x03*p\n\x16PAMOperationResultType\x12\x0f\n\x0bPOT_SUCCESS\x10\x00\x12\x15\n\x11POT_UNKNOWN_ERROR\x10\x01\x12\x16\n\x12POT_ALREADY_EXISTS\x10\x02\x12\x16\n\x12POT_DOES_NOT_EXIST\x10\x03*\xc9\x01\n\x15\x43ontrollerMessageType\x12\x0f\n\x0b\x43MT_GENERAL\x10\x00\x12\x0e\n\nCMT_ROTATE\x10\x01\x12\x11\n\rCMT_DISCOVERY\x10\x02\x12\x0f\n\x0b\x43MT_CONNECT\x10\x03\x12\x19\n\x15\x43MT_ANALYZE_RECORDING\x10\x04\x12!\n\x1d\x43MT_WORKFLOW_ACCESS_ELEVATION\x10\x05\x12\x0b\n\x07\x43MT_USS\x10\x06\x12\x0c\n\x08\x43MT_INFO\x10\x07\x12\x12\n\x0e\x43MT_AUTOMATION\x10\x08*V\n\x10PAMRecordingType\x12\x0f\n\x0bPRT_SESSION\x10\x00\x12\x12\n\x0ePRT_TYPESCRIPT\x10\x01\x12\x0c\n\x08PRT_TIME\x10\x02\x12\x0f\n\x0bPRT_SUMMARY\x10\x03*i\n\x15PAMRecordingRiskLevel\x12\x13\n\x0fPRR_UNSPECIFIED\x10\x00\x12\x0b\n\x07PRR_LOW\x10\x01\x12\x0e\n\nPRR_MEDIUM\x10\x02\x12\x0c\n\x08PRR_HIGH\x10\x03\x12\x10\n\x0cPRR_CRITICAL\x10\x04*/\n\x0bGitHubScope\x12\x0e\n\nREPOSITORY\x10\x00\x12\x10\n\x0cORGANIZATION\x10\x01*B\n\x1cGitHubOrganizationVisibility\x12\x07\n\x03\x41LL\x10\x00\x12\x0b\n\x07PRIVATE\x10\x01\x12\x0c\n\x08SELECTED\x10\x02*`\n\x0bNhiCategory\x12\x18\n\x14NHI_CATEGORY_UNKNOWN\x10\x00\x12\x0c\n\x08PAM_USER\x10\x01\x12\x10\n\x0cPAM_RESOURCE\x10\x02\x12\x0b\n\x07GATEWAY\x10\x03\x12\n\n\x06\x44\x45VICE\x10\x04\x42\x1f\n\x18\x63om.keepersecurity.protoB\x03PAMb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tpam.proto\x12\x03PAM\x1a\x10enterprise.proto\x1a\x0crecord.proto"\x83\x01\n\x13PAMRotationSchedule\x12\x11\n\trecordUid\x18\x01 \x01(\x0c\x12\x18\n\x10configurationUid\x18\x02 \x01(\x0c\x12\x15\n\rcontrollerUid\x18\x03 \x01(\x0c\x12\x14\n\x0cscheduleData\x18\x04 \x01(\t\x12\x12\n\nnoSchedule\x18\x05 \x01(\x08"K\n\x1cPAMRotationSchedulesResponse\x12+\n\tschedules\x18\x01 \x03(\x0b2\x18.PAM.PAMRotationSchedule"\x94\x01\n\x13PAMOnlineController\x12\x15\n\rcontrollerUid\x18\x01 \x01(\x0c\x12\x13\n\x0bconnectedOn\x18\x02 \x01(\x03\x12\x11\n\tipAddress\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\t\x12-\n\x0bconnections\x18\x05 \x03(\x0b2\x18.PAM.PAMWebRtcConnection"\xa7\x01\n\x13PAMWebRtcConnection\x12\x15\n\rconnectionUid\x18\x01 \x01(\x0c\x12\'\n\x04type\x18\x02 \x01(\x0e2\x19.PAM.WebRtcConnectionType\x12\x11\n\trecordUid\x18\x03 \x01(\x0c\x12\x10\n\x08userName\x18\x04 \x01(\t\x12\x11\n\tstartedOn\x18\x05 \x01(\x03\x12\x18\n\x10configurationUid\x18\x06 \x01(\x0c"Y\n\x14PAMOnlineControllers\x12\x12\n\ndeprecated\x18\x01 \x03(\x0c\x12-\n\x0bcontrollers\x18\x02 \x03(\x0b2\x18.PAM.PAMOnlineController"9\n\x10PAMRotateRequest\x12\x12\n\nrequestUid\x18\x01 \x01(\x0c\x12\x11\n\trecordUid\x18\x02 \x01(\x0c"A\n\x16PAMControllersResponse\x12\'\n\x0bcontrollers\x18\x01 \x03(\x0b2\x12.PAM.PAMController"=\n\x13PAMRemoveController\x12\x15\n\rcontrollerUid\x18\x01 \x01(\x0c\x12\x0f\n\x07message\x18\x02 \x01(\t"L\n\x1bPAMRemoveControllerResponse\x12-\n\x0bcontrollers\x18\x01 \x03(\x0b2\x18.PAM.PAMRemoveController"=\n\x10PAMModifyRequest\x12)\n\noperations\x18\x01 \x03(\x0b2\x15.PAM.PAMDataOperation"\x98\x01\n\x10PAMDataOperation\x12,\n\roperationType\x18\x01 \x01(\x0e2\x15.PAM.PAMOperationType\x120\n\rconfiguration\x18\x02 \x01(\x0b2\x19.PAM.PAMConfigurationData\x12$\n\x07element\x18\x03 \x01(\x0b2\x13.PAM.PAMElementData"e\n\x14PAMConfigurationData\x12\x18\n\x10configurationUid\x18\x01 \x01(\x0c\x12\x0e\n\x06nodeId\x18\x02 \x01(\x03\x12\x15\n\rcontrollerUid\x18\x03 \x01(\x0c\x12\x0c\n\x04data\x18\x04 \x01(\x0c"E\n\x0ePAMElementData\x12\x12\n\nelementUid\x18\x01 \x01(\x0c\x12\x11\n\tparentUid\x18\x02 \x01(\x0c\x12\x0c\n\x04data\x18\x03 \x01(\x0c"m\n\x19PAMElementOperationResult\x12\x12\n\nelementUid\x18\x01 \x01(\x0c\x12+\n\x06result\x18\x02 \x01(\x0e2\x1b.PAM.PAMOperationResultType\x12\x0f\n\x07message\x18\x03 \x01(\t"B\n\x0fPAMModifyResult\x12/\n\x07results\x18\x01 \x03(\x0b2\x1e.PAM.PAMElementOperationResult"x\n\nPAMElement\x12\x12\n\nelementUid\x18\x01 \x01(\x0c\x12\x0c\n\x04data\x18\x02 \x01(\x0c\x12\x0f\n\x07created\x18\x03 \x01(\x03\x12\x14\n\x0clastModified\x18\x04 \x01(\x03\x12!\n\x08children\x18\x05 \x03(\x0b2\x0f.PAM.PAMElement"#\n\x14PAMGenericUidRequest\x12\x0b\n\x03uid\x18\x01 \x01(\x0c"%\n\x15PAMGenericUidsRequest\x12\x0c\n\x04uids\x18\x01 \x03(\x0c"\xab\x01\n\x10PAMConfiguration\x12\x18\n\x10configurationUid\x18\x01 \x01(\x0c\x12\x0e\n\x06nodeId\x18\x02 \x01(\x03\x12\x15\n\rcontrollerUid\x18\x03 \x01(\x0c\x12\x0c\n\x04data\x18\x04 \x01(\x0c\x12\x0f\n\x07created\x18\x05 \x01(\x03\x12\x14\n\x0clastModified\x18\x06 \x01(\x03\x12!\n\x08children\x18\x07 \x03(\x0b2\x0f.PAM.PAMElement"B\n\x11PAMConfigurations\x12-\n\x0econfigurations\x18\x01 \x03(\x0b2\x15.PAM.PAMConfiguration"\xab\x02\n\rPAMController\x12\x15\n\rcontrollerUid\x18\x01 \x01(\x0c\x12\x16\n\x0econtrollerName\x18\x02 \x01(\t\x12\x13\n\x0bdeviceToken\x18\x03 \x01(\t\x12\x12\n\ndeviceName\x18\x04 \x01(\t\x12\x0e\n\x06nodeId\x18\x05 \x01(\x03\x12\x0f\n\x07created\x18\x06 \x01(\x03\x12\x14\n\x0clastModified\x18\x07 \x01(\x03\x12\x16\n\x0eapplicationUid\x18\x08 \x01(\x0c\x120\n\rappClientType\x18\t \x01(\x0e2\x19.Enterprise.AppClientType\x12\x15\n\risInitialized\x18\n \x01(\x08\x12\x18\n\x10maxInstanceCount\x18\x0b \x01(\x05\x12\x10\n\x08lastSeen\x18\x0c \x01(\x03"P\n\x1dPAMSetMaxInstanceCountRequest\x12\x15\n\rcontrollerUid\x18\x01 \x01(\x0c\x12\x18\n\x10maxInstanceCount\x18\x02 \x01(\x05"%\n\x12ControllerResponse\x12\x0f\n\x07payload\x18\x01 \x01(\t"M\n\x1aPAMConfigurationController\x12\x18\n\x10configurationUid\x18\x01 \x01(\x0c\x12\x15\n\rcontrollerUid\x18\x02 \x01(\x0c"\xa3\x01\n\x17ConfigurationAddRequest\x12\x18\n\x10configurationUid\x18\x01 \x01(\x0c\x12\x11\n\trecordKey\x18\x02 \x01(\x0c\x12\x0c\n\x04data\x18\x03 \x01(\x0c\x12(\n\x0brecordLinks\x18\x04 \x03(\x0b2\x13.Records.RecordLink\x12#\n\x05audit\x18\x05 \x01(\x0b2\x14.Records.RecordAudit"J\n\x10RelayAccessCreds\x12\x10\n\x08username\x18\x01 \x01(\t\x12\x10\n\x08password\x18\x02 \x01(\t\x12\x12\n\nserverTime\x18\x03 \x01(\x03"\\\n\x10KturnAccessCreds\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\r\n\x05token\x18\x02 \x01(\x0c\x12\x18\n\x10cert_fingerprint\x18\x03 \x01(\t\x12\x12\n\nexpires_at\x18\x04 \x01(\x04"\x81\x02\n\x14PAMRecordingsRequest\x12\x11\n\trecordUid\x18\x01 \x01(\x0c\x12\x10\n\x08maxCount\x18\x02 \x01(\x05\x12\x17\n\nrangeStart\x18\x03 \x01(\x03H\x00\x88\x01\x01\x12\x15\n\x08rangeEnd\x18\x04 \x01(\x03H\x01\x88\x01\x01\x12$\n\x05types\x18\x05 \x03(\x0e2\x15.PAM.PAMRecordingType\x12)\n\x05risks\x18\x06 \x03(\x0e2\x1a.PAM.PAMRecordingRiskLevel\x12\x11\n\tprotocols\x18\x07 \x03(\t\x12\x14\n\x0ccloseReasons\x18\x08 \x03(\x05B\r\n\x0b_rangeStartB\x0b\n\t_rangeEnd"\x89\x02\n\x1cPAMRecordingsForUsersRequest\x12\x11\n\tusernames\x18\x01 \x03(\t\x12\x10\n\x08maxCount\x18\x02 \x01(\x05\x12\x17\n\nrangeStart\x18\x03 \x01(\x03H\x00\x88\x01\x01\x12\x15\n\x08rangeEnd\x18\x04 \x01(\x03H\x01\x88\x01\x01\x12$\n\x05types\x18\x05 \x03(\x0e2\x15.PAM.PAMRecordingType\x12)\n\x05risks\x18\x06 \x03(\x0e2\x1a.PAM.PAMRecordingRiskLevel\x12\x11\n\tprotocols\x18\x07 \x03(\t\x12\x14\n\x0ccloseReasons\x18\x08 \x03(\x05B\r\n\x0b_rangeStartB\x0b\n\t_rangeEnd"\xd4\x02\n\x0cPAMRecording\x12\x15\n\rconnectionUid\x18\x01 \x01(\x0c\x12,\n\rrecordingType\x18\x02 \x01(\x0e2\x15.PAM.PAMRecordingType\x12\x11\n\trecordUid\x18\x03 \x01(\x0c\x12\x10\n\x08userName\x18\x04 \x01(\t\x12\x11\n\tstartedOn\x18\x05 \x01(\x03\x12\x0e\n\x06length\x18\x06 \x01(\x05\x12\x10\n\x08fileSize\x18\x07 \x01(\x03\x12\x11\n\tcreatedOn\x18\x08 \x01(\x03\x12\x10\n\x08protocol\x18\t \x01(\t\x12\x13\n\x0bcloseReason\x18\n \x01(\x05\x12\x19\n\x11recordingDuration\x18\x0b \x01(\x05\x126\n\x12aiOverallRiskLevel\x18\x0c \x01(\x0e2\x1a.PAM.PAMRecordingRiskLevel\x12\x18\n\x10aiOverallSummary\x18\r \x01(\x0c"O\n\x15PAMRecordingsResponse\x12%\n\nrecordings\x18\x01 \x03(\x0b2\x11.PAM.PAMRecording\x12\x0f\n\x07hasMore\x18\x02 \x01(\x08"*\n\x07PAMData\x12\x0e\n\x06vertex\x18\x01 \x01(\x0c\x12\x0f\n\x07content\x18\x02 \x01(\x0c"\x17\n\x07UidList\x12\x0c\n\x04uids\x18\x01 \x03(\x0c"5\n\x0fPAMServiceNames\x12\x13\n\x0bresourceUid\x18\x01 \x01(\x0c\x12\r\n\x05names\x18\x02 \x01(\x0c"\xb4\x03\n\x11PAMResourceConfig\x12\x11\n\trecordUid\x18\x01 \x01(\x0c\x12\x17\n\nnetworkUid\x18\x02 \x01(\x0cH\x00\x88\x01\x01\x12\x15\n\x08adminUid\x18\x03 \x01(\x0cH\x01\x88\x01\x01\x12\x11\n\x04meta\x18\x04 \x01(\x0cH\x02\x88\x01\x01\x12\x1f\n\x12connectionSettings\x18\x05 \x01(\x0cH\x03\x88\x01\x01\x12\'\n\x0cconnectUsers\x18\x06 \x01(\x0b2\x0c.PAM.UidListH\x04\x88\x01\x01\x12\x16\n\tdomainUid\x18\x07 \x01(\x0cH\x05\x88\x01\x01\x12\x18\n\x0bjitSettings\x18\x08 \x01(\x0cH\x06\x88\x01\x01\x12\x1d\n\x10keeperAiSettings\x18\t \x01(\x0cH\x07\x88\x01\x01\x12\x1b\n\x0eupdateServices\x18\n \x01(\x08H\x08\x88\x01\x01B\r\n\x0b_networkUidB\x0b\n\t_adminUidB\x07\n\x05_metaB\x15\n\x13_connectionSettingsB\x0f\n\r_connectUsersB\x0c\n\n_domainUidB\x0e\n\x0c_jitSettingsB\x13\n\x11_keeperAiSettingsB\x11\n\x0f_updateServices"%\n\x16PAMUniversalSyncFolder\x12\x0b\n\x03uid\x18\x01 \x01(\x0c" \n\x10GitHubRepository\x12\x0c\n\x04name\x18\x01 \x01(\x0c"\xd6\x01\n\x0cGitHubConfig\x12$\n\x05scope\x18\x01 \x01(\x0e2\x10.PAM.GitHubScopeH\x00\x88\x01\x01\x12\r\n\x05owner\x18\x02 \x01(\x0c\x12F\n\x16organizationVisibility\x18\x03 \x01(\x0e2!.PAM.GitHubOrganizationVisibilityH\x01\x88\x01\x01\x12$\n\x05repos\x18\x04 \x03(\x0b2\x15.PAM.GitHubRepositoryB\x08\n\x06_scopeB\x19\n\x17_organizationVisibility"\xe9\x02\n\x16PAMUniversalSyncConfig\x12\x12\n\nnetworkUid\x18\x01 \x01(\x0c\x12\x14\n\x07enabled\x18\x02 \x01(\x08H\x00\x88\x01\x01\x12\x1a\n\rdryRunEnabled\x18\x03 \x01(\x08H\x01\x88\x01\x01\x12,\n\x07folders\x18\x04 \x03(\x0b2\x1b.PAM.PAMUniversalSyncFolder\x12\x19\n\x0csyncIdentity\x18\x05 \x01(\x0cH\x02\x88\x01\x01\x12\x16\n\tvaultName\x18\x06 \x01(\x0cH\x03\x88\x01\x01\x12&\n\x06github\x18\x07 \x01(\x0b2\x11.PAM.GitHubConfigH\x04\x88\x01\x01\x12 \n\x13objectStorageBucket\x18\x08 \x01(\x0cH\x05\x88\x01\x01B\n\n\x08_enabledB\x10\n\x0e_dryRunEnabledB\x0f\n\r_syncIdentityB\x0c\n\n_vaultNameB\t\n\x07_githubB\x16\n\x14_objectStorageBucket"7\n\x11NhiMetricsRequest\x12\x11\n\tstartTime\x18\x01 \x01(\x03\x12\x0f\n\x07endTime\x18\x02 \x01(\x03"\x9c\x02\n\x0ePamUsageByUser\x12\x0e\n\x06userId\x18\x01 \x01(\x05\x12!\n\x19recordRotationScheduledOk\x18\x02 \x01(\x05\x12\x1c\n\x14pamConnectionStarted\x18\x03 \x01(\x05\x12\x18\n\x10pamTunnelStarted\x18\x04 \x01(\x05\x12\x1b\n\x13discoveryJobStarted\x18\x05 \x01(\x05\x12 \n\x18recordRotationOnDemandOk\x18\x06 \x01(\x05\x12"\n\x1apamSessionRecordingStarted\x18\x07 \x01(\x05\x12\x15\n\rpamRbiStarted\x18\x08 \x01(\x05\x12%\n\x1dpamSessionRbiRecordingStarted\x18\t \x01(\x05"p\n\x0eNhiUsageByUser\x12\x0e\n\x06userId\x18\x01 \x01(\x05\x12\x11\n\trotations\x18\x02 \x01(\x05\x12\x0f\n\x07tunnels\x18\x03 \x01(\x05\x12\x13\n\x0bconnections\x18\x04 \x01(\x05\x12\x15\n\rdiscoveryJobs\x18\x05 \x01(\x05"\x84\x02\n\x12NhiMetricsResponse\x12\x14\n\x0centerpriseId\x18\x01 \x01(\x03\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07endTime\x18\x03 \x01(\x03\x12\x18\n\x10uniqueKsmDevices\x18\x04 \x01(\x05\x12\x18\n\x10pamGatewayOnline\x18\x05 \x01(\x05\x12/\n\x0epamUsageByUser\x18\x06 \x03(\x0b2\x13.PAM.PamUsageByUserB\x02\x18\x01\x12\x10\n\x08nhiCount\x18\x07 \x01(\x05\x12\x13\n\x0bksmNhiCount\x18\x08 \x01(\x05\x12(\n\x0busageByUser\x18\t \x03(\x0b2\x13.PAM.NhiUsageByUser"D\n\x16NhiBulkMetricsResponse\x12*\n\tresponses\x18\x01 \x03(\x0b2\x17.PAM.NhiMetricsResponse"^\n\x0bNhiUidEntry\x12\x0b\n\x03uid\x18\x01 \x01(\t\x12"\n\x08category\x18\x02 \x01(\x0e2\x10.PAM.NhiCategory\x12\x0e\n\x06ksmNhi\x18\x03 \x01(\x08\x12\x0e\n\x06appUid\x18\x04 \x01(\t"7\n\x11GetNhiUidsRequest\x12\x11\n\tstartTime\x18\x01 \x01(\x03\x12\x0f\n\x07endTime\x18\x02 \x01(\x03"4\n\x12GetNhiUidsResponse\x12\x1e\n\x04uids\x18\x01 \x03(\x0b2\x10.PAM.NhiUidEntry"6\n\x1dSetNhiKsmEffectiveDateRequest\x12\x15\n\reffectiveDate\x18\x01 \x01(\x03"L\n\x1eGetNhiKsmEffectiveDateResponse\x12\x15\n\reffectiveDate\x18\x01 \x01(\x03\x12\x13\n\x0bdefaultDate\x18\x02 \x01(\x03"I\n\x1fPAMUniversalSyncPreCheckRequest\x12\x12\n\nnetworkUid\x18\x01 \x01(\x0c\x12\x12\n\nfolderUids\x18\x02 \x03(\x0c"C\n\x1ePAMUniversalSyncPreCheckResult\x12\x11\n\tfolderUid\x18\x01 \x01(\x0c\x12\x0e\n\x06isUsed\x18\x02 \x01(\x08"X\n PAMUniversalSyncPreCheckResponse\x124\n\x07results\x18\x01 \x03(\x0b2#.PAM.PAMUniversalSyncPreCheckResult*\x9e\x01\n\x14WebRtcConnectionType\x12\x0e\n\nCONNECTION\x10\x00\x12\n\n\x06TUNNEL\x10\x01\x12\x07\n\x03SSH\x10\x02\x12\x07\n\x03RDP\x10\x03\x12\x08\n\x04HTTP\x10\x04\x12\x07\n\x03VNC\x10\x05\x12\n\n\x06TELNET\x10\x06\x12\t\n\x05MYSQL\x10\x07\x12\x0e\n\nSQL_SERVER\x10\x08\x12\x0e\n\nPOSTGRESQL\x10\t\x12\x0e\n\nKUBERNETES\x10\n*@\n\x10PAMOperationType\x12\x07\n\x03ADD\x10\x00\x12\n\n\x06UPDATE\x10\x01\x12\x0b\n\x07REPLACE\x10\x02\x12\n\n\x06DELETE\x10\x03*p\n\x16PAMOperationResultType\x12\x0f\n\x0bPOT_SUCCESS\x10\x00\x12\x15\n\x11POT_UNKNOWN_ERROR\x10\x01\x12\x16\n\x12POT_ALREADY_EXISTS\x10\x02\x12\x16\n\x12POT_DOES_NOT_EXIST\x10\x03*\xc9\x01\n\x15ControllerMessageType\x12\x0f\n\x0bCMT_GENERAL\x10\x00\x12\x0e\n\nCMT_ROTATE\x10\x01\x12\x11\n\rCMT_DISCOVERY\x10\x02\x12\x0f\n\x0bCMT_CONNECT\x10\x03\x12\x19\n\x15CMT_ANALYZE_RECORDING\x10\x04\x12!\n\x1dCMT_WORKFLOW_ACCESS_ELEVATION\x10\x05\x12\x0b\n\x07CMT_USS\x10\x06\x12\x0c\n\x08CMT_INFO\x10\x07\x12\x12\n\x0eCMT_AUTOMATION\x10\x08*V\n\x10PAMRecordingType\x12\x0f\n\x0bPRT_SESSION\x10\x00\x12\x12\n\x0ePRT_TYPESCRIPT\x10\x01\x12\x0c\n\x08PRT_TIME\x10\x02\x12\x0f\n\x0bPRT_SUMMARY\x10\x03*i\n\x15PAMRecordingRiskLevel\x12\x13\n\x0fPRR_UNSPECIFIED\x10\x00\x12\x0b\n\x07PRR_LOW\x10\x01\x12\x0e\n\nPRR_MEDIUM\x10\x02\x12\x0c\n\x08PRR_HIGH\x10\x03\x12\x10\n\x0cPRR_CRITICAL\x10\x04*/\n\x0bGitHubScope\x12\x0e\n\nREPOSITORY\x10\x00\x12\x10\n\x0cORGANIZATION\x10\x01*B\n\x1cGitHubOrganizationVisibility\x12\x07\n\x03ALL\x10\x00\x12\x0b\n\x07PRIVATE\x10\x01\x12\x0c\n\x08SELECTED\x10\x02*`\n\x0bNhiCategory\x12\x18\n\x14NHI_CATEGORY_UNKNOWN\x10\x00\x12\x0c\n\x08PAM_USER\x10\x01\x12\x10\n\x0cPAM_RESOURCE\x10\x02\x12\x0b\n\x07GATEWAY\x10\x03\x12\n\n\x06DEVICE\x10\x04B\x1f\n\x18com.keepersecurity.protoB\x03PAMb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -36,24 +36,24 @@ _globals['DESCRIPTOR']._serialized_options = b'\n\030com.keepersecurity.protoB\003PAM' _globals['_NHIMETRICSRESPONSE'].fields_by_name['pamUsageByUser']._loaded_options = None _globals['_NHIMETRICSRESPONSE'].fields_by_name['pamUsageByUser']._serialized_options = b'\030\001' - _globals['_WEBRTCCONNECTIONTYPE']._serialized_start=6267 - _globals['_WEBRTCCONNECTIONTYPE']._serialized_end=6425 - _globals['_PAMOPERATIONTYPE']._serialized_start=6427 - _globals['_PAMOPERATIONTYPE']._serialized_end=6491 - _globals['_PAMOPERATIONRESULTTYPE']._serialized_start=6493 - _globals['_PAMOPERATIONRESULTTYPE']._serialized_end=6605 - _globals['_CONTROLLERMESSAGETYPE']._serialized_start=6608 - _globals['_CONTROLLERMESSAGETYPE']._serialized_end=6809 - _globals['_PAMRECORDINGTYPE']._serialized_start=6811 - _globals['_PAMRECORDINGTYPE']._serialized_end=6897 - _globals['_PAMRECORDINGRISKLEVEL']._serialized_start=6899 - _globals['_PAMRECORDINGRISKLEVEL']._serialized_end=7004 - _globals['_GITHUBSCOPE']._serialized_start=7006 - _globals['_GITHUBSCOPE']._serialized_end=7053 - _globals['_GITHUBORGANIZATIONVISIBILITY']._serialized_start=7055 - _globals['_GITHUBORGANIZATIONVISIBILITY']._serialized_end=7121 - _globals['_NHICATEGORY']._serialized_start=7123 - _globals['_NHICATEGORY']._serialized_end=7219 + _globals['_WEBRTCCONNECTIONTYPE']._serialized_start=6325 + _globals['_WEBRTCCONNECTIONTYPE']._serialized_end=6483 + _globals['_PAMOPERATIONTYPE']._serialized_start=6485 + _globals['_PAMOPERATIONTYPE']._serialized_end=6549 + _globals['_PAMOPERATIONRESULTTYPE']._serialized_start=6551 + _globals['_PAMOPERATIONRESULTTYPE']._serialized_end=6663 + _globals['_CONTROLLERMESSAGETYPE']._serialized_start=6666 + _globals['_CONTROLLERMESSAGETYPE']._serialized_end=6867 + _globals['_PAMRECORDINGTYPE']._serialized_start=6869 + _globals['_PAMRECORDINGTYPE']._serialized_end=6955 + _globals['_PAMRECORDINGRISKLEVEL']._serialized_start=6957 + _globals['_PAMRECORDINGRISKLEVEL']._serialized_end=7062 + _globals['_GITHUBSCOPE']._serialized_start=7064 + _globals['_GITHUBSCOPE']._serialized_end=7111 + _globals['_GITHUBORGANIZATIONVISIBILITY']._serialized_start=7113 + _globals['_GITHUBORGANIZATIONVISIBILITY']._serialized_end=7179 + _globals['_NHICATEGORY']._serialized_start=7181 + _globals['_NHICATEGORY']._serialized_end=7277 _globals['_PAMROTATIONSCHEDULE']._serialized_start=51 _globals['_PAMROTATIONSCHEDULE']._serialized_end=182 _globals['_PAMROTATIONSCHEDULESRESPONSE']._serialized_start=184 @@ -131,31 +131,31 @@ _globals['_GITHUBCONFIG']._serialized_start=4378 _globals['_GITHUBCONFIG']._serialized_end=4592 _globals['_PAMUNIVERSALSYNCCONFIG']._serialized_start=4595 - _globals['_PAMUNIVERSALSYNCCONFIG']._serialized_end=4898 - _globals['_NHIMETRICSREQUEST']._serialized_start=4900 - _globals['_NHIMETRICSREQUEST']._serialized_end=4955 - _globals['_PAMUSAGEBYUSER']._serialized_start=4958 - _globals['_PAMUSAGEBYUSER']._serialized_end=5242 - _globals['_NHIUSAGEBYUSER']._serialized_start=5244 - _globals['_NHIUSAGEBYUSER']._serialized_end=5356 - _globals['_NHIMETRICSRESPONSE']._serialized_start=5359 - _globals['_NHIMETRICSRESPONSE']._serialized_end=5619 - _globals['_NHIBULKMETRICSRESPONSE']._serialized_start=5621 - _globals['_NHIBULKMETRICSRESPONSE']._serialized_end=5689 - _globals['_NHIUIDENTRY']._serialized_start=5691 - _globals['_NHIUIDENTRY']._serialized_end=5785 - _globals['_GETNHIUIDSREQUEST']._serialized_start=5787 - _globals['_GETNHIUIDSREQUEST']._serialized_end=5842 - _globals['_GETNHIUIDSRESPONSE']._serialized_start=5844 - _globals['_GETNHIUIDSRESPONSE']._serialized_end=5896 - _globals['_SETNHIKSMEFFECTIVEDATEREQUEST']._serialized_start=5898 - _globals['_SETNHIKSMEFFECTIVEDATEREQUEST']._serialized_end=5952 - _globals['_GETNHIKSMEFFECTIVEDATERESPONSE']._serialized_start=5954 - _globals['_GETNHIKSMEFFECTIVEDATERESPONSE']._serialized_end=6030 - _globals['_PAMUNIVERSALSYNCPRECHECKREQUEST']._serialized_start=6032 - _globals['_PAMUNIVERSALSYNCPRECHECKREQUEST']._serialized_end=6105 - _globals['_PAMUNIVERSALSYNCPRECHECKRESULT']._serialized_start=6107 - _globals['_PAMUNIVERSALSYNCPRECHECKRESULT']._serialized_end=6174 - _globals['_PAMUNIVERSALSYNCPRECHECKRESPONSE']._serialized_start=6176 - _globals['_PAMUNIVERSALSYNCPRECHECKRESPONSE']._serialized_end=6264 + _globals['_PAMUNIVERSALSYNCCONFIG']._serialized_end=4956 + _globals['_NHIMETRICSREQUEST']._serialized_start=4958 + _globals['_NHIMETRICSREQUEST']._serialized_end=5013 + _globals['_PAMUSAGEBYUSER']._serialized_start=5016 + _globals['_PAMUSAGEBYUSER']._serialized_end=5300 + _globals['_NHIUSAGEBYUSER']._serialized_start=5302 + _globals['_NHIUSAGEBYUSER']._serialized_end=5414 + _globals['_NHIMETRICSRESPONSE']._serialized_start=5417 + _globals['_NHIMETRICSRESPONSE']._serialized_end=5677 + _globals['_NHIBULKMETRICSRESPONSE']._serialized_start=5679 + _globals['_NHIBULKMETRICSRESPONSE']._serialized_end=5747 + _globals['_NHIUIDENTRY']._serialized_start=5749 + _globals['_NHIUIDENTRY']._serialized_end=5843 + _globals['_GETNHIUIDSREQUEST']._serialized_start=5845 + _globals['_GETNHIUIDSREQUEST']._serialized_end=5900 + _globals['_GETNHIUIDSRESPONSE']._serialized_start=5902 + _globals['_GETNHIUIDSRESPONSE']._serialized_end=5954 + _globals['_SETNHIKSMEFFECTIVEDATEREQUEST']._serialized_start=5956 + _globals['_SETNHIKSMEFFECTIVEDATEREQUEST']._serialized_end=6010 + _globals['_GETNHIKSMEFFECTIVEDATERESPONSE']._serialized_start=6012 + _globals['_GETNHIKSMEFFECTIVEDATERESPONSE']._serialized_end=6088 + _globals['_PAMUNIVERSALSYNCPRECHECKREQUEST']._serialized_start=6090 + _globals['_PAMUNIVERSALSYNCPRECHECKREQUEST']._serialized_end=6163 + _globals['_PAMUNIVERSALSYNCPRECHECKRESULT']._serialized_start=6165 + _globals['_PAMUNIVERSALSYNCPRECHECKRESULT']._serialized_end=6232 + _globals['_PAMUNIVERSALSYNCPRECHECKRESPONSE']._serialized_start=6234 + _globals['_PAMUNIVERSALSYNCPRECHECKRESPONSE']._serialized_end=6322 # @@protoc_insertion_point(module_scope) diff --git a/keepercommander/proto/pam_pb2.pyi b/keepercommander/proto/pam_pb2.pyi index b83fdc735..5b5a29ce3 100644 --- a/keepercommander/proto/pam_pb2.pyi +++ b/keepercommander/proto/pam_pb2.pyi @@ -555,7 +555,7 @@ class GitHubConfig(_message.Message): def __init__(self, scope: _Optional[_Union[GitHubScope, str]] = ..., owner: _Optional[bytes] = ..., organizationVisibility: _Optional[_Union[GitHubOrganizationVisibility, str]] = ..., repos: _Optional[_Iterable[_Union[GitHubRepository, _Mapping]]] = ...) -> None: ... class PAMUniversalSyncConfig(_message.Message): - __slots__ = ("networkUid", "enabled", "dryRunEnabled", "folders", "syncIdentity", "vaultName", "github") + __slots__ = ("networkUid", "enabled", "dryRunEnabled", "folders", "syncIdentity", "vaultName", "github", "objectStorageBucket") NETWORKUID_FIELD_NUMBER: _ClassVar[int] ENABLED_FIELD_NUMBER: _ClassVar[int] DRYRUNENABLED_FIELD_NUMBER: _ClassVar[int] @@ -563,6 +563,7 @@ class PAMUniversalSyncConfig(_message.Message): SYNCIDENTITY_FIELD_NUMBER: _ClassVar[int] VAULTNAME_FIELD_NUMBER: _ClassVar[int] GITHUB_FIELD_NUMBER: _ClassVar[int] + OBJECTSTORAGEBUCKET_FIELD_NUMBER: _ClassVar[int] networkUid: bytes enabled: bool dryRunEnabled: bool @@ -570,7 +571,8 @@ class PAMUniversalSyncConfig(_message.Message): syncIdentity: bytes vaultName: bytes github: GitHubConfig - def __init__(self, networkUid: _Optional[bytes] = ..., enabled: _Optional[bool] = ..., dryRunEnabled: _Optional[bool] = ..., folders: _Optional[_Iterable[_Union[PAMUniversalSyncFolder, _Mapping]]] = ..., syncIdentity: _Optional[bytes] = ..., vaultName: _Optional[bytes] = ..., github: _Optional[_Union[GitHubConfig, _Mapping]] = ...) -> None: ... + objectStorageBucket: bytes + def __init__(self, networkUid: _Optional[bytes] = ..., enabled: _Optional[bool] = ..., dryRunEnabled: _Optional[bool] = ..., folders: _Optional[_Iterable[_Union[PAMUniversalSyncFolder, _Mapping]]] = ..., syncIdentity: _Optional[bytes] = ..., vaultName: _Optional[bytes] = ..., github: _Optional[_Union[GitHubConfig, _Mapping]] = ..., objectStorageBucket: _Optional[bytes] = ...) -> None: ... class NhiMetricsRequest(_message.Message): __slots__ = ("startTime", "endTime") diff --git a/keepercommander/sanitization.py b/keepercommander/sanitization.py new file mode 100644 index 000000000..22df619a6 --- /dev/null +++ b/keepercommander/sanitization.py @@ -0,0 +1,71 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | ' Union[str, list, dict]: + """Mask a Keeper record field's `value`, preserving its container shape.""" + if isinstance(value, list): + return ['***' for _ in value] + if isinstance(value, dict): + return {k: '***' for k in value} + return '***' + + +def sanitize_nested_data(data: Any) -> Any: + """Recursively sanitize nested data structures for logging.""" + if isinstance(data, dict): + field_type = data.get('type') + if isinstance(field_type, str) and field_type.lower() in SENSITIVE_FIELD_TYPES and 'value' in data: + sanitized = dict(data) + sanitized['value'] = mask_field_value(data['value']) + return sanitized + + sanitized = {} + for key, value in data.items(): + if key.lower() in SENSITIVE_DICT_KEYS: + if isinstance(value, str) and len(value) > 0: + sanitized[key] = '*' * min(len(value), 15) + else: + sanitized[key] = '***' + else: + sanitized[key] = sanitize_nested_data(value) + return sanitized + elif isinstance(data, list): + return [sanitize_nested_data(item) for item in data] + else: + return data + + +def sanitize_protobuf_json(json_str: str) -> str: + """Sanitize sensitive data from protobuf JSON before logging.""" + try: + data = json.loads(json_str) + sanitized = sanitize_nested_data(data) + return json.dumps(sanitized) + except (json.JSONDecodeError, TypeError): + return json_str diff --git a/keepercommander/service/config/cloudflare_config.py b/keepercommander/service/config/cloudflare_config.py index ea9fda01c..a6a80d871 100644 --- a/keepercommander/service/config/cloudflare_config.py +++ b/keepercommander/service/config/cloudflare_config.py @@ -16,7 +16,7 @@ from ..decorators.logging import logger, debug_decorator from .service_config import ServiceConfig -from ..util.tunneling import generate_cloudflare_url +from ..util.tunneling import generate_cloudflare_url, get_tunnel_log_file from ..util.exceptions import ValidationError class CloudflareConfigurator: @@ -86,8 +86,7 @@ def _check_cloudflare_tunnel_health(max_wait_seconds=None): @staticmethod def _get_cloudflare_log_path() -> str: """Get the path to the Cloudflare tunnel log file.""" - service_core_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "core") - return os.path.join(service_core_dir, "logs", "cloudflare_tunnel_subprocess.log") + return get_tunnel_log_file("cloudflare_tunnel_subprocess.log") @staticmethod def _analyze_tunnel_log(log_file: str) -> Tuple[Optional[bool], str]: diff --git a/keepercommander/service/core/service_app.py b/keepercommander/service/core/service_app.py index 051df174a..3d182a5ef 100644 --- a/keepercommander/service/core/service_app.py +++ b/keepercommander/service/core/service_app.py @@ -9,17 +9,33 @@ # Contact: ops@keepersecurity.com # -from ...service.app import create_app -from ...service.config.service_config import ServiceConfig -from ...service.core.service_manager import ServiceManager +import sys -flask_app = create_app() +# Argv flag (not an env var, which could leak into subprocesses) signaling a frozen background service. +SERVICE_MODE_FLAG = '--internal-run-service' -if __name__ == '__main__': +def run_background_service(): + """ + Run the Flask service in background mode. + This function is called both when running as a module (-m) and + when the frozen executable is invoked with SERVICE_MODE_FLAG. + """ + # PyInstaller's bootloader doesn't reliably honor PYTHONUNBUFFERED, so force it here too. + for stream in (sys.stdout, sys.stderr): + if stream is not None and hasattr(stream, 'reconfigure'): + stream.reconfigure(line_buffering=True) + + from ...service.app import create_app + from ...service.config.service_config import ServiceConfig + from ...service.core.service_manager import ServiceManager + + flask_app = create_app() + service_config = ServiceConfig() config_data = service_config.load_config() + # Pre-load Keeper parameters for background mode try: from ...service.core.globals import ensure_params_loaded print("Pre-loading Keeper parameters for background mode...") @@ -29,10 +45,10 @@ print(f"Warning: Failed to pre-load parameters during startup: {e}") print("Parameters will be loaded on first API call if needed") - ssl_context = None - - if not (port := config_data.get("port")): + port = config_data.get("port") + if not port: print("Error: Service configuration is incomplete. Please configure the service port in service_config") + sys.exit(1) ssl_context = ServiceManager.get_ssl_context(config_data) @@ -42,3 +58,7 @@ ssl_context=ssl_context ) + +if __name__ == '__main__': + run_background_service() + diff --git a/keepercommander/service/core/service_manager.py b/keepercommander/service/core/service_manager.py index 0d910237c..d65d0c449 100644 --- a/keepercommander/service/core/service_manager.py +++ b/keepercommander/service/core/service_manager.py @@ -17,6 +17,7 @@ from ... import utils from ...service.config.service_config import ServiceConfig from ..decorators.logging import logger, debug_decorator +from ..util.process_util import spawn_detached_process, CREATE_NO_WINDOW from .process_info import ProcessInfo from .terminal_handler import TerminalHandler from .signal_handler import SignalHandler @@ -92,7 +93,12 @@ def start_service(cls) -> None: else: print(f"Commander Service starting on \033[1m{protocol}://localhost:{port}/api/v1/executecommand\033[0m") - ngrok_pid = NgrokConfigurator.configure_ngrok(config_data, service_config) + try: + ngrok_pid = NgrokConfigurator.configure_ngrok(config_data, service_config) + except Exception as e: + ProcessInfo.clear() + logger.error(f"\n{str(e)}") + return cloudflare_pid = None try: @@ -110,7 +116,7 @@ def start_service(cls) -> None: ProcessInfo.clear() - logger.info(f"\n{str(e)}") + logger.error(f"\n{str(e)}") return # Custom logging filter to replace SSL handshake errors with user-friendly message @@ -130,43 +136,41 @@ def filter(self, record): werkzeug_logger.addFilter(SSLHandshakeFilter()) if config_data.get("run_mode") == "background": - - base_dir = os.path.dirname(os.path.abspath(__file__)) - service_module = "keepercommander.service.core.service_app" # Use module path instead of file path - python_executable = sys.executable - - # Create logs directory for subprocess output - log_dir = os.path.join(base_dir, "logs") + + # Detect if running as PyInstaller executable + is_frozen = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') + + # Create logs directory for subprocess output in user-writable location + log_dir = os.path.join(utils.get_default_path(), "service_logs") os.makedirs(log_dir, exist_ok=True) log_file = os.path.join(log_dir, "service_subprocess.log") try: - if sys.platform == "win32": - subprocess.DETACHED_PROCESS = 0x00000008 - with open(log_file, 'w') as log_f: - cls = subprocess.Popen( - [python_executable, '-m', service_module], - creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP, - stdout=log_f, - stderr=subprocess.STDOUT, # Combine stderr with stdout - cwd=os.getcwd(), # Use current working directory to access config files - env=os.environ.copy() # Inherit environment variables - ) + python_executable = sys.executable + + # Set up environment for subprocess + subprocess_env = os.environ.copy() + # Redirected (non-TTY) stdout defaults to full buffering, delaying log output. + subprocess_env['PYTHONUNBUFFERED'] = '1' + + if is_frozen: + # -m doesn't work for a frozen exe, so pass an explicit internal flag instead. + from .service_app import SERVICE_MODE_FLAG + cmd = [python_executable, SERVICE_MODE_FLAG] else: - # For macOS and Linux - improved subprocess handling - with open(log_file, 'w') as log_f: - cls = subprocess.Popen( - [python_executable, '-m', service_module], - stdout=log_f, - stderr=subprocess.STDOUT, # Combine stderr with stdout - preexec_fn=os.setpgrp, - cwd=os.getcwd(), # Use current working directory to access config files - env=os.environ.copy() # Inherit environment variables - ) - + # Running as Python script - use -m flag + cmd = [python_executable, '-m', 'keepercommander.service.core.service_app'] + + # append=True to preserve history across restarts (tunnel logs truncate instead). + process = spawn_detached_process( + cmd, log_file, cwd=os.getcwd(), env=subprocess_env, append=True + ) + # Command output can include vault data - don't leave it world-readable. + utils.set_file_permissions(log_file) + logger.debug(f"Service subprocess logs available at: {log_file}") - print(f"Commander Service started with PID: {cls.pid}") - ProcessInfo.save(cls.pid, is_running, ngrok_pid) + print(f"Commander Service started with PID: {process.pid}") + ProcessInfo.save(process.pid, is_running, ngrok_pid, cloudflare_pid) except Exception as e: logger.error(f"Failed to start service subprocess: {e}") @@ -257,7 +261,7 @@ def foreground_signal_handler(signum, frame): cls._flask_app = create_app() cls._is_running = True - ProcessInfo.save(os.getpid(), is_running, ngrok_pid) + ProcessInfo.save(os.getpid(), is_running, ngrok_pid, cloudflare_pid) ssl_context = ServiceManager.get_ssl_context(config_data) try: @@ -268,9 +272,6 @@ def foreground_signal_handler(signum, frame): ) finally: cleanup_cloudflare_on_foreground_exit() - - # Save the process ID for future reference - ProcessInfo.save(cls.pid, is_running, ngrok_pid, cloudflare_pid) except FileNotFoundError: logging.info("Error: Service configuration file not found. Please use 'service-create' command to create a service_config file.") @@ -459,7 +460,13 @@ def kill_process_by_pid(pid: int): try: if sys.platform.startswith("win"): # Windows logger.debug(f"Using Windows taskkill for PID {pid}") - subprocess.run(["taskkill", "/PID", str(pid), "/F"], check=True) + subprocess.run( + ["taskkill", "/PID", str(pid), "/F"], + check=True, + creationflags=CREATE_NO_WINDOW, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) return True else: # Linux & macOS try: diff --git a/keepercommander/service/decorators/api_logging.py b/keepercommander/service/decorators/api_logging.py index d06ae84f2..74ee593d2 100644 --- a/keepercommander/service/decorators/api_logging.py +++ b/keepercommander/service/decorators/api_logging.py @@ -14,11 +14,8 @@ from typing import Callable, Any from flask import request import time -from .logging import logger, sanitize_command_fields, SENSITIVE_FIELD_TYPES - -# Legacy generic keys (kept for JSON payloads that aren't shaped like Keeper -# record fields, e.g. arbitrary nested config blobs). -_SENSITIVE_DICT_KEYS = frozenset({'password', 'login', 'secret', 'token', 'key'}) | SENSITIVE_FIELD_TYPES +from .logging import logger, sanitize_command_fields +from ...sanitization import sanitize_nested_data class SSLHandshakeFilter(logging.Filter): @@ -44,48 +41,9 @@ def sanitize_password_in_command(data): # Sanitize filedata if present if 'filedata' in sanitized: - sanitized['filedata'] = _sanitize_nested_data(sanitized['filedata']) - - return sanitized - -def _mask_field_value(value): - """Mask a Keeper record field's `value`, preserving its container shape.""" - if isinstance(value, list): - return ['***' for _ in value] - if isinstance(value, dict): - return {k: '***' for k in value} - return '***' - + sanitized['filedata'] = sanitize_nested_data(sanitized['filedata']) -def _sanitize_nested_data(data): - """Recursively sanitize nested data structures""" - if isinstance(data, dict): - field_type = data.get('type') - if isinstance(field_type, str) and field_type.lower() in SENSITIVE_FIELD_TYPES and 'value' in data: - sanitized = dict(data) - sanitized['value'] = _mask_field_value(data['value']) - return sanitized - - sanitized = {} - for key, value in data.items(): - # Sanitize sensitive field names - if key.lower() in _SENSITIVE_DICT_KEYS: - if isinstance(value, str) and len(value) > 0: - sanitized[key] = '*' * min(len(value), 15) - else: - sanitized[key] = '***' - else: - sanitized[key] = _sanitize_nested_data(value) - return sanitized - elif isinstance(data, list): - return [_sanitize_nested_data(item) for item in data] - elif isinstance(data, str): - # Sanitize email addresses in string values to protect PII - import re - sanitized_str = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '***@***.***', data) - return sanitized_str - else: - return data + return sanitized def _get_sanitized_request_data(): """Extract and sanitize request data for logging (only for JSON POST requests)""" @@ -94,6 +52,9 @@ def _get_sanitized_request_data(): try: json_data = request.get_json(silent=True) sanitized_data = sanitize_password_in_command(json_data) + # Additional sanitization for nested structures that might contain sensitive data + if sanitized_data and isinstance(sanitized_data, dict): + sanitized_data = sanitize_nested_data(sanitized_data) except Exception: sanitized_data = None return f"data={sanitized_data}" if sanitized_data else "no-data" diff --git a/keepercommander/service/decorators/logging.py b/keepercommander/service/decorators/logging.py index 7542aeb0f..44b4f8825 100644 --- a/keepercommander/service/decorators/logging.py +++ b/keepercommander/service/decorators/logging.py @@ -17,14 +17,7 @@ import shlex from enum import Enum from ... import utils - -# Values that must never reach the logs when set via record-add/record-update/ -# nsf-record-* CLI args. -SENSITIVE_FIELD_TYPES = frozenset({ - 'password', 'login', 'secret', 'onetimecode', 'pincode', 'keypair', - 'privatekey', 'passphrase', 'paymentcard', 'bankaccount', - 'securityquestion', 'passkey', -}) +from ...sanitization import SENSITIVE_FIELD_TYPES class LogLevel(Enum): ERROR = logging.ERROR @@ -68,7 +61,7 @@ def _creat_logging_config(self): return default_config["logging"] def _load_config(self): - config_path = utils.get_default_path() / "logging_config.yaml"; + config_path = utils.get_default_path() / "logging_config.yaml" # config_path = os.getenv("LOGGING_CONFIG_PATH", "logging_config.yaml") if os.path.exists(config_path): @@ -150,24 +143,24 @@ def sanitize_debug_data(data: str) -> str: """Sanitize sensitive data from debug output.""" if not data: return data - + sanitized = data - + # Sanitize common password patterns patterns = [ (r'"password"\s*:\s*"[^"]*"', '"password": "***"'), - (r'"login"\s*:\s*"[^"]*"', '"login": "***"'), (r'"secret"\s*:\s*"[^"]*"', '"secret": "***"'), (r'"token"\s*:\s*"[^"]*"', '"token": "***"'), (r'"key"\s*:\s*"[^"]*"', '"key": "***"'), + (r'"licenseNumber"\s*:\s*"[^"]*"', '"licenseNumber": "***"'), + (r'"encryptedNote"\s*:\s*"[^"]*"', '"encryptedNote": "***"'), + (r'"note"\s*:\s*"[^"]*"', '"note": "***"'), + # Bare field formats (e.g., password=value, secret=value) (r'\bpassword=[^\s]*', 'password=***'), - (r'\blogin=[^\s]*', 'login=***'), - # oneTimeCode=otpauth://totp/...?secret=... — mask the whole value, TOTP seed included (r'\boneTimeCode=[^\s]*', 'oneTimeCode=***'), (r'\bsecret=[^\s]*', 'secret=***'), - # Other sensitive record field types (see SENSITIVE_FIELD_TYPES) that can - # appear as bare CLI args on record-add/record-update/nsf-* commands. (r'\bpinCode=[^\s]*', 'pinCode=***'), + (r'\bnote=[^\s]*', 'note=***'), (r'\bkeyPair=[^\s]*', 'keyPair=***'), (r'\bprivateKey=[^\s]*', 'privateKey=***'), (r'\bpassphrase=[^\s]*', 'passphrase=***'), @@ -175,10 +168,39 @@ def sanitize_debug_data(data: str) -> str: (r'\bbankAccount=[^\s]*', 'bankAccount=***'), (r'\bsecurityQuestion=[^\s]*', 'securityQuestion=***'), (r'\bpasskey=[^\s]*', 'passkey=***'), - # Sanitize email addresses in logs to protect PII - (r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '***@***.***'), + (r'\blicenseNumber=[^\s]*', 'licenseNumber=***'), + (r'\bencryptedNote=[^\s]*', 'encryptedNote=***'), + # Command options (--password, etc) + (r'--password=[^\s]*', '--password=***'), + (r'--password\s+[^\s]+', '--password ***'), + # Prefixed field formats (f.fieldName=value, c.fieldName=value) + (r'\bf\.bankAccount\.accountNumber=[^\s]*', 'f.bankAccount.accountNumber=***'), + (r'\bf\.bankAccount\.routingNumber=[^\s]*', 'f.bankAccount.routingNumber=***'), + (r'\bc\.bankAccount\.accountNumber=[^\s]*', 'c.bankAccount.accountNumber=***'), + (r'\bc\.bankAccount\.routingNumber=[^\s]*', 'c.bankAccount.routingNumber=***'), + (r'\bf\.paymentCard\.cardNumber=[^\s]*', 'f.paymentCard.cardNumber=***'), + (r'\bf\.paymentCard\.cardSecurityCode=[^\s]*', 'f.paymentCard.cardSecurityCode=***'), + (r'\bc\.paymentCard\.cardNumber=[^\s]*', 'c.paymentCard.cardNumber=***'), + (r'\bc\.paymentCard\.cardSecurityCode=[^\s]*', 'c.paymentCard.cardSecurityCode=***'), + (r'\bf\.keyPair\.privateKey=[^\s]*', 'f.keyPair.privateKey=***'), + (r'\bf\.keyPair\.publicKey=[^\s]*', 'f.keyPair.publicKey=***'), + (r'\bc\.keyPair\.privateKey=[^\s]*', 'c.keyPair.privateKey=***'), + (r'\bc\.keyPair\.publicKey=[^\s]*', 'c.keyPair.publicKey=***'), + (r'\bf\.licenseNumber=[^\s]*', 'f.licenseNumber=***'), + (r'\bc\.licenseNumber=[^\s]*', 'c.licenseNumber=***'), + (r'\bf\.encryptedNote=[^\s]*', 'f.encryptedNote=***'), + (r'\bc\.encryptedNote=[^\s]*', 'c.encryptedNote=***'), + # Custom field labels with sensitive types (f.password.Label=, c.secret.Label=, etc.) + (r'\b[fc]\.password\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.secret\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.keypair\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.privatekey\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.bankaccount\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.paymentcard\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.licensenumber\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), + (r'\b[fc]\.encryptednote\.[^=]+=\S*', lambda m: m.group(0).split('=')[0] + '=***'), ] - + for pattern, replacement in patterns: sanitized = re.sub(pattern, replacement, sanitized, flags=re.IGNORECASE) diff --git a/keepercommander/service/util/process_util.py b/keepercommander/service/util/process_util.py new file mode 100644 index 000000000..d02fffac4 --- /dev/null +++ b/keepercommander/service/util/process_util.py @@ -0,0 +1,46 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | ' Optional[Tuple]: logger.info("Request validation failed: Content-Type must be application/json") return jsonify({"status": "error", "error": "Content-Type must be application/json"}), 400 - if not request.json: - logger.info("Request validation failed: Invalid or empty JSON") - return jsonify({"status": "error", "error": "Invalid or empty JSON"}), 400 - + try: + json_data = request.get_json(force=True, silent=False) + if not json_data: + logger.info("Request validation failed: Invalid or empty JSON") + return jsonify({"status": "error", "error": "Invalid or empty JSON"}), 400 + except (BadRequest, ValueError) as e: + # The parser's message can include a payload excerpt - log it, don't echo it back. + logger.warning(f"Request validation failed: JSON parsing error - {e}") + return jsonify({"status": "error", "error": "Invalid JSON format"}), 400 + return None diff --git a/keepercommander/service/util/tunneling.py b/keepercommander/service/util/tunneling.py index afc8c95d7..e3bf2dca3 100644 --- a/keepercommander/service/util/tunneling.py +++ b/keepercommander/service/util/tunneling.py @@ -20,13 +20,28 @@ import json import tempfile +from ... import utils +from .process_util import spawn_detached_process + +def get_tunnel_log_file(name): + # Resolved per call, not cached at import time, so a later --data-dir override is respected. + log_dir = os.path.join(utils.get_default_path(), "service_logs") + os.makedirs(log_dir, exist_ok=True) + return os.path.join(log_dir, name) + + +# DOA check only catches immediate crashes; a slower failure (bad auth) is caught later below. +NGROK_STARTUP_CHECK_DELAY_SECONDS = 0.5 + def start_ngrok(port, auth_token=None, subdomain=None): """ Start ngrok as a fully detached subprocess and return the PID. """ - ngrok_cmd = ["ngrok", "http", str(port), "--log=stdout", "--log-level=info"] - + ngrok_config = conf.get_default() + ngrok.install_ngrok(ngrok_config) + ngrok_cmd = [ngrok_config.ngrok_path, "http", str(port), "--log=stdout", "--log-level=info"] + if subdomain: ngrok_cmd += ["--subdomain", subdomain] if auth_token: @@ -34,37 +49,20 @@ def start_ngrok(port, auth_token=None, subdomain=None): service_core_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "core") - log_dir = os.path.join(service_core_dir, "logs") - os.makedirs(log_dir, exist_ok=True) - log_file = os.path.join(log_dir, "ngrok_subprocess.log") - - if sys.platform == "win32": - subprocess.DETACHED_PROCESS = 0x00000008 - with open(log_file, 'w') as log_f: - process = subprocess.Popen( - ngrok_cmd, - creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP, - stdout=log_f, - stderr=subprocess.STDOUT, # Combine stderr with stdout - cwd=service_core_dir, # Set working directory - env=os.environ.copy() # Inherit environment variables - ) - else: - with open(log_file, 'w') as log_f: - process = subprocess.Popen( - ngrok_cmd, - stdout=log_f, - stderr=subprocess.STDOUT, # Combine stderr with stdout - preexec_fn=os.setpgrp, - cwd=service_core_dir, # Set working directory - env=os.environ.copy() # Inherit environment variables - ) + log_file = get_tunnel_log_file("ngrok_subprocess.log") + process = spawn_detached_process(ngrok_cmd, log_file, cwd=service_core_dir, env=os.environ.copy()) + utils.set_file_permissions(log_file) + + time.sleep(NGROK_STARTUP_CHECK_DELAY_SECONDS) + if process.poll() is not None: + raise RuntimeError( + f"ngrok exited immediately (exit code {process.returncode}); see {log_file} for details" + ) actual_ngrok_pid = process.pid try: import psutil - time.sleep(0.5) # Give ngrok a moment to start - + # Look for the actual ngrok binary process for proc in psutil.process_iter(['pid', 'ppid', 'name', 'cmdline']): try: @@ -166,15 +164,22 @@ def start_ngrok_with_url(port, auth_token=None, subdomain=None): # If API method fails, try parsing the log file if not public_url: - service_core_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "core") - log_file = os.path.join(service_core_dir, "logs", "ngrok_subprocess.log") - public_url = get_ngrok_url_from_log(log_file) + public_url = get_ngrok_url_from_log(get_tunnel_log_file("ngrok_subprocess.log")) # If we still don't have a URL and subdomain was provided, construct it if not public_url and subdomain: public_url = f"https://{subdomain}.ngrok.io" logging.warning("Could not retrieve dynamic ngrok URL, using constructed URL") - + + # No URL and the process is gone (e.g. bad auth token) is a real failure, not just slow. + if not public_url: + try: + import psutil + if not psutil.pid_exists(pid): + raise RuntimeError(f"ngrok process {pid} is no longer running; see logs for details") + except ImportError: + pass + return pid, public_url def generate_ngrok_url(port, auth_token, ngrok_custom_domain, run_mode): @@ -200,32 +205,49 @@ def generate_ngrok_url(port, auth_token, ngrok_custom_domain, run_mode): log_event_callback=None, ) - with open(os.devnull, 'w') as devnull: + if run_mode == "background": + # Own subprocess with its own log file - skip the console-unsafe fd redirection below. + if ngrok_custom_domain: + ngrok_pid, public_url = start_ngrok_with_url(port=port, auth_token=auth_token, subdomain=ngrok_custom_domain) + else: + ngrok_pid, public_url = start_ngrok_with_url(port=port, auth_token=auth_token) + return public_url, ngrok_pid + + old_stdout_fd = None + old_stderr_fd = None + try: old_stdout_fd = os.dup(1) old_stderr_fd = os.dup(2) - os.dup2(devnull.fileno(), 1) - os.dup2(devnull.fileno(), 2) - + devnull_fd = os.open(os.devnull, os.O_WRONLY) try: - if run_mode == "background": - # Background mode: use subprocess for both custom and non-custom domains - if ngrok_custom_domain: - ngrok_pid, public_url = start_ngrok_with_url(port=port, auth_token=auth_token, subdomain=ngrok_custom_domain) - else: - ngrok_pid, public_url = start_ngrok_with_url(port=port, auth_token=auth_token) - return public_url, ngrok_pid - else: - # Foreground mode: use pyngrok library - if ngrok_custom_domain: - tunnel = ngrok.connect(port, subdomain=ngrok_custom_domain, pyngrok_config=ngrok_config) - else: - tunnel = ngrok.connect(port, pyngrok_config=ngrok_config) - return tunnel.public_url, None - + os.dup2(devnull_fd, 1) + os.dup2(devnull_fd, 2) finally: + os.close(devnull_fd) + except OSError: + # Restore anything already redirected (else fd 1 could stay pointed at devnull forever). + for fd, target in ((old_stdout_fd, 1), (old_stderr_fd, 2)): + if fd is not None: + try: + os.dup2(fd, target) + finally: + os.close(fd) + old_stdout_fd = None + old_stderr_fd = None + + try: + if ngrok_custom_domain: + tunnel = ngrok.connect(port, subdomain=ngrok_custom_domain, pyngrok_config=ngrok_config) + else: + tunnel = ngrok.connect(port, pyngrok_config=ngrok_config) + return tunnel.public_url, None + + finally: + if old_stdout_fd is not None: os.dup2(old_stdout_fd, 1) - os.dup2(old_stderr_fd, 2) os.close(old_stdout_fd) + if old_stderr_fd is not None: + os.dup2(old_stderr_fd, 2) os.close(old_stderr_fd) @@ -236,14 +258,15 @@ def _download_cloudflared(): Download cloudflared binary if not available. Returns path to cloudflared binary. """ - try: - # First try to find existing cloudflared - result = subprocess.run(['which', 'cloudflared'], capture_output=True, text=True) - if result.returncode == 0: - return result.stdout.strip() - except: - pass - + # Windows `where`/shutil.which both search cwd before PATH (planted-binary risk) - only search on POSIX. + if sys.platform != "win32": + try: + result = subprocess.run(['which', 'cloudflared'], capture_output=True, text=True) + if result.returncode == 0: + return result.stdout.strip().splitlines()[0] + except Exception as e: + logging.debug(f"Could not find existing cloudflared on PATH: {e}") + # Download cloudflared binary import platform import urllib.request @@ -326,32 +349,10 @@ def _start_cloudflare_with_binary(port, tunnel_token, custom_domain=None): ) service_core_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "core") - log_dir = os.path.join(service_core_dir, "logs") - os.makedirs(log_dir, exist_ok=True) - log_file = os.path.join(log_dir, "cloudflare_tunnel_subprocess.log") - - if sys.platform == "win32": - subprocess.DETACHED_PROCESS = 0x00000008 - with open(log_file, 'w') as log_f: - process = subprocess.Popen( - cloudflared_cmd, - creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP, - stdout=log_f, - stderr=subprocess.STDOUT, - cwd=service_core_dir, - env=os.environ.copy() - ) - else: - with open(log_file, 'w') as log_f: - process = subprocess.Popen( - cloudflared_cmd, - stdout=log_f, - stderr=subprocess.STDOUT, - preexec_fn=os.setpgrp, - cwd=service_core_dir, - env=os.environ.copy() - ) - + log_file = get_tunnel_log_file("cloudflare_tunnel_subprocess.log") + process = spawn_detached_process(cloudflared_cmd, log_file, cwd=service_core_dir, env=os.environ.copy()) + utils.set_file_permissions(log_file) + tunnel_url = get_cloudflare_url_from_log(log_file, custom_domain) return process.pid, tunnel_url @@ -380,9 +381,9 @@ def get_cloudflare_url_from_log(log_file, custom_domain=None, max_retries=10, re matches = re.findall(pattern, content) for match in matches: # Filter out localhost and other non-public URLs - if ('localhost' not in match and - '127.0.0.1' not in match and - 'trycloudflare.com' in match or 'cfargotunnel.com' in match or custom_domain in match if custom_domain else True): + if ('localhost' not in match and '127.0.0.1' not in match and + ('trycloudflare.com' in match or 'cfargotunnel.com' in match + or (custom_domain and custom_domain in match))): return match except Exception as e: @@ -410,36 +411,22 @@ def start_cloudflare_tunnel_with_url(port, tunnel_token, custom_domain=None): def generate_cloudflare_url(port, tunnel_token, custom_domain, run_mode): """ - Start a Cloudflare tunnel with complete log suppression. - Returns a tuple of (public_url, tunnel_pid) for background mode, or (public_url, None) for foreground mode. + Start a Cloudflare tunnel as a detached subprocess and return its public URL. + Returns a tuple of (public_url, tunnel_pid). """ if not port: raise ValueError("Port must be provided for Cloudflare tunnel.") - + if not tunnel_token or not tunnel_token.strip(): raise ValueError( "Tunnel token is required for secure Cloudflare tunnel operation. " "Temporary tunnels are not supported for production use." ) - - # Cloudflare tunnel configuration - - with open(os.devnull, 'w') as devnull: - old_stdout_fd = os.dup(1) - old_stderr_fd = os.dup(2) - os.dup2(devnull.fileno(), 1) - os.dup2(devnull.fileno(), 2) - - try: - tunnel_pid, public_url = start_cloudflare_tunnel_with_url( - port=port, - tunnel_token=tunnel_token, - custom_domain=custom_domain - ) - return public_url, tunnel_pid - - finally: - os.dup2(old_stdout_fd, 1) - os.dup2(old_stderr_fd, 2) - os.close(old_stdout_fd) - os.close(old_stderr_fd) \ No newline at end of file + + # Always runs as a detached subprocess with its own log file - nothing to suppress here. + tunnel_pid, public_url = start_cloudflare_tunnel_with_url( + port=port, + tunnel_token=tunnel_token, + custom_domain=custom_domain + ) + return public_url, tunnel_pid diff --git a/tests/test_cyberark_pam_import.py b/tests/test_cyberark_pam_import.py index 019677715..1b4352b55 100644 --- a/tests/test_cyberark_pam_import.py +++ b/tests/test_cyberark_pam_import.py @@ -1,18 +1,27 @@ -# _ __ -# | |/ /___ ___ _ __ ___ _ _ ® +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® # | ' monthly def test_matching_interval_without_flag_inherits_master(self): - """No flag, and the interval matches master — no exception exists.""" + """No flag, and the interval matches master — no exception exists.""" client = MagicMock() client.fetch_platform_rotation_policy.return_value = { "change": {"interval": 90, "allowedPeriodic": True}, @@ -3571,7 +4065,7 @@ def test_returns_none_on_network_error(self, mock_dns, mock_requests): assert result is None -# ── Red Team Coverage Tests ────────────────────────────────── +# ── Red Team Coverage Tests ────────────────────────────────── class TestEscFunction: """Tests for _esc() HTML + control char sanitizer.""" @@ -3691,7 +4185,7 @@ def test_fetch_users_respects_cap(self, mock_dns, mock_requests): client = CyberArkPVWAClient("pvwa.example.com") client.auth_token = "test" result = client.fetch_users() - # Should stop well before 999999 — capped at MAX_FETCH_RECORDS + # Should stop well before 999999 — capped at MAX_FETCH_RECORDS assert len(result) <= MAX_FETCH_RECORDS + 100 # allow 1 page overshoot @@ -3719,7 +4213,7 @@ def mock_get(url, **kwargs): "nextLink": "https://attacker.example.com/steal?token=x", } else: - # If we DID follow it, we'd hit this — test asserts we don't + # If we DID follow it, we'd hit this — test asserts we don't resp.json.return_value = {"Users": [], "nextLink": None} return resp @@ -3775,7 +4269,7 @@ def test_returns_429_after_max_retries(self, mock_dns, mock_requests, mock_sleep assert mock_requests.get.call_count == 3 # MAX_RETRIES -# ── Phase 7 Tests: Enhanced Report + Cleanup ───────────────── +# ── Phase 7 Tests: Enhanced Report + Cleanup ───────────────── class TestEnhancedReport: """Tests for the enhanced build_report with all sections.""" @@ -3803,13 +4297,13 @@ def test_report_has_all_sections(self, caplog): assert "IMPORT RESULTS" in report assert "PLATFORM MAPPING" in report assert "SKIPPED ACCOUNTS" in report - assert "UNMAPPED — REQUIRES MANUAL ACTION" not in report + assert "UNMAPPED — REQUIRES MANUAL ACTION" not in report assert "Dual control" not in report assert "debug log" in report assert "NEXT STEPS" in report assert "COMMAND" in report assert "pvwa.example.com" in report - assert "UNMAPPED — REQUIRES MANUAL ACTION" in caplog.text + assert "UNMAPPED — REQUIRES MANUAL ACTION" in caplog.text assert "Dual control" in caplog.text def test_report_gateway_token(self): @@ -3852,14 +4346,12 @@ class TestCleanupCommand: """Tests for CyberArkPAMCleanupCommand.""" def test_missing_args_raises(self): - from keepercommander.commands.pam_import.cyberark_import import CyberArkPAMCleanupCommand from keepercommander.error import CommandError cmd = CyberArkPAMCleanupCommand() with pytest.raises(CommandError): cmd.execute(MagicMock(), project_name="", config_uid="") def test_parser_has_flags(self): - from keepercommander.commands.pam_import.cyberark_import import CyberArkPAMCleanupCommand cmd = CyberArkPAMCleanupCommand() args = cmd.parser.parse_args(["--name", "Test", "--dry-run", "--yes"]) assert args.project_name == "Test" @@ -3867,12 +4359,124 @@ def test_parser_has_flags(self): assert args.auto_confirm is True def test_parser_config_flag(self): - from keepercommander.commands.pam_import.cyberark_import import CyberArkPAMCleanupCommand cmd = CyberArkPAMCleanupCommand() args = cmd.parser.parse_args(["--config", "uid123"]) assert args.config_uid == "uid123" +class TestCyberArkImportNsfSupport: + """NSF (--nsf) wiring for cyberark-import / cleanup / discovery.""" + + def test_single_batch_passes_use_nsf_to_import(self): + cmd = CyberArkPAMImportCommand() + params = MagicMock() + import_data = {"pam_data": {"resources": [], "users": []}} + + with patch.object(_temp_store, "write_json", return_value="/tmp/x.json"), \ + patch.object(_temp_store, "remove"), \ + patch("keepercommander.commands.pam_import.edit.PAMProjectImportCommand") as mock_import, \ + patch.object(cmd, "_find_config_uid", return_value="cfg-uid"): + mock_import.return_value.execute.return_value = None + result = cmd._single_batch_import( + params, import_data, "Proj", "", use_nsf=True, + ) + + assert result["config_uid"] == "cfg-uid" + kwargs = mock_import.return_value.execute.call_args.kwargs + assert kwargs.get("use_nsf") is True + assert kwargs.get("project_name") == "Proj" + + def test_single_batch_extend_ignores_use_nsf_flag(self): + cmd = CyberArkPAMImportCommand() + params = MagicMock() + import_data = {"pam_data": {"resources": [], "users": []}} + + with patch.object(_temp_store, "write_json", return_value="/tmp/x.json"), \ + patch.object(_temp_store, "remove"), \ + patch("keepercommander.commands.pam_import.extend.PAMProjectExtendCommand") as mock_extend: + mock_extend.return_value.execute.return_value = None + result = cmd._single_batch_import( + params, import_data, "Proj", "existing-cfg", use_nsf=True, + ) + + assert result["config_uid"] == "existing-cfg" + kwargs = mock_extend.return_value.execute.call_args.kwargs + assert "use_nsf" not in kwargs + assert kwargs.get("config") == "existing-cfg" + + def test_find_nsf_project_wrapper_uids(self): + root = NestedShareFolderNode() + root.uid = "nsf-root" + root.name = PAM_ROOT_FOLDER_NAME + root.parent_uid = None + root.subfolders = ["nsf-proj"] + + proj = NestedShareFolderNode() + proj.uid = "nsf-proj" + proj.name = DEFAULT_PROJECT_NAME + proj.parent_uid = "nsf-root" + proj.subfolders = ["nsf-safe"] + + safe = NestedShareFolderNode() + safe.uid = "nsf-safe" + safe.name = SAFE_FOLDER_NAME + safe.parent_uid = "nsf-proj" + safe.subfolders = [] + + params = SimpleNamespace( + folder_cache={ + "nsf-root": root, + "nsf-proj": proj, + "nsf-safe": safe, + }, + nested_share_folders={ + "nsf-root": {"name": PAM_ROOT_FOLDER_NAME, "parent_uid": None}, + "nsf-proj": {"name": DEFAULT_PROJECT_NAME, "parent_uid": "nsf-root"}, + "nsf-safe": {"name": SAFE_FOLDER_NAME, "parent_uid": "nsf-proj"}, + }, + shared_folder_cache={}, + subfolder_record_cache={}, + nested_share_folder_records={"nsf-safe": {"rec-1"}}, + ) + + wrappers = CyberArkPAMCleanupCommand._find_project_wrapper_folder_uids( + params, DEFAULT_PROJECT_NAME, + ) + assert wrappers == ["nsf-proj"] + + children = list(CyberArkPAMCleanupCommand._iter_project_child_folders( + params, "nsf-proj", + )) + assert children == [("nsf-safe", SAFE_FOLDER_NAME)] + + def test_find_classic_wrapper_still_works(self): + root = SimpleNamespace( + uid="uf-root", name=PAM_ROOT_FOLDER_NAME, parent_uid=None, + type=BaseFolderNode.UserFolderType, subfolders=["uf-proj"], + ) + proj = SimpleNamespace( + uid="uf-proj", name="MyProj", parent_uid="uf-root", + type=BaseFolderNode.UserFolderType, subfolders=["sf-safe"], + ) + safe = SimpleNamespace( + uid="sf-safe", name="SafeA", parent_uid="uf-proj", + type=BaseFolderNode.SharedFolderType, subfolders=[], + ) + params = SimpleNamespace( + folder_cache={"uf-root": root, "uf-proj": proj, "sf-safe": safe}, + nested_share_folders={}, + shared_folder_cache={}, + ) + wrappers = CyberArkPAMCleanupCommand._find_project_wrapper_folder_uids( + params, "MyProj", + ) + assert wrappers == ["uf-proj"] + children = list(CyberArkPAMCleanupCommand._iter_project_child_folders( + params, "uf-proj", + )) + assert children == [("sf-safe", "SafeA")] + + class TestSSHKeyImport: """C4: SSH key platforms store private key in private_pem_key, not password.""" @@ -4016,7 +4620,7 @@ def test_unmatched_members_excluded_from_permissions(self): } matcher = UserTeamMatcher(keeper_users=[{"email": "admin@corp.com"}]) result = build_shared_folder_permissions(safe_members, matcher) - assert result == {} # No matched members → no shared folder perms + assert result == {} # No matched members → no shared folder perms def test_team_members_matched(self): from keepercommander.importer.cyberark.cyberark_pam import build_shared_folder_permissions @@ -4075,13 +4679,13 @@ def test_empty_platform_id_uses_fallback(self): } result = mapper.map_account(account, None) assert result is not None - # No address + no platform → routed to login (not an unreachable pamMachine) + # No address + no platform → routed to login (not an unreachable pamMachine) assert result["type"] == "login" assert mapper.unmapped_platforms.get("(empty)", 0) == 1 def test_no_address_account_routed_to_login(self): """CyberArk accounts without an address can't be PAM resources - (gateway has nothing to connect to) — import as login.""" + (gateway has nothing to connect to) — import as login.""" mapper = AccountMapper() account = { "id": "1", "name": "floating-cred", "platformId": "UnixSSH", @@ -4092,7 +4696,7 @@ def test_no_address_account_routed_to_login(self): assert result["type"] == "login" assert result["login"] == "svc" assert result["password"] == "s3cret" - assert "No address" in result["notes"] + assert "notes" not in result def test_no_address_ssh_key_stays_pam_machine(self): """SSH keys without address stay as pamMachine so the @@ -4108,7 +4712,7 @@ def test_no_address_ssh_key_stays_pam_machine(self): assert result["type"] == "pamMachine" assert result["users"][0]["private_pem_key"] == fake_key - def test_missing_platform_id_uses_fallback(self): + def test_missing_platform_id_without_pattern_maps_to_login(self): mapper = AccountMapper() account = { "id": "36_3", "name": "PSMServer", @@ -4116,7 +4720,8 @@ def test_missing_platform_id_uses_fallback(self): } result = mapper.map_account(account, "pass") assert result is not None - assert result["type"] == "pamMachine" + assert result["type"] == "login" + assert "users" not in result def test_generic_network_account_maps_to_ssh(self): mapper = AccountMapper() @@ -4161,7 +4766,7 @@ def test_logon_domain_not_set_for_databases(self): result = mapper.map_account(account, "pass") assert "domain_name" not in result # Only pamMachine gets domain_name - def test_cpm_failure_status_annotated(self): + def test_cpm_failure_status_does_not_write_notes(self): mapper = AccountMapper() account = { "id": "25_7", "name": "x_accountB", "platformId": "WinDesktopLocal", @@ -4174,8 +4779,7 @@ def test_cpm_failure_status_annotated(self): } result = mapper.map_account(account, "pass") user = result["users"][0] - assert "FAILURE" in user.get("notes", "") - assert "(CPM)MaxRetries" in user.get("notes", "") + assert "notes" not in user def test_system_safes_expanded(self): from keepercommander.importer.cyberark.cyberark_pam import SYSTEM_SAFES @@ -4204,7 +4808,7 @@ class TestFolderDeduplication: def test_ksm_mode_dedup_adds_suffix(self): mapper = SafeFolderMapper(mode="ksm") name1 = mapper.map_safe("IT Safe", "proj") - name2 = mapper.map_safe("IT Safe!", "proj") # "!" stripped → "IT Safe" collision + name2 = mapper.map_safe("IT Safe!", "proj") # "!" stripped → "IT Safe" collision assert name1 != name2 assert "#2" in name2 @@ -4286,17 +4890,17 @@ def test_default_without_policy(self): assert cfg["text_session_recording"] == "off" -# ═══════════════════════════════════════════════════════════════════════ -# INTEGRATION TESTS — Real CyberArk PVWA JSON → Keeper Vault JSON -# ═══════════════════════════════════════════════════════════════════════ +# ═══════════════════════════════════════════════════════════════════════ +# INTEGRATION TESTS — Real CyberArk PVWA JSON → Keeper Vault JSON +# ═══════════════════════════════════════════════════════════════════════ # These tests use actual CyberArk API response shapes captured from a -# real on-prem PVWA environment. They verify the full input→output -# pipeline: API response → AccountMapper → SafeFolderMapper → -# validate_import_data → build_import_json → vault-ready JSON. +# real on-prem PVWA environment. They verify the full input→output +# pipeline: API response → AccountMapper → SafeFolderMapper → +# validate_import_data → build_import_json → vault-ready JSON. # Real CyberArk PVWA /api/Accounts response (subset from on-prem env) REAL_PVWA_ACCOUNTS = [ - # Unix SSH — CPM enabled, standard account + # Unix SSH — CPM enabled, standard account {"id": "28_11", "name": "Operating System-UnixSSH-10.0.1.30-simon", "platformId": "UnixSSH", "safeName": "partner", "address": "10.0.1.30", "userName": "simon", "secretType": "password", @@ -4305,7 +4909,7 @@ def test_default_without_policy(self): "lastModifiedTime": 1674712329}, "createdTime": 1674712329}, - # MSSQL Database — CPM disabled, custom port + database + # MSSQL Database — CPM disabled, custom port + database {"id": "25_15", "name": "db1", "platformId": "MSSql", "safeName": "Test", "address": "dbserver1.cyberark.local", "userName": "sa", "secretType": "password", @@ -4314,7 +4918,7 @@ def test_default_without_policy(self): "manualManagementReason": "NoReason"}, "createdTime": 1563922540}, - # Windows Desktop — CPM enabled, LogonDomain set + # Windows Desktop — CPM enabled, LogonDomain set {"id": "25_3", "name": "windows1", "platformId": "WinDesktopLocal", "safeName": "Test", "address": "components", "userName": "svc_account", "secretType": "password", @@ -4323,7 +4927,7 @@ def test_default_without_policy(self): "status": "success"}, "createdTime": 1551300856}, - # SSH Keys — CPM disabled, secretType=key + # SSH Keys — CPM disabled, secretType=key {"id": "25_14", "name": "Linux2", "platformId": "UnixSSHKeys", "safeName": "Test", "address": "linux2.cyberark.local", "userName": "root", "secretType": "key", "platformAccountProperties": {}, @@ -4331,7 +4935,7 @@ def test_default_without_policy(self): "manualManagementReason": "NoReason"}, "createdTime": 1563922244}, - # Windows — CPM failure status + # Windows — CPM failure status {"id": "25_7", "name": "Operating System-WindowsDesktopLocalAccountsRotationalPolicy-10.0.1.20-x_accountB", "platformId": "WinDesktopLocal", "safeName": "Test", @@ -4342,7 +4946,7 @@ def test_default_without_policy(self): "status": "failure"}, "createdTime": 1551306296}, - # BusinessWebsite → login record (not pamMachine) + # BusinessWebsite → login record (not pamMachine) {"id": "25_5", "name": "web-portal", "platformId": "BusinessWebsite", "safeName": "Test", "address": "", "userName": "admin", "platformAccountProperties": {"URL": "https://portal.company.com"}, @@ -4385,7 +4989,7 @@ def test_default_without_policy(self): "secretManagement": {"automaticManagementEnabled": True}, "createdTime": 1563908159}, - # Oracle Database — custom port + database + # Oracle Database — custom port + database {"id": "25_16", "name": "db2", "platformId": "Oracle", "safeName": "Test", "address": "dbserver2.cyberark.local", "userName": "oradb", "secretType": "password", @@ -4394,7 +4998,7 @@ def test_default_without_policy(self): "manualManagementReason": "NoReason"}, "createdTime": 1563922714}, - # Empty platformId — should use fallback + # Empty platformId — should use fallback {"id": "25_25", "name": "testobject", "platformId": "", "safeName": "Test", "address": "", "userName": "", "secretType": "password", "platformAccountProperties": {}, @@ -4415,7 +5019,7 @@ def _run_full_pipeline(accounts=None, project_name="CyberArk-Test", filtered = exclude_system_safes(all_safes) ok_safes = {s["safeName"] for s in filtered} - # Step 2: Map accounts → resources + users + # Step 2: Map accounts → resources + users mapper = AccountMapper() folder_mapper = SafeFolderMapper(mode=folder_mode) resources, users, skipped = [], [], [] @@ -4457,7 +5061,7 @@ def _run_full_pipeline(accounts=None, project_name="CyberArk-Test", class TestEndToEndPipeline: - """Integration: real PVWA JSON → full vault import JSON.""" + """Integration: real PVWA JSON → full vault import JSON.""" def test_overall_structure(self): data, _, _, _ = _run_full_pipeline() @@ -4490,7 +5094,7 @@ def test_resource_count(self): data, _, _, _ = _run_full_pipeline() resources = data["pam_data"]["resources"] # 12 accounts - 2 system safe - 1 login = 9 resources - # BUT empty platformId testobject has no address → still created as resource + # BUT empty platformId testobject has no address → still created as resource assert len(resources) >= 8 def test_login_records_separated(self): @@ -4548,7 +5152,7 @@ def test_mssql_database_resource(self): u = r["users"][0] assert u["connect_database"] == "hr" assert u["rotation_settings"]["enabled"] == "off" - assert "CPM disabled" in u.get("notes", "") + assert "notes" not in u def test_oracle_database_resource(self): data, _, _, _ = _run_full_pipeline() @@ -4588,14 +5192,11 @@ def test_ssh_key_resource(self): assert u["password"] == "" assert u["rotation_settings"]["enabled"] == "off" - def test_cpm_failure_annotated(self): + def test_cpm_failure_does_not_write_notes(self): data, _, _, _ = _run_full_pipeline() r = self._find_resource(data, "x_accountB") u = r["users"][0] - notes = u.get("notes", "") - assert "CPM disabled" in notes - assert "(CPM)MaxRetries" in notes - assert "FAILURE" in notes + assert "notes" not in u assert u["rotation_settings"]["enabled"] == "off" def test_generic_network_device(self): @@ -4625,7 +5226,7 @@ def test_every_nested_user_is_pam_user(self): assert u["type"] == "pamUser" def test_launch_credentials_matches_user_title(self): - """edit.py resolves launch_credentials by title → UID.""" + """edit.py resolves launch_credentials by title → UID.""" data, _, _, _ = _run_full_pipeline() for r in data["pam_data"]["resources"]: ps = r.get("pam_settings", {}) @@ -4644,7 +5245,7 @@ def test_rotation_settings_resource_field(self): rs = u.get("rotation_settings", {}) if rs.get("rotation") == "general": # edit.py auto-sets resourceUid to parent machine UID - # We don't set it here — edit.py handles it at import time + # We don't set it here — edit.py handles it at import time assert rs["rotation"] == "general" def test_managed_flag_on_users_with_password(self): @@ -4656,7 +5257,7 @@ def test_managed_flag_on_users_with_password(self): class TestFolderAssignment: - """Verify folder_path flows from safe → resource + nested users.""" + """Verify folder_path flows from safe → resource + nested users.""" def test_folder_path_on_resources(self): data, _, _, _ = _run_full_pipeline(folder_mode="ksm") @@ -4683,7 +5284,7 @@ def test_nested_user_folder_under_users_root(self): assert fp.startswith("CyberArk-Test - Users/"), f"Nested user folder not under Users root: {fp}" def test_resource_and_user_share_safe_subfolder(self): - """Resource in 'Test' safe → Resources/Test, its user → Users/Test.""" + """Resource in 'Test' safe → Resources/Test, its user → Users/Test.""" data, _, _, _ = _run_full_pipeline(folder_mode="ksm") for r in data["pam_data"]["resources"]: res_safe = r["folder_path"].split("/", 1)[1] if "/" in r["folder_path"] else "" @@ -4714,12 +5315,12 @@ def test_real_data_warnings(self): def test_no_address_account_becomes_login(self): """testobject (no address) is routed to a login record rather than - an unreachable pamMachine — validated at the data level.""" + an unreachable pamMachine — validated at the data level.""" data, _, _, _ = _run_full_pipeline() logins = data["pam_data"]["users"] testobject = next((l for l in logins if l["title"] == "testobject"), None) assert testobject is not None - assert "no address" in testobject.get("notes", "").lower() + assert "notes" not in testobject def test_clean_data_no_warnings(self): clean = [REAL_PVWA_ACCOUNTS[0]] # Just the Unix SSH account @@ -4811,7 +5412,7 @@ def test_output_is_json_serializable(self): assert len(roundtrip["pam_data"]["resources"]) == len(data["pam_data"]["resources"]) -# ── Optional local PVWA sample under .sample-data/ (gitignored; skip if absent) ── +# ── Optional local PVWA sample under .sample-data/ (gitignored; skip if absent) ── _SAMPLE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".sample-data")) _SAMPLE_FILES = ("safes.json", "accounts.json", "passwords.json") @@ -4911,7 +5512,7 @@ def test_vault_json_builds_and_serializes(self): assert "pam_data" in data -# ── Record-kind discriminator + ApplicationMapper stub ─────── +# ── Record-kind discriminator + ApplicationMapper stub ─────── class TestRecordKindDiscriminator: @@ -4993,9 +5594,9 @@ def test_custom_record_type_keeps_unmapped_cyberark_data_as_custom_fields(self): CyberArkImporter._add_account_metadata(record, { "platformId": "GenericPlatform", "platformAccountProperties": { - "OwnerName": "service-owner", + "OwnerName": "sample-owner", "Protocol": "SSH", - "Environment": {"Name": "prod"}, + "Environment": {"Name": "sample-env"}, }, }) @@ -5015,9 +5616,9 @@ def test_custom_record_type_keeps_unmapped_cyberark_data_as_custom_fields(self): assert custom_by_type["url"] == "https://server.example.com" assert custom_by_type["host"] == {"hostName": "server.example.com", "port": "22"} assert custom_by_label["Platform Name"] == "GenericPlatform" - assert custom_by_label["Owner Name"] == "service-owner" + assert custom_by_label["Owner Name"] == "sample-owner" assert custom_by_label["Protocol"] == "SSH" - assert custom_by_label["Environment.Name"] == "prod" + assert custom_by_label["Environment.Name"] == "sample-env" def test_import_engine_forwards_record_type_to_importer(self, monkeypatch): from keepercommander.importer import imp_exp @@ -5044,3 +5645,4 @@ def execute(self, filename, **kwargs): assert captured["filename"] == "https://pvwa.example.com" assert captured["kwargs"]["record_type"] == "test custom" + diff --git a/unit-tests/data_enterprise.py b/unit-tests/data_enterprise.py index e59cc10c8..230f9430e 100644 --- a/unit-tests/data_enterprise.py +++ b/unit-tests/data_enterprise.py @@ -26,6 +26,10 @@ _ROLE1_ID = (_ENTERPRISE_ID << 32) + 301 _ROLE1_NAME = 'Role 1' +_ROLE2_ID = (_ENTERPRISE_ID << 32) + 302 +_ROLE2_NAME = 'Role 2' +_ROLE_ADMIN_ID = (_ENTERPRISE_ID << 32) + 303 +_ROLE_ADMIN_NAME = 'Admin Role' _LAST_ID = 1000 @@ -45,6 +49,10 @@ def __init__(self): self.user2_email = _USER2_EMAIL self.role1_id = _ROLE1_ID self.role1_name = _ROLE1_NAME + self.role2_id = _ROLE2_ID + self.role2_name = _ROLE2_NAME + self.role_admin_id = _ROLE_ADMIN_ID + self.role_admin_name = _ROLE_ADMIN_NAME def enterprise_allocate_ids(params, request): @@ -122,6 +130,22 @@ def get_enterprise_data(params): crypto.encrypt_aes_v1(json.dumps({'displayname': _ROLE1_NAME}).encode('utf-8'), _TREE_KEY)), 'visible_below': True, 'new_user_inherit': True + }, + { + 'role_id': _ROLE2_ID, + 'node_id': _NODE1_ID, + 'encrypted_data': utils.base64_url_encode( + crypto.encrypt_aes_v1(json.dumps({'displayname': _ROLE2_NAME}).encode('utf-8'), _TREE_KEY)), + 'visible_below': False, + 'new_user_inherit': False + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'node_id': _NODE1_ID, + 'encrypted_data': utils.base64_url_encode( + crypto.encrypt_aes_v1(json.dumps({'displayname': _ROLE_ADMIN_NAME}).encode('utf-8'), _TREE_KEY)), + 'visible_below': False, + 'new_user_inherit': False } ] rs['managed_nodes'] = [ @@ -129,6 +153,16 @@ def get_enterprise_data(params): 'role_id': _ROLE1_ID, 'managed_node_id': _NODE1_ID, 'cascade_node_management': True, + }, + { + 'role_id': _ROLE2_ID, + 'managed_node_id': _NODE1_ID, + 'cascade_node_management': False, + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'cascade_node_management': True, } ] rs['role_users'] = [ @@ -137,6 +171,49 @@ def get_enterprise_data(params): 'enterprise_user_id': _USER1_ID } ] + # KC-1412: Add role_privileges to test authorization checks + rs['role_privileges'] = [ + { + 'role_id': _ROLE1_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_nodes' + }, + { + 'role_id': _ROLE1_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_user' + }, + { + 'role_id': _ROLE1_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_roles' + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_nodes' + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_user' + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_roles' + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'transfer_account' + }, + { + 'role_id': _ROLE_ADMIN_ID, + 'managed_node_id': _NODE1_ID, + 'privilege': 'manage_teams' + } + ] rs['teams'] = [ { 'team_uid': _TEAM1_UID, diff --git a/unit-tests/pam/test_gateway_enforcement_gate.py b/unit-tests/pam/test_gateway_enforcement_gate.py new file mode 100644 index 000000000..ea090dbdb --- /dev/null +++ b/unit-tests/pam/test_gateway_enforcement_gate.py @@ -0,0 +1,323 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | ' MagicMock + """mode: 'allowed' (key present, True), 'denied_explicit_false' (key + present, False), 'denied_absent' (booleans non-empty but key missing — + what Commander's account-summary parser actually produces when a + checkbox is unchecked, since it drops `:false` entries), or + 'no_enterprise' (no enforcement context at all — personal account).""" + params = MagicMock() + if mode == 'no_enterprise': + params.enforcements = None + return params + if mode == 'allowed': + booleans = [{'key': 'allow_pam_gateway', 'value': True}] + elif mode == 'denied_explicit_false': + booleans = [{'key': 'allow_pam_gateway', 'value': False}] + elif mode == 'denied_absent': + booleans = [{'key': 'some_other_key', 'value': True}] + else: + raise ValueError(f'unknown mode: {mode}') + params.enforcements = {'booleans': booleans} + return params + + +# A real multi-key `booleans` payload (captured from a live account_summary +# response) — models a role with many unrelated PAM permissions granted, +# to confirm the key lookup isn't fooled by a "some true entries exist" +# shortcut and actually finds/omits allow_pam_gateway specifically. +_MIXED_ROLE_BOOLEANS_TEMPLATE = [ + {'key': 'send_breach_watch_events', 'value': True}, + {'key': 'allow_alternate_passwords', 'value': True}, + {'key': 'allow_pam_discovery', 'value': True}, + {'key': 'allow_pam_rotation', 'value': True}, + {'key': 'allow_configure_pam_cloud_connection_settings', 'value': True}, + {'key': 'restrict_mac_fingerprint', 'value': True}, + {'key': 'allow_launch_pam_on_cloud_connection', 'value': True}, + {'key': 'allow_configure_rbi', 'value': True}, + {'key': 'allow_launch_rbi', 'value': True}, + {'key': 'allow_secrets_manager', 'value': True}, + {'key': 'allow_launch_pam_tunnels', 'value': True}, + {'key': 'allow_configure_workflow_settings', 'value': True}, + {'key': 'allow_configure_uss_settings', 'value': True}, + {'key': 'allow_rotate_credentials', 'value': True}, + {'key': 'allow_view_kcm_recordings', 'value': True}, + {'key': 'allow_view_rbi_recordings', 'value': True}, + {'key': 'allow_configure_rotation_settings', 'value': True}, + {'key': 'allow_configure_pam_tunneling_settings', 'value': True}, + {'key': 'allow_can_edit_external_shares', 'value': True}, +] + + +def _params_with_mixed_role(gateway_allowed): + # type: (bool) -> MagicMock + """A role holding ~19 unrelated PAM/vault permissions, with + allow_pam_gateway either granted or omitted alongside them — the + real-world "partial permissions" shape, as opposed to a single-key + toy list.""" + params = MagicMock() + booleans = list(_MIXED_ROLE_BOOLEANS_TEMPLATE) + if gateway_allowed: + booleans = booleans + [{'key': 'allow_pam_gateway', 'value': True}] + params.enforcements = {'booleans': booleans} + return params + + +class TestPAMCreateGatewayCommandEnforcement(unittest.TestCase): + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_denied_when_enforcement_absent(self, mock_get_app_record, mock_create_gateway): + params = _params_with_enforcement('denied_absent') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + result = PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application') + + self.assertIsNone(result) + self.assertFalse(mock_get_app_record.called) + self.assertFalse(mock_create_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_denied_when_enforcement_explicit_false(self, mock_get_app_record, mock_create_gateway): + params = _params_with_enforcement('denied_explicit_false') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + result = PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application') + + self.assertIsNone(result) + self.assertFalse(mock_get_app_record.called) + self.assertFalse(mock_create_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_ksm_app_display_info') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_allowed_when_enforcement_true( + self, mock_get_app_record, mock_get_ksm_app_display_info, mock_create_gateway): + params = _params_with_enforcement('allowed') + mock_get_app_record.return_value = {'record_uid': 'app_uid'} + mock_get_ksm_app_display_info.return_value = ('PAM_application', True, 'PAM_application (app_uid)') + mock_create_gateway.return_value = 'US:one-time-token' + + PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application', return_value=True) + + self.assertTrue(mock_create_gateway.called) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_ksm_app_display_info') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_allowed_when_no_enforcements( + self, mock_get_app_record, mock_get_ksm_app_display_info, mock_create_gateway): + # Personal / non-enterprise account: no enforcement context, fail open. + params = _params_with_enforcement('no_enterprise') + mock_get_app_record.return_value = {'record_uid': 'app_uid'} + mock_get_ksm_app_display_info.return_value = ('PAM_application', True, 'PAM_application (app_uid)') + mock_create_gateway.return_value = 'US:one-time-token' + + PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application', return_value=True) + + self.assertTrue(mock_create_gateway.called) + + +class TestPAMGatewayRemoveCommandEnforcement(unittest.TestCase): + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + def test_denied_when_enforcement_absent(self, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('denied_absent') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + PAMGatewayRemoveCommand().execute(params, gateway='some_gateway_uid') + + self.assertFalse(mock_get_all_gateways.called) + self.assertFalse(mock_remove_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + def test_denied_when_enforcement_explicit_false(self, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('denied_explicit_false') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + PAMGatewayRemoveCommand().execute(params, gateway='some_gateway_uid') + + self.assertFalse(mock_get_all_gateways.called) + self.assertFalse(mock_remove_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + @patch('keepercommander.commands.discoveryrotation.utils.base64_url_encode') + def test_allowed_when_enforcement_true( + self, mock_base64_url_encode, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('allowed') + gateway = MagicMock(controllerUid=b'controller_uid', controllerName='pocbyuser') + mock_get_all_gateways.return_value = [gateway] + mock_base64_url_encode.return_value = 'gateway_uid' + + PAMGatewayRemoveCommand().execute(params, gateway='gateway_uid') + + self.assertTrue(mock_remove_gateway.called) + + +# --- Legacy discoveryrotation_v1 module (reachable via `pam legacy`) --- +# Same enforcement gate, imported from discoveryrotation and reused, but +# exercised against the v1 command classes to guard the legacy bypass path. + +class TestLegacyPAMCreateGatewayCommandEnforcement(unittest.TestCase): + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.create_gateway') + def test_denied_when_enforcement_absent(self, mock_create_gateway): + params = _params_with_enforcement('denied_absent') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + result = discoveryrotation_v1.PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application') + + self.assertIsNone(result) + self.assertFalse(mock_create_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.create_gateway') + def test_denied_when_enforcement_explicit_false(self, mock_create_gateway): + params = _params_with_enforcement('denied_explicit_false') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + result = discoveryrotation_v1.PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application') + + self.assertIsNone(result) + self.assertFalse(mock_create_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.create_gateway') + def test_allowed_when_enforcement_true(self, mock_create_gateway): + params = _params_with_enforcement('allowed') + mock_create_gateway.return_value = 'US:one-time-token' + + discoveryrotation_v1.PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application', return_value=True) + + self.assertTrue(mock_create_gateway.called) + + +class TestLegacyPAMGatewayRemoveCommandEnforcement(unittest.TestCase): + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.get_all_gateways') + def test_denied_when_enforcement_absent(self, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('denied_absent') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + discoveryrotation_v1.PAMGatewayRemoveCommand().execute(params, gateway='some_gateway_uid') + + self.assertFalse(mock_get_all_gateways.called) + self.assertFalse(mock_remove_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.get_all_gateways') + def test_denied_when_enforcement_explicit_false(self, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('denied_explicit_false') + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + discoveryrotation_v1.PAMGatewayRemoveCommand().execute(params, gateway='some_gateway_uid') + + self.assertFalse(mock_get_all_gateways.called) + self.assertFalse(mock_remove_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation_v1.gateway_helper.get_all_gateways') + @patch('keepercommander.commands.discoveryrotation_v1.utils.base64_url_encode') + def test_allowed_when_enforcement_true( + self, mock_base64_url_encode, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_enforcement('allowed') + gateway = MagicMock(controllerUid=b'controller_uid', controllerName='pocbyuser') + mock_get_all_gateways.return_value = [gateway] + mock_base64_url_encode.return_value = 'gateway_uid' + + discoveryrotation_v1.PAMGatewayRemoveCommand().execute(params, gateway='gateway_uid') + + self.assertTrue(mock_remove_gateway.called) + + +class TestMixedRolePermissions(unittest.TestCase): + """Partial-permission / mixed-role coverage: a role holding many other + PAM permissions must still be gated on allow_pam_gateway specifically, + neither over-denying (other permissions present) nor over-allowing + (mistaking "booleans list is non-empty" for "gateway is allowed").""" + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_create_denied_with_many_other_permissions_granted( + self, mock_get_app_record, mock_create_gateway): + params = _params_with_mixed_role(gateway_allowed=False) + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application') + + self.assertFalse(mock_get_app_record.called) + self.assertFalse(mock_create_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.create_gateway') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_ksm_app_display_info') + @patch('keepercommander.commands.discoveryrotation.KSMCommand.get_app_record') + def test_create_allowed_with_many_other_permissions_granted( + self, mock_get_app_record, mock_get_ksm_app_display_info, mock_create_gateway): + params = _params_with_mixed_role(gateway_allowed=True) + mock_get_app_record.return_value = {'record_uid': 'app_uid'} + mock_get_ksm_app_display_info.return_value = ('PAM_application', True, 'PAM_application (app_uid)') + mock_create_gateway.return_value = 'US:one-time-token' + + PAMCreateGatewayCommand().execute( + params, gateway_name='pocbyuser', ksm_app='PAM_application', return_value=True) + + self.assertTrue(mock_create_gateway.called) + + @patch('keepercommander.commands.discoveryrotation.gateway_helper.remove_gateway') + @patch('keepercommander.commands.discoveryrotation.gateway_helper.get_all_gateways') + def test_remove_denied_with_many_other_permissions_granted( + self, mock_get_all_gateways, mock_remove_gateway): + params = _params_with_mixed_role(gateway_allowed=False) + with patch('sys.stdout', new_callable=io.StringIO) as stdout: + PAMGatewayRemoveCommand().execute(params, gateway='some_gateway_uid') + + self.assertFalse(mock_get_all_gateways.called) + self.assertFalse(mock_remove_gateway.called) + self.assertIn('gateway management', stdout.getvalue().lower()) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/pam/test_pam_nsf_folder_batch.py b/unit-tests/pam/test_pam_nsf_folder_batch.py new file mode 100644 index 000000000..ba15e7d84 --- /dev/null +++ b/unit-tests/pam/test_pam_nsf_folder_batch.py @@ -0,0 +1,163 @@ +"""Tests for batched NSF folder creation used by PAM CyberArk import.""" + +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + +from keepercommander.commands.pam_import.nsf_helpers import create_nsf_folders_batch +from keepercommander.error import CommandError + + +class TestCreateNsfFoldersBatch(unittest.TestCase): + + def _params(self): + params = MagicMock() + params.nested_share_folders = {} + params.subfolder_cache = {} + params.folder_cache = {} + params.environment_variables = {} + return params + + @patch('keepercommander.commands.pam_import.nsf_helpers.sync_down_preserving_nsf_keys') + @patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3') + def test_batches_and_seeds_cache(self, mock_batch, mock_sync): + params = self._params() + mock_batch.return_value = [ + { + 'folder_uid': 'uid-a', + 'folder_key_unencrypted': b'key-a', + 'name': 'SafeA', + 'success': True, + 'message': '', + }, + { + 'folder_uid': 'uid-b', + 'folder_key_unencrypted': b'key-b', + 'name': 'SafeB', + 'success': True, + 'message': '', + }, + ] + + specs = [ + {'name': 'SafeA', 'parent_uid': 'proj'}, + {'name': 'SafeB', 'parent_uid': 'proj'}, + ] + results = create_nsf_folders_batch(params, specs, sync=True, command='pam') + + self.assertEqual(len(results), 2) + mock_batch.assert_called_once_with(params, specs) + mock_sync.assert_called_once_with(params) + self.assertEqual(params.nested_share_folders['uid-a']['name'], 'SafeA') + self.assertEqual(params.nested_share_folders['uid-b']['parent_uid'], 'proj') + + @patch('keepercommander.commands.pam_import.nsf_helpers.sync_down_preserving_nsf_keys') + @patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3') + def test_skips_sync_when_requested(self, mock_batch, mock_sync): + params = self._params() + mock_batch.return_value = [{ + 'folder_uid': 'uid-a', + 'folder_key_unencrypted': b'k', + 'name': 'Config', + 'success': True, + 'message': '', + }] + + create_nsf_folders_batch( + params, [{'name': 'Config', 'parent_uid': 'proj'}], + sync=False, command='pam', + ) + mock_sync.assert_not_called() + + @patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3') + def test_raises_on_failed_folder(self, mock_batch): + params = self._params() + mock_batch.return_value = [{ + 'folder_uid': 'uid-a', + 'success': False, + 'message': 'denied', + 'name': 'Bad', + }] + + with self.assertRaises(CommandError): + create_nsf_folders_batch( + params, [{'name': 'Bad', 'parent_uid': 'proj'}], + sync=False, command='pam', + ) + + @patch('keepercommander.commands.pam_import.nsf_helpers.sync_down_preserving_nsf_keys') + @patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3') + def test_chunks_over_100(self, mock_batch, mock_sync): + params = self._params() + + def _side_effect(_params, chunk): + return [{ + 'folder_uid': f'uid-{i}', + 'folder_key_unencrypted': b'k', + 'name': s['name'], + 'success': True, + 'message': '', + } for i, s in enumerate(chunk)] + + mock_batch.side_effect = _side_effect + specs = [{'name': f'F{i}', 'parent_uid': 'proj'} for i in range(105)] + results = create_nsf_folders_batch(params, specs, sync=True, command='pam') + + self.assertEqual(len(results), 105) + self.assertEqual(mock_batch.call_count, 2) + self.assertEqual(len(mock_batch.call_args_list[0][0][1]), 100) + self.assertEqual(len(mock_batch.call_args_list[1][0][1]), 5) + + +class TestCreateSafeFoldersNsfBatch(unittest.TestCase): + + @patch('keepercommander.commands.pam_import.edit.PAMProjectImportCommand.add_folder_permissions') + @patch('keepercommander.commands.pam_import.nsf_helpers.create_nsf_folders_batch') + def test_two_layer_batch_layout(self, mock_batch, _mock_perms): + from keepercommander.commands.pam_import.edit import PAMProjectImportCommand + + cmd = PAMProjectImportCommand() + params = MagicMock() + res = {"project_folder": "CyberArk Migration", "safe_folders": []} + safe_folder_map = {} + records = [ + {"name": "Win_Local", "safe_name": "Win_Local", "fperm": {}, "uperm": []}, + {"name": "Linux", "safe_name": "Linux", "fperm": {}, "uperm": [{"name": "u@x.com"}]}, + ] + + # Layer1: Config + 2 safes; Layer2: 4 children + def _batch(_params, specs, sync=True, command='pam'): + return [{ + 'folder_uid': f'uid-{spec["name"]}', + 'name': spec['name'], + 'success': True, + } for spec in specs] + + mock_batch.side_effect = _batch + + cmd._create_safe_folders_nsf_batch( + params, 'proj-uid', res, records, + 'CyberArk Migration - Config', safe_folder_map, + ) + + self.assertEqual(mock_batch.call_count, 2) + layer1_specs = mock_batch.call_args_list[0][0][1] + layer2_specs = mock_batch.call_args_list[1][0][1] + self.assertEqual( + [s['name'] for s in layer1_specs], + ['CyberArk Migration - Config', 'Win_Local', 'Linux'], + ) + self.assertEqual(len(layer2_specs), 4) + self.assertEqual(res['config_folder_uid'], 'uid-CyberArk Migration - Config') + self.assertEqual(safe_folder_map['Win_Local'], 'uid-Win_Local') + self.assertIn('Win_Local/Win_Local - Resources', safe_folder_map) + self.assertIn('Linux/Linux - Users', safe_folder_map) + self.assertEqual(len(res['safe_folders']), 2) + _mock_perms.assert_called_once() + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/pam/test_pam_project_import_nsf.py b/unit-tests/pam/test_pam_project_import_nsf.py index 02a6a6be1..0ace49947 100644 --- a/unit-tests/pam/test_pam_project_import_nsf.py +++ b/unit-tests/pam/test_pam_project_import_nsf.py @@ -95,7 +95,10 @@ def test_import_record_objects_use_nsf_aware_record_add_helper(): assert uid == 'record_uid' add_record.assert_called_once() assert add_record.call_args.args[2] == 'root_nsf' - assert add_record.call_args.kwargs == {'command': 'pam-project-import'} + assert add_record.call_args.kwargs == { + 'command': 'pam-project-import', + 'sync_after': False, + } ROOT_NAME = PAMProjectImportCommand.PAM_ROOT_FOLDER_NAME @@ -244,22 +247,21 @@ def test_process_folders_uses_existing_nsf_root_and_creates_nsf_children(): } created = [] - def create_folder(params_arg, folder_name, parent_uid=None): - uid = f'nsf_{len(created) + 1}' - created.append((folder_name, parent_uid, uid)) - params_arg.nested_share_folders[uid] = { - 'name': folder_name, - 'parent_uid': parent_uid, - 'folder_key_unencrypted': b'k' * 32, - } - return { - 'success': True, - 'folder_uid': uid, - 'folder_key_unencrypted': b'k' * 32, - } - - with patch('keepercommander.nested_share_folder.folder_api.create_folder_v3', - side_effect=create_folder) as create_folder_v3, \ + def create_folders_batch(_params, folder_specs): + results = [] + for spec in folder_specs: + uid = f'nsf_{len(created) + 1}' + created.append((spec['name'], spec.get('parent_uid'), uid)) + results.append({ + 'success': True, + 'folder_uid': uid, + 'folder_key_unencrypted': b'k' * 32, + 'name': spec['name'], + }) + return results + + with patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3', + side_effect=create_folders_batch) as create_batch, \ patch('keepercommander.commands.pam_import.nsf_helpers.api.sync_down'), \ patch('keepercommander.commands.pam_import.edit.api.sync_down'): result = PAMProjectImportCommand().process_folders(params, project) @@ -268,7 +270,7 @@ def create_folder(params_arg, folder_name, parent_uid=None): assert result['project_folder_uid'] == 'nsf_1' assert result['resources_folder_uid'] == 'nsf_2' assert result['users_folder_uid'] == 'nsf_3' - assert create_folder_v3.call_count == 3 + assert create_batch.call_count == 3 assert created == [ ('Project 1', 'root_nsf', 'nsf_1'), ('Project 1 - Resources', 'nsf_1', 'nsf_2'), @@ -298,22 +300,21 @@ def test_process_folders_with_nsf_flag_ignores_legacy_root_folder(): } created = [] - def create_folder(params_arg, folder_name, parent_uid=None): - uid = f'nsf_{len(created) + 1}' - created.append((folder_name, parent_uid, uid)) - params_arg.nested_share_folders[uid] = { - 'name': folder_name, - 'parent_uid': parent_uid, - 'folder_key_unencrypted': b'k' * 32, - } - return { - 'success': True, - 'folder_uid': uid, - 'folder_key_unencrypted': b'k' * 32, - } - - with patch('keepercommander.nested_share_folder.folder_api.create_folder_v3', - side_effect=create_folder), \ + def create_folders_batch(_params, folder_specs): + results = [] + for spec in folder_specs: + uid = f'nsf_{len(created) + 1}' + created.append((spec['name'], spec.get('parent_uid'), uid)) + results.append({ + 'success': True, + 'folder_uid': uid, + 'folder_key_unencrypted': b'k' * 32, + 'name': spec['name'], + }) + return results + + with patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3', + side_effect=create_folders_batch), \ patch('keepercommander.commands.pam_import.nsf_helpers.api.sync_down'), \ patch('keepercommander.commands.pam_import.edit.api.sync_down'): result = PAMProjectImportCommand().process_folders(params, project) @@ -354,18 +355,19 @@ def test_create_subfolder_seeds_folder_key_and_survives_sync_wipe(): params = _params() folder_key = b'f' * 32 - def create_folder(_params, folder_name, parent_uid=None): - return { + def create_folders_batch(_params, folder_specs): + return [{ 'success': True, 'folder_uid': 'new_nsf', 'folder_key_unencrypted': folder_key, - } + 'name': folder_specs[0]['name'], + }] def wipe_nsf(_params): _params.nested_share_folders.clear() - with patch('keepercommander.nested_share_folder.folder_api.create_folder_v3', - side_effect=create_folder), \ + with patch('keepercommander.nested_share_folder.folder_api.create_folders_batch_v3', + side_effect=create_folders_batch), \ patch('keepercommander.commands.pam_import.nsf_helpers.api.sync_down', side_effect=wipe_nsf): uid = PAMProjectImportCommand().create_subfolder( diff --git a/unit-tests/service/test_cloudflare_config.py b/unit-tests/service/test_cloudflare_config.py new file mode 100644 index 000000000..9701073ec --- /dev/null +++ b/unit-tests/service/test_cloudflare_config.py @@ -0,0 +1,20 @@ +import unittest + +from keepercommander.service.config.cloudflare_config import CloudflareConfigurator +from keepercommander.service.util.tunneling import get_tunnel_log_file + + +class TestCloudflareLogPath(unittest.TestCase): + def test_health_check_reads_the_same_file_the_tunnel_writes_to(self): + """_get_cloudflare_log_path() used to hardcode its own copy of the log path, + which drifted out of sync when the tunnel log location moved to the shared + service_logs dir - the health check kept reading the old, now-empty file and + always timed out with 'status could not be determined'.""" + self.assertEqual( + CloudflareConfigurator._get_cloudflare_log_path(), + get_tunnel_log_file("cloudflare_tunnel_subprocess.log"), + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/service/test_log_sanitization.py b/unit-tests/service/test_log_sanitization.py index 681fa3add..3690e13bb 100644 --- a/unit-tests/service/test_log_sanitization.py +++ b/unit-tests/service/test_log_sanitization.py @@ -6,9 +6,11 @@ sanitize_debug_data, ) from keepercommander.service.decorators.api_logging import ( - _sanitize_nested_data, sanitize_password_in_command, ) +from keepercommander.sanitization import ( + sanitize_nested_data, +) from keepercommander.service.util.command_util import CommandExecutor from keepercommander.service.util.exceptions import CommandExecutionError from keepercommander.service.util.request_validation import RequestValidator @@ -86,7 +88,7 @@ def test_sensitive_type_value_shape_is_masked(self): "cardExpirationDate": "04/2026", "cardSecurityCode": "123"}]}, ] - sanitized = _sanitize_nested_data(filedata) + sanitized = sanitize_nested_data(filedata) dumped = str(sanitized) self.assertNotIn(SECRET_VALUE, dumped) self.assertNotIn('4111111111111111', dumped) @@ -94,7 +96,7 @@ def test_sensitive_type_value_shape_is_masked(self): def test_non_sensitive_type_value_untouched(self): filedata = [{"type": "text", "value": ["not a secret"]}] - sanitized = _sanitize_nested_data(filedata) + sanitized = sanitize_nested_data(filedata) self.assertEqual(sanitized, filedata) def test_sanitize_password_in_command_masks_command_and_filedata(self): diff --git a/unit-tests/service/test_main_dispatch.py b/unit-tests/service/test_main_dispatch.py new file mode 100644 index 000000000..c712d3801 --- /dev/null +++ b/unit-tests/service/test_main_dispatch.py @@ -0,0 +1,50 @@ +import unittest +from unittest import mock + +from keepercommander.__main__ import main + + +class TestServiceModeDispatch(unittest.TestCase): + """The frozen background-service subprocess is detected via an explicit argv flag + (SERVICE_MODE_FLAG), not an env var - see service_app.py for why.""" + + def test_frozen_with_internal_flag_runs_background_service_and_returns(self): + with mock.patch('sys.frozen', True, create=True), \ + mock.patch('sys._MEIPASS', '/frozen/path', create=True), \ + mock.patch('sys.argv', ['keeper.exe', '--internal-run-service']), \ + mock.patch('keepercommander.service.core.service_app.run_background_service') as mock_run, \ + mock.patch('keepercommander.utils.get_ssl_cert_file', return_value=None), \ + mock.patch('argparse.ArgumentParser.parse_known_args') as mock_parse_args: + main() + + mock_run.assert_called_once() + # If dispatch didn't return early, normal argument parsing would have run too. + mock_parse_args.assert_not_called() + + def test_frozen_without_flag_does_not_start_the_service(self): + with mock.patch('sys.frozen', True, create=True), \ + mock.patch('sys._MEIPASS', '/frozen/path', create=True), \ + mock.patch('sys.argv', ['keeper.exe', '--help']), \ + mock.patch('keepercommander.service.core.service_app.run_background_service') as mock_run, \ + mock.patch('keepercommander.utils.get_ssl_cert_file', return_value=None), \ + mock.patch('argparse.ArgumentParser.parse_known_args', side_effect=SystemExit(0)): + with self.assertRaises(SystemExit): + main() + + mock_run.assert_not_called() + + def test_not_frozen_ignores_the_flag_entirely(self): + """Running from source, the flag has no special meaning and normal parsing proceeds.""" + with mock.patch('sys.frozen', False, create=True), \ + mock.patch('sys.argv', ['keeper', '--internal-run-service']), \ + mock.patch('keepercommander.service.core.service_app.run_background_service') as mock_run, \ + mock.patch('keepercommander.utils.get_ssl_cert_file', return_value=None), \ + mock.patch('argparse.ArgumentParser.parse_known_args', side_effect=SystemExit(2)): + with self.assertRaises(SystemExit): + main() + + mock_run.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/service/test_process_util.py b/unit-tests/service/test_process_util.py new file mode 100644 index 000000000..642b412da --- /dev/null +++ b/unit-tests/service/test_process_util.py @@ -0,0 +1,64 @@ +import unittest +from unittest import mock + +from keepercommander.service.util.process_util import ( + spawn_detached_process, CREATE_NO_WINDOW, DETACHED_PROCESS, CREATE_NEW_PROCESS_GROUP +) + + +class TestSpawnDetachedProcess(unittest.TestCase): + def test_default_mode_truncates_log_file(self): + with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + mock.patch('keepercommander.service.util.process_util.subprocess.Popen'), \ + mock.patch('keepercommander.service.util.process_util.sys.platform', 'darwin'): + spawn_detached_process(['cmd'], '/tmp/test.log') + mock_open.assert_called_once_with('/tmp/test.log', 'w') + + def test_append_mode_preserves_log_history(self): + with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + mock.patch('keepercommander.service.util.process_util.subprocess.Popen'), \ + mock.patch('keepercommander.service.util.process_util.sys.platform', 'darwin'): + spawn_detached_process(['cmd'], '/tmp/test.log', append=True) + mock_open.assert_called_once_with('/tmp/test.log', 'a') + + def test_windows_uses_hidden_detached_creation_flags(self): + with mock.patch('builtins.open', mock.mock_open()), \ + mock.patch('keepercommander.service.util.process_util.subprocess.Popen') as mock_popen, \ + mock.patch('keepercommander.service.util.process_util.sys.platform', 'win32'): + spawn_detached_process(['cmd'], '/tmp/test.log', cwd='/work', env={'A': '1'}) + + _, kwargs = mock_popen.call_args + self.assertEqual( + kwargs['creationflags'], + DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW + ) + self.assertEqual(kwargs['cwd'], '/work') + self.assertEqual(kwargs['env'], {'A': '1'}) + self.assertNotIn('start_new_session', kwargs) + + def test_posix_uses_new_session_not_preexec_fn(self): + """start_new_session=True (os.setsid in the child) is the thread-safe replacement + for preexec_fn=os.setpgrp - same effect, without the multi-threading hazard.""" + with mock.patch('builtins.open', mock.mock_open()), \ + mock.patch('keepercommander.service.util.process_util.subprocess.Popen') as mock_popen, \ + mock.patch('keepercommander.service.util.process_util.sys.platform', 'darwin'): + spawn_detached_process(['cmd'], '/tmp/test.log') + + _, kwargs = mock_popen.call_args + self.assertNotIn('creationflags', kwargs) + self.assertNotIn('preexec_fn', kwargs) + self.assertTrue(kwargs['start_new_session']) + + def test_returns_the_popen_object(self): + with mock.patch('builtins.open', mock.mock_open()), \ + mock.patch('keepercommander.service.util.process_util.subprocess.Popen') as mock_popen, \ + mock.patch('keepercommander.service.util.process_util.sys.platform', 'darwin'): + mock_process = mock.Mock() + mock_popen.return_value = mock_process + + result = spawn_detached_process(['cmd'], '/tmp/test.log') + self.assertIs(result, mock_process) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/service/test_request_validation.py b/unit-tests/service/test_request_validation.py new file mode 100644 index 000000000..b8b918e73 --- /dev/null +++ b/unit-tests/service/test_request_validation.py @@ -0,0 +1,50 @@ +import json +import unittest + +from flask import Flask + +from keepercommander.service.util.request_validation import RequestValidator + +app = Flask(__name__) + + +class TestValidateRequestJson(unittest.TestCase): + def test_valid_json_passes(self): + with app.test_request_context('/', method='POST', + data=json.dumps({"command": "ls"}), + content_type='application/json'): + self.assertIsNone(RequestValidator.validate_request_json()) + + def test_wrong_content_type_rejected(self): + with app.test_request_context('/', method='POST', + data='{"command": "ls"}', + content_type='text/plain'): + response, status = RequestValidator.validate_request_json() + self.assertEqual(status, 400) + self.assertIn('Content-Type', response.get_json()['error']) + + def test_empty_json_object_rejected(self): + """{} is falsy in Python, and was rejected by the old `if not request.json` + check - the get_json()-based rewrite must preserve that, not just check for None.""" + with app.test_request_context('/', method='POST', + data='{}', + content_type='application/json'): + response, status = RequestValidator.validate_request_json() + self.assertEqual(status, 400) + self.assertEqual(response.get_json()['error'], 'Invalid or empty JSON') + + def test_malformed_json_does_not_leak_parser_detail(self): + with app.test_request_context('/', method='POST', + data='{not valid json', + content_type='application/json'): + response, status = RequestValidator.validate_request_json() + self.assertEqual(status, 400) + error_message = response.get_json()['error'] + self.assertEqual(error_message, 'Invalid JSON format') + self.assertNotIn('not valid json', error_message) + self.assertNotIn('line', error_message) + self.assertNotIn('column', error_message) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/service/test_service_app.py b/unit-tests/service/test_service_app.py new file mode 100644 index 000000000..895a633e2 --- /dev/null +++ b/unit-tests/service/test_service_app.py @@ -0,0 +1,35 @@ +import unittest +from unittest import mock + +from keepercommander.service.core.service_app import run_background_service + + +class TestRunBackgroundService(unittest.TestCase): + def test_exits_nonzero_when_port_missing(self): + """A missing port must abort startup, not fall through into flask_app.run(port=None).""" + with mock.patch('keepercommander.service.app.create_app', return_value=mock.Mock()), \ + mock.patch('keepercommander.service.config.service_config.ServiceConfig') as mock_config, \ + mock.patch('keepercommander.service.core.globals.ensure_params_loaded'): + mock_config.return_value.load_config.return_value = {} + + with self.assertRaises(SystemExit) as ctx: + run_background_service() + + self.assertEqual(ctx.exception.code, 1) + + def test_runs_flask_app_when_port_present(self): + mock_flask_app = mock.Mock() + with mock.patch('keepercommander.service.app.create_app', return_value=mock_flask_app), \ + mock.patch('keepercommander.service.config.service_config.ServiceConfig') as mock_config, \ + mock.patch('keepercommander.service.core.globals.ensure_params_loaded'), \ + mock.patch('keepercommander.service.core.service_manager.ServiceManager.get_ssl_context', + return_value=None): + mock_config.return_value.load_config.return_value = {"port": 8000} + + run_background_service() + + mock_flask_app.run.assert_called_once_with(host='0.0.0.0', port=8000, ssl_context=None) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/service/test_service_manager.py b/unit-tests/service/test_service_manager.py index 45ac3925b..aad1e049b 100644 --- a/unit-tests/service/test_service_manager.py +++ b/unit-tests/service/test_service_manager.py @@ -1,3 +1,4 @@ +import os import unittest from unittest import mock from pathlib import Path @@ -7,18 +8,31 @@ from keepercommander.service.core.process_info import ProcessInfo from keepercommander.service.commands.handle_service import StartService, StopService, ServiceStatus +# ProcessInfo.load() only clears os.environ keys present in the *current* .env file, +# so a PID set by one test can leak into a later test's load() via os.environ if that +# test's own save() call doesn't pass the same key. Clear these explicitly per test. +_PROCESS_INFO_ENV_KEYS = ( + 'KEEPER_SERVICE_PID', 'KEEPER_SERVICE_TERMINAL', 'KEEPER_SERVICE_IS_RUNNING', + 'KEEPER_SERVICE_NGROK_PID', 'KEEPER_SERVICE_CLOUDFLARE_PID', +) + + class TestServiceManagement(unittest.TestCase): def setUp(self): self.params = mock.Mock(spec=KeeperParams) ProcessInfo._env_file = Path(__file__).parent / ".test_service.env" - + if ProcessInfo._env_file.exists(): ProcessInfo._env_file.unlink() + for key in _PROCESS_INFO_ENV_KEYS: + os.environ.pop(key, None) def tearDown(self): if ProcessInfo._env_file.exists(): ProcessInfo._env_file.unlink() - + for key in _PROCESS_INFO_ENV_KEYS: + os.environ.pop(key, None) + def test_start_service_when_not_running(self): """Test starting service when no existing service is running""" with mock.patch('keepercommander.service.core.service_manager.ServiceConfig') as mock_config, \ @@ -183,5 +197,132 @@ def test_start_service_with_missing_port(self): mock_print.assert_called_with( "Error: Service configuration is incomplete. Please configure the service port in service_config" ) - + mock_app.run.assert_not_called() + + def _start_background_service(self, config_overrides=None): + """Drive StartService with run_mode=background and no tunnels enabled, capturing + the spawn_detached_process call so tests can assert on cmd/env without spawning anything.""" + config_data = {"port": 8000, "run_mode": "background"} + config_data.update(config_overrides or {}) + + with mock.patch('keepercommander.service.core.service_manager.ServiceConfig') as mock_config, \ + mock.patch('keepercommander.service.core.service_manager.spawn_detached_process') as mock_spawn, \ + mock.patch('sys.executable', '/usr/bin/python3'): + mock_config.return_value.load_config.return_value = config_data + mock_spawn.return_value = mock.Mock(pid=99999) + + start_cmd = StartService() + start_cmd.execute(self.params) + + return mock_spawn + + def test_start_service_background_frozen_uses_internal_flag(self): + """A frozen exe can't be invoked with -m, so background mode must signal it via an + explicit argv flag - not an env var, which would leak into subprocesses the service + itself spawns and could be tripped by a stale value left in the environment.""" + with mock.patch('sys.frozen', True, create=True), \ + mock.patch('sys._MEIPASS', '/frozen/path', create=True): + mock_spawn = self._start_background_service() + + mock_spawn.assert_called_once() + args, kwargs = mock_spawn.call_args + cmd = args[0] + self.assertEqual(cmd, ['/usr/bin/python3', '--internal-run-service']) + self.assertNotIn('KEEPER_SERVICE_MODE', kwargs['env']) + + def test_start_service_background_not_frozen_uses_module_flag(self): + """Running from source (not frozen) must use -m, and must not pass the frozen-only flag.""" + mock_spawn = self._start_background_service() + + mock_spawn.assert_called_once() + args, kwargs = mock_spawn.call_args + cmd = args[0] + self.assertEqual(cmd, ['/usr/bin/python3', '-m', 'keepercommander.service.core.service_app']) + self.assertNotIn('--internal-run-service', cmd) + + def test_start_service_background_forces_unbuffered_child_output(self): + mock_spawn = self._start_background_service() + + _, kwargs = mock_spawn.call_args + self.assertEqual(kwargs['env']['PYTHONUNBUFFERED'], '1') + self.assertTrue(kwargs['append']) + + def test_start_service_background_saves_ngrok_and_cloudflare_pids(self): + """service-stop can only learn tunnel PIDs for a background-mode service via the + saved .env file (a later CLI invocation has no shared in-memory state), so both + ngrok_pid and cloudflare_pid must be persisted, not just the service's own pid.""" + with mock.patch('keepercommander.service.core.service_manager.ServiceConfig') as mock_config, \ + mock.patch('keepercommander.service.config.ngrok_config.NgrokConfigurator.configure_ngrok', + return_value=5555), \ + mock.patch('keepercommander.service.config.cloudflare_config.CloudflareConfigurator.configure_cloudflare', + return_value=6666), \ + mock.patch('keepercommander.service.core.service_manager.spawn_detached_process', + return_value=mock.Mock(pid=99999)), \ + mock.patch('sys.executable', '/usr/bin/python3'): + mock_config.return_value.load_config.return_value = { + "port": 8000, "run_mode": "background", "ngrok": "y", "cloudflare": "y" + } + + start_cmd = StartService() + start_cmd.execute(self.params) + + process_info = ProcessInfo.load() + self.assertEqual(process_info.pid, 99999) + self.assertEqual(process_info.ngrok_pid, 5555) + self.assertEqual(process_info.cloudflare_pid, 6666) + + def test_start_service_foreground_saves_ngrok_and_cloudflare_pids(self): + """service-stop from a second terminal can only learn tunnel PIDs for a foreground + service via the saved .env file too - the in-process cleanup closure isn't reachable + from a separate CLI invocation, so both PIDs must be persisted here as well.""" + # flask_app.run() is mocked as a no-op, so it "returns" immediately and the + # foreground branch's cleanup-on-exit runs right after (as it would on a real + # Ctrl+C) - capture ProcessInfo state during the run() call, before that cleanup + # clears it. + captured = {} + + def capture_process_info(**kwargs): + info = ProcessInfo.load() + captured['pid'] = info.pid + captured['ngrok_pid'] = info.ngrok_pid + captured['cloudflare_pid'] = info.cloudflare_pid + + mock_flask_app = mock.Mock() + mock_flask_app.run.side_effect = capture_process_info + + with mock.patch('keepercommander.service.core.service_manager.ServiceConfig') as mock_config, \ + mock.patch('keepercommander.service.config.ngrok_config.NgrokConfigurator.configure_ngrok', + return_value=5555), \ + mock.patch('keepercommander.service.config.cloudflare_config.CloudflareConfigurator.configure_cloudflare', + return_value=6666), \ + mock.patch('keepercommander.service.app.create_app', return_value=mock_flask_app), \ + mock.patch('os.getpid', return_value=88888): + mock_config.return_value.load_config.return_value = { + "port": 8000, "run_mode": "foreground", "ngrok": "y", "cloudflare": "y" + } + + start_cmd = StartService() + start_cmd.execute(self.params) + + self.assertEqual(captured['pid'], 88888) + self.assertEqual(captured['ngrok_pid'], 5555) + self.assertEqual(captured['cloudflare_pid'], 6666) + + def test_start_service_ngrok_configure_failure_is_handled_gracefully(self): + """An ngrok setup failure (e.g. the binary being deleted by AV, or a failed + download) must not crash the whole service start or skip cleanup.""" + with mock.patch('keepercommander.service.core.service_manager.ServiceConfig') as mock_config, \ + mock.patch('keepercommander.service.config.ngrok_config.NgrokConfigurator.configure_ngrok', + side_effect=RuntimeError("ngrok binary not found")) as mock_configure_ngrok, \ + mock.patch('keepercommander.service.core.service_manager.spawn_detached_process') as mock_spawn: + mock_config.return_value.load_config.return_value = { + "port": 8000, "run_mode": "background", "ngrok": "y" + } + + start_cmd = StartService() + start_cmd.execute(self.params) # must not raise + + mock_configure_ngrok.assert_called_once() + mock_spawn.assert_not_called() + self.assertFalse(ProcessInfo._env_file.exists()) diff --git a/unit-tests/service/test_tunneling.py b/unit-tests/service/test_tunneling.py new file mode 100644 index 000000000..6de58970d --- /dev/null +++ b/unit-tests/service/test_tunneling.py @@ -0,0 +1,272 @@ +import os +import tempfile +import unittest +from unittest import mock + +from keepercommander.service.util import tunneling + + +class TestStartNgrok(unittest.TestCase): + def _mock_ngrok_setup(self): + return [ + mock.patch('keepercommander.service.util.tunneling.conf.get_default', return_value=mock.Mock(ngrok_path='/bin/ngrok')), + mock.patch('keepercommander.service.util.tunneling.ngrok.install_ngrok'), + mock.patch('keepercommander.service.util.tunneling.time.sleep'), + ] + + def test_raises_when_ngrok_exits_immediately(self): + """A dead-on-arrival ngrok process (bad token, invalid flags, ...) must be + reported as a failure, not returned to the caller as if it started fine.""" + mock_process = mock.Mock(pid=111, returncode=1) + mock_process.poll.return_value = 1 # process has already exited + + patches = self._mock_ngrok_setup() + with patches[0], patches[1], patches[2], \ + mock.patch('keepercommander.service.util.tunneling.spawn_detached_process', return_value=mock_process): + with self.assertRaises(RuntimeError): + tunneling.start_ngrok(port=8000, auth_token='tok') + + def test_returns_pid_when_ngrok_stays_running(self): + mock_process = mock.Mock(pid=111, returncode=None) + mock_process.poll.return_value = None # still running + + patches = self._mock_ngrok_setup() + with patches[0], patches[1], patches[2], \ + mock.patch('keepercommander.service.util.tunneling.spawn_detached_process', return_value=mock_process): + # No real process has ppid 111, so the psutil lookup falls through and + # start_ngrok returns the wrapper PID unchanged. + pid = tunneling.start_ngrok(port=8000, auth_token='tok') + + self.assertEqual(pid, 111) + + +class TestStartNgrokWithUrl(unittest.TestCase): + def test_raises_when_process_died_after_the_initial_doa_check(self): + """A slower failure (e.g. a bad auth token) can exit ngrok after start_ngrok's + initial liveness check but before any URL ever becomes available - that must + still be reported as a failure, not returned as a silent (pid, None).""" + with mock.patch('keepercommander.service.util.tunneling.start_ngrok', return_value=111), \ + mock.patch('keepercommander.service.util.tunneling.time.sleep'), \ + mock.patch('keepercommander.service.util.tunneling.get_ngrok_url_from_api', return_value=None), \ + mock.patch('keepercommander.service.util.tunneling.get_ngrok_url_from_log', return_value=None), \ + mock.patch('psutil.pid_exists', return_value=False): + with self.assertRaises(RuntimeError): + tunneling.start_ngrok_with_url(port=8000, auth_token='tok') + + def test_returns_url_when_process_is_alive(self): + with mock.patch('keepercommander.service.util.tunneling.start_ngrok', return_value=111), \ + mock.patch('keepercommander.service.util.tunneling.time.sleep'), \ + mock.patch('keepercommander.service.util.tunneling.get_ngrok_url_from_api', + return_value='https://example.ngrok.io'): + pid, public_url = tunneling.start_ngrok_with_url(port=8000, auth_token='tok') + + self.assertEqual(pid, 111) + self.assertEqual(public_url, 'https://example.ngrok.io') + + def test_no_url_but_process_still_alive_does_not_raise(self): + """A tunnel that's just slow to report its URL (still alive) shouldn't be + treated as a failure - only an actually-dead process should be.""" + with mock.patch('keepercommander.service.util.tunneling.start_ngrok', return_value=111), \ + mock.patch('keepercommander.service.util.tunneling.time.sleep'), \ + mock.patch('keepercommander.service.util.tunneling.get_ngrok_url_from_api', return_value=None), \ + mock.patch('keepercommander.service.util.tunneling.get_ngrok_url_from_log', return_value=None), \ + mock.patch('psutil.pid_exists', return_value=True): + pid, public_url = tunneling.start_ngrok_with_url(port=8000, auth_token='tok') + + self.assertEqual(pid, 111) + self.assertIsNone(public_url) + + +class TestGenerateNgrokUrl(unittest.TestCase): + def test_background_mode_skips_fd_redirection_and_uses_subprocess(self): + with mock.patch('keepercommander.service.util.tunneling.start_ngrok_with_url', + return_value=(1234, 'https://example.ngrok.io')) as mock_start, \ + mock.patch('keepercommander.service.util.tunneling.os.dup') as mock_dup: + public_url, ngrok_pid = tunneling.generate_ngrok_url( + port=8000, auth_token='tok', ngrok_custom_domain=None, run_mode='background' + ) + + self.assertEqual(public_url, 'https://example.ngrok.io') + self.assertEqual(ngrok_pid, 1234) + mock_start.assert_called_once_with(port=8000, auth_token='tok') + # Background mode must not touch the process's own stdout/stderr fds - + # doing so previously corrupted the interactive shell's console on Windows. + mock_dup.assert_not_called() + + def test_foreground_mode_uses_pyngrok_connect(self): + mock_tunnel = mock.Mock(public_url='https://example.ngrok.io') + with mock.patch('keepercommander.service.util.tunneling.ngrok.connect', return_value=mock_tunnel) as mock_connect, \ + mock.patch('keepercommander.service.util.tunneling.os.dup', side_effect=[1, 2]), \ + mock.patch('keepercommander.service.util.tunneling.os.dup2'), \ + mock.patch('keepercommander.service.util.tunneling.os.open', return_value=3), \ + mock.patch('keepercommander.service.util.tunneling.os.close'): + public_url, ngrok_pid = tunneling.generate_ngrok_url( + port=8000, auth_token='tok', ngrok_custom_domain=None, run_mode='foreground' + ) + + self.assertEqual(public_url, 'https://example.ngrok.io') + self.assertIsNone(ngrok_pid) + mock_connect.assert_called_once() + + def test_missing_port_or_token_raises(self): + with self.assertRaises(ValueError): + tunneling.generate_ngrok_url(port=None, auth_token='tok', ngrok_custom_domain=None, run_mode='background') + with self.assertRaises(ValueError): + tunneling.generate_ngrok_url(port=8000, auth_token=None, ngrok_custom_domain=None, run_mode='background') + + def test_foreground_mode_closes_dupd_fds_if_redirection_setup_fails_partway(self): + """If os.dup(1)/os.dup(2) succeed but the later os.open(devnull) fails, the + duplicated fds must be closed instead of leaked - and ngrok should still start.""" + mock_tunnel = mock.Mock(public_url='https://example.ngrok.io') + with mock.patch('keepercommander.service.util.tunneling.ngrok.connect', return_value=mock_tunnel) as mock_connect, \ + mock.patch('keepercommander.service.util.tunneling.os.dup', side_effect=[10, 11]), \ + mock.patch('keepercommander.service.util.tunneling.os.open', side_effect=OSError("too many open files")), \ + mock.patch('keepercommander.service.util.tunneling.os.dup2'), \ + mock.patch('keepercommander.service.util.tunneling.os.close') as mock_close: + public_url, ngrok_pid = tunneling.generate_ngrok_url( + port=8000, auth_token='tok', ngrok_custom_domain=None, run_mode='foreground' + ) + + self.assertEqual(public_url, 'https://example.ngrok.io') + mock_connect.assert_called_once() + mock_close.assert_any_call(10) + mock_close.assert_any_call(11) + + def test_foreground_mode_restores_stdout_if_second_dup2_fails(self): + """If dup2(devnull, 1) succeeds but dup2(devnull, 2) then fails, fd 1 must be + restored back to the original console fd before the backup is closed - otherwise + fd 1 is left pointed at devnull permanently, silently losing all console output.""" + mock_tunnel = mock.Mock(public_url='https://example.ngrok.io') + dup2_calls = [] + + def dup2_side_effect(fd, target): + dup2_calls.append((fd, target)) + if fd == 20 and target == 2: + raise OSError("bad file descriptor") + + with mock.patch('keepercommander.service.util.tunneling.ngrok.connect', return_value=mock_tunnel) as mock_connect, \ + mock.patch('keepercommander.service.util.tunneling.os.dup', side_effect=[10, 11]), \ + mock.patch('keepercommander.service.util.tunneling.os.open', return_value=20), \ + mock.patch('keepercommander.service.util.tunneling.os.dup2', side_effect=dup2_side_effect), \ + mock.patch('keepercommander.service.util.tunneling.os.close') as mock_close: + public_url, _ = tunneling.generate_ngrok_url( + port=8000, auth_token='tok', ngrok_custom_domain=None, run_mode='foreground' + ) + + self.assertEqual(public_url, 'https://example.ngrok.io') + mock_connect.assert_called_once() + # fd 1 was redirected to devnull (20) then must be restored back to the + # original console fd (10) before that backup is closed. + self.assertIn((20, 1), dup2_calls) + self.assertIn((10, 1), dup2_calls) + mock_close.assert_any_call(10) + mock_close.assert_any_call(11) + + +class TestGetCloudflareUrlFromLog(unittest.TestCase): + """The localhost/127.0.0.1 exclusion must gate every OR-branch (trycloudflare.com, + cfargotunnel.com, custom_domain) - a prior operator-precedence bug let the + cfargotunnel.com/custom_domain branches bypass that exclusion entirely.""" + + def setUp(self): + fd, self.log_path = tempfile.mkstemp() + os.close(fd) + + def tearDown(self): + os.unlink(self.log_path) + + def _write_log(self, content): + with open(self.log_path, 'w') as f: + f.write(content) + + def test_rejects_a_match_that_contains_localhost_even_if_it_also_contains_the_custom_domain(self): + # The regex excludes '/', so this is a single continuous match containing both + # 'localhost' and the custom domain - exactly what let the old buggy precedence + # accept it via the unguarded custom_domain OR-branch. No match should pass the + # filter, so this falls through to the constructed https://{custom_domain} URL, + # not the bad captured match. + self._write_log("https://localhost.acme.example.com\n") + url = tunneling.get_cloudflare_url_from_log(self.log_path, custom_domain='acme.example.com', max_retries=1) + self.assertEqual(url, 'https://acme.example.com') + + def test_accepts_cfargotunnel_url(self): + self._write_log("connected to https://mytunnel.cfargotunnel.com\n") + url = tunneling.get_cloudflare_url_from_log(self.log_path, custom_domain=None, max_retries=1) + self.assertEqual(url, 'https://mytunnel.cfargotunnel.com') + + def test_no_custom_domain_still_rejects_localhost(self): + """Previously, a falsy custom_domain made the whole filter degrade to accept-anything.""" + self._write_log("https://localhost\n") + url = tunneling.get_cloudflare_url_from_log(self.log_path, custom_domain=None, max_retries=1) + self.assertIsNone(url) + + +class TestGenerateCloudflareUrl(unittest.TestCase): + def test_returns_public_url_and_pid_in_correct_order(self): + # start_cloudflare_tunnel_with_url returns (pid, url); generate_cloudflare_url + # must swap that to (url, pid) to match what CloudflareConfigurator expects. + with mock.patch('keepercommander.service.util.tunneling.start_cloudflare_tunnel_with_url', + return_value=(4321, 'https://tunnel.example.com')): + public_url, tunnel_pid = tunneling.generate_cloudflare_url( + port=8000, tunnel_token='tok', custom_domain=None, run_mode='background' + ) + + self.assertEqual(public_url, 'https://tunnel.example.com') + self.assertEqual(tunnel_pid, 4321) + + def test_missing_port_raises(self): + with self.assertRaises(ValueError): + tunneling.generate_cloudflare_url(port=None, tunnel_token='tok', custom_domain=None, run_mode='background') + + def test_missing_tunnel_token_raises(self): + with self.assertRaises(ValueError): + tunneling.generate_cloudflare_url(port=8000, tunnel_token=' ', custom_domain=None, run_mode='background') + + +class TestDownloadCloudflared(unittest.TestCase): + def test_lookup_failure_is_logged_not_swallowed_silently(self): + # Force platform.system() to an unsupported value so _download_cloudflared + # raises right after the (logged) lookup failure, without attempting a real download. + with mock.patch('keepercommander.service.util.tunneling.subprocess.run', + side_effect=OSError("cloudflared not found")), \ + mock.patch('keepercommander.service.util.tunneling.logging.debug') as mock_debug, \ + mock.patch('platform.system', return_value='unsupported'): + with self.assertRaises(Exception): + tunneling._download_cloudflared() + + mock_debug.assert_called_once() + self.assertIn('cloudflared', mock_debug.call_args[0][0]) + + def test_windows_never_searches_path_or_cwd(self): + """`where` (like shutil.which) searches the current directory before PATH on + Windows - a planted cloudflared.exe in whatever directory the user happens to + run Commander from would get executed. Only search PATH on POSIX, where `which` + doesn't consult the current directory.""" + with mock.patch('keepercommander.service.util.tunneling.sys.platform', 'win32'), \ + mock.patch('keepercommander.service.util.tunneling.subprocess.run') as mock_run, \ + mock.patch('platform.system', return_value='unsupported'): + with self.assertRaises(Exception): + tunneling._download_cloudflared() + + mock_run.assert_not_called() + + +class TestTunnelLogFiles(unittest.TestCase): + def test_ngrok_and_cloudflare_logs_share_the_same_directory(self): + ngrok_log = tunneling.get_tunnel_log_file('ngrok_subprocess.log') + cloudflare_log = tunneling.get_tunnel_log_file('cloudflare_tunnel_subprocess.log') + + self.assertEqual(os.path.dirname(ngrok_log), os.path.dirname(cloudflare_log)) + + def test_resolves_the_data_dir_at_call_time_not_import_time(self): + """A --data-dir/KEEPER_DATA_HOME override set after this module is imported + must still be honored - the log dir can't be cached as a module-level constant.""" + import tempfile + overridden_dir = tempfile.mkdtemp() + with mock.patch('keepercommander.utils.get_default_path', return_value=overridden_dir): + log_file = tunneling.get_tunnel_log_file('ngrok_subprocess.log') + self.assertTrue(log_file.startswith(os.path.join(overridden_dir, 'service_logs'))) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_api_sanitization.py b/unit-tests/test_api_sanitization.py new file mode 100644 index 000000000..7928d14f4 --- /dev/null +++ b/unit-tests/test_api_sanitization.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import unittest +import json +from keepercommander.sanitization import ( + mask_field_value, + sanitize_protobuf_json +) + +class TestAPISanitization(unittest.TestCase): + """Test cases for sensitive data sanitization in API logging.""" + + def test_sanitize_protobuf_json_with_payment_card(self): + """Test sanitization of paymentCard data.""" + json_data = { + "records": [{ + "recordUid": "test123", + "data": { + "type": "paymentCard", + "value": { + "cardNumber": "4111111111111111", + "cardExpirationDate": "05/2025", + "cardSecurityCode": "123" + } + } + }] + } + json_str = json.dumps(json_data) + sanitized = sanitize_protobuf_json(json_str) + result = json.loads(sanitized) + + # Verify sensitive fields are masked + self.assertEqual(result["records"][0]["data"]["value"]["cardNumber"], "***") + self.assertEqual(result["records"][0]["data"]["value"]["cardExpirationDate"], "***") + self.assertEqual(result["records"][0]["data"]["value"]["cardSecurityCode"], "***") + + def test_sanitize_protobuf_json_with_bank_account(self): + """Test sanitization of bankAccount data.""" + json_data = { + "records": [{ + "recordUid": "test456", + "data": { + "type": "bankAccount", + "value": { + "accountType": "Checking", + "routingNumber": "123456789", + "accountNumber": "98765432109876", + "otherType": "" + } + } + }] + } + json_str = json.dumps(json_data) + sanitized = sanitize_protobuf_json(json_str) + result = json.loads(sanitized) + + # Verify sensitive fields are masked + self.assertEqual(result["records"][0]["data"]["value"]["routingNumber"], "***") + self.assertEqual(result["records"][0]["data"]["value"]["accountNumber"], "***") + + def test_sanitize_protobuf_json_with_ssh_keys(self): + """Test sanitization of sshKeys (keyPair) data.""" + private_key = "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDU5Z8P2Z9q\n-----END PRIVATE KEY-----" + public_key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1OWfD9mfagMCACEA\n-----END PUBLIC KEY-----" + + json_data = { + "records": [{ + "recordUid": "test789", + "data": { + "type": "keyPair", + "value": { + "privateKey": private_key, + "publicKey": public_key + } + } + }] + } + json_str = json.dumps(json_data) + sanitized = sanitize_protobuf_json(json_str) + result = json.loads(sanitized) + + # Verify sensitive fields are masked + self.assertEqual(result["records"][0]["data"]["value"]["privateKey"], "***") + self.assertEqual(result["records"][0]["data"]["value"]["publicKey"], "***") + + def test_sanitize_protobuf_json_with_software_license(self): + """Test sanitization of softwareLicense data.""" + json_data = { + "records": [{ + "recordUid": "test101", + "fields": [{ + "type": "licenseNumber", + "value": ["LICENSE-2024-0123456789"] + }] + }] + } + json_str = json.dumps(json_data) + sanitized = sanitize_protobuf_json(json_str) + result = json.loads(sanitized) + + # Verify sensitive field is masked + self.assertEqual(result["records"][0]["fields"][0]["value"], ["***"]) + + def test_sanitize_preserves_non_sensitive_data(self): + """Test that non-sensitive data is preserved.""" + json_data = { + "records": [{ + "recordUid": "test202", + "title": "Test Record", + "notes": "Some notes" + }] + } + json_str = json.dumps(json_data) + sanitized = sanitize_protobuf_json(json_str) + result = json.loads(sanitized) + + # Verify non-sensitive data is preserved + self.assertEqual(result["records"][0]["recordUid"], "test202") + self.assertEqual(result["records"][0]["title"], "Test Record") + self.assertEqual(result["records"][0]["notes"], "Some notes") + + def test_mask_field_value_with_dict(self): + """Test masking of dict values.""" + value = {"key1": "value1", "key2": "value2"} + masked = mask_field_value(value) + self.assertEqual(masked, {"key1": "***", "key2": "***"}) + + def test_mask_field_value_with_list(self): + """Test masking of list values.""" + value = ["value1", "value2", "value3"] + masked = mask_field_value(value) + self.assertEqual(masked, ["***", "***", "***"]) + + def test_mask_field_value_with_string(self): + """Test masking of string values.""" + value = "sensitive_data" + masked = mask_field_value(value) + self.assertEqual(masked, "***") + + def test_sanitize_invalid_json(self): + """Test sanitization with invalid JSON returns the original string.""" + invalid_json = "not valid json" + result = sanitize_protobuf_json(invalid_json) + self.assertEqual(result, invalid_json) + + def test_sanitize_nested_list_of_records(self): + """Test sanitization of nested list structures.""" + json_data = { + "records": [ + { + "recordUid": "uid1", + "data": { + "type": "bankAccount", + "value": { + "accountNumber": "1234567890", + "routingNumber": "0987654321" + } + } + }, + { + "recordUid": "uid2", + "data": { + "type": "paymentCard", + "value": { + "cardNumber": "4111111111111111" + } + } + } + ] + } + json_str = json.dumps(json_data) + sanitized = sanitize_protobuf_json(json_str) + result = json.loads(sanitized) + + # Verify all sensitive data in list is masked + self.assertEqual(result["records"][0]["data"]["value"]["accountNumber"], "***") + self.assertEqual(result["records"][0]["data"]["value"]["routingNumber"], "***") + self.assertEqual(result["records"][1]["data"]["value"]["cardNumber"], "***") + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_command_enterprise.py b/unit-tests/test_command_enterprise.py index 9e56f87a4..32dd05e9c 100644 --- a/unit-tests/test_command_enterprise.py +++ b/unit-tests/test_command_enterprise.py @@ -87,6 +87,83 @@ def test_enterprise_info_users_verbose_returns_ids(self): 'role_name': ent_env.role1_name, }]) + def test_enterprise_info_uses_root_displayname(self): + params = get_connected_params() + api.query_enterprise(params) + params.enterprise['nodes'][0]['data']['displayname'] = 'ECC-cmdr' + cmd = enterprise.EnterpriseInfoCommand() + + report = json.loads(cmd.execute(params, nodes=True, format='json', quiet=True)) + root = next(x for x in report if x['node_id'] == ent_env.node1_id) + self.assertEqual(root['name'], 'ECC-cmdr') + self.assertIn('ECC-cmdr', cmd.execute(params, quiet=True)) + self.assertIn('ECC-cmdr', cmd.execute(params, nodes=True, format='csv', quiet=True)) + + self.assertEqual( + [x['node_id'] for x in cmd.resolve_nodes(params, 'ECC-cmdr')], + [ent_env.node1_id], + ) + self.assertEqual( + [x['node_id'] for x in cmd.resolve_nodes(params, 'Enterprise 1')], + [ent_env.node1_id], + ) + self.assertEqual(cmd.get_node_path(params, ent_env.node2_id), 'ECC-cmdr\\Sub node 1') + + def test_enterprise_node_path_cache_refreshes_displayname(self): + params = get_connected_params() + api.query_enterprise(params) + cmd = enterprise.EnterpriseInfoCommand() + + self.assertEqual(cmd.get_node_path(params, ent_env.node2_id), 'Enterprise 1\\Sub node 1') + params.enterprise['nodes'][0]['data']['displayname'] = 'ECC-cmdr' + self.assertEqual(cmd.get_node_path(params, ent_env.node2_id), 'ECC-cmdr\\Sub node 1') + + def test_enterprise_node_rename_root_omits_parent_id(self): + params = get_connected_params() + api.query_enterprise(params) + cmd = enterprise.EnterpriseNodeCommand() + + with mock.patch('keepercommander.commands.enterprise.api.execute_batch', + return_value=[{'result': 'success'}]) as execute_batch: + cmd.execute(params, node=[str(ent_env.node1_id)], displayname='Renamed Enterprise') + + request = execute_batch.call_args.args[1][0] + self.assertNotIn('parent_id', request) + data = crypto.decrypt_aes_v1( + utils.base64_url_decode(request['encrypted_data']), + params.enterprise['unencrypted_tree_key'], + ) + self.assertEqual(json.loads(data.decode('utf-8'))['displayname'], 'Renamed Enterprise') + + def test_enterprise_node_rename_child_preserves_parent_id(self): + params = get_connected_params() + api.query_enterprise(params) + cmd = enterprise.EnterpriseNodeCommand() + + with mock.patch('keepercommander.commands.enterprise.api.execute_batch', + return_value=[{'result': 'success'}]) as execute_batch: + cmd.execute(params, node=[str(ent_env.node2_id)], displayname='Renamed Child') + + request = execute_batch.call_args.args[1][0] + self.assertEqual(request['parent_id'], ent_env.node1_id) + data = crypto.decrypt_aes_v1( + utils.base64_url_decode(request['encrypted_data']), + params.enterprise['unencrypted_tree_key'], + ) + self.assertEqual(json.loads(data.decode('utf-8'))['displayname'], 'Renamed Child') + + def test_enterprise_node_move_sets_selected_parent_id(self): + params = get_connected_params() + api.query_enterprise(params) + cmd = enterprise.EnterpriseNodeCommand() + + with mock.patch('keepercommander.commands.enterprise.api.execute_batch', + return_value=[{'result': 'success'}]) as execute_batch: + cmd.execute(params, node=[str(ent_env.node2_id)], parent=str(ent_env.node1_id)) + + request = execute_batch.call_args.args[1][0] + self.assertEqual(request['parent_id'], ent_env.node1_id) + def test_enterprise_add_user(self): params = get_connected_params() api.query_enterprise(params) @@ -191,6 +268,253 @@ def test_enterprise_role(self): with mock.patch('builtins.print'): cmd.execute(params, add_user=['invalid@keepersecurity.com'], verbose=True, role=[ent_env.role1_name]) + def test_enterprise_role_add_transfer_account_privilege_denied(self): + """KC-1412: Delegated admin without transfer_account privilege cannot grant it (CVE fix)""" + params = get_connected_params() + api.query_enterprise(params) + + cmd = enterprise.EnterpriseRoleCommand() + # Role1 (current user) has only manage_nodes, manage_user, manage_roles (no transfer_account) + # Try to grant transfer_account to Role2 - should be denied by KC-1412 fix + with self.assertLogs(level=logging.WARNING) as log: + cmd.execute(params, add_privilege=['transfer_account'], role=[ent_env.role2_name], + node='Enterprise 1') + # KC-1412 fix: Check for the privilege denial message + self.assertTrue(any('You do not have the required privilege' in msg for msg in log.output)) + + # Expected: no command sent to server (returned early due to lack of authorization) + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_add_manage_teams_privilege_denied(self): + """KC-1412: Delegated admin without manage_teams privilege cannot grant it (CVE fix)""" + params = get_connected_params() + api.query_enterprise(params) + + cmd = enterprise.EnterpriseRoleCommand() + # Role1 has no manage_teams privilege - try to grant it, should be denied + with self.assertLogs(level=logging.WARNING) as log: + cmd.execute(params, add_privilege=['manage_teams'], role=[ent_env.role2_name], + node='Enterprise 1') + # KC-1412 fix: Check for the privilege denial message + self.assertTrue(any('You do not have the required privilege' in msg for msg in log.output)) + + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_add_privilege_with_authorization(self): + """KC-1412: Admin with transfer_account privilege CAN grant it (regression test)""" + params = get_connected_params() + api.query_enterprise(params) + + # Use the admin role which has transfer_account in test data + # Add admin role to User1 so they have the privilege + params.enterprise['role_users'].append({ + 'role_id': ent_env.role_admin_id, + 'enterprise_user_id': ent_env.user1_id + }) + + cmd = enterprise.EnterpriseRoleCommand() + TestEnterprise.expected_commands = ['managed_node_privilege_add'] + # Now User1 has transfer_account via Admin role, can grant it to Role2 + cmd.execute(params, add_privilege=['transfer_account'], role=[ent_env.role2_name], + node='Enterprise 1') + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_add_manage_companies_privilege_denied(self): + """Delegated admin without manage_companies privilege cannot grant it""" + params = get_connected_params() + api.query_enterprise(params) + + cmd = enterprise.EnterpriseRoleCommand() + with self.assertLogs(level=logging.WARNING) as log: + cmd.execute(params, add_privilege=['manage_companies'], role=[ent_env.role2_name], + node='Enterprise 1') + self.assertTrue(any('You do not have the required privilege' in msg for msg in log.output)) + + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_remove_transfer_account_privilege_denied(self): + """Delegated admin without transfer_account privilege cannot strip it from another role""" + params = get_connected_params() + api.query_enterprise(params) + + # Role2 already holds transfer_account; User1 (Role1) does not. + params.enterprise['role_privileges'].append({ + 'role_id': ent_env.role2_id, + 'managed_node_id': ent_env.node1_id, + 'privilege': 'transfer_account' + }) + + cmd = enterprise.EnterpriseRoleCommand() + with self.assertLogs(level=logging.WARNING) as log: + cmd.execute(params, remove_privilege=['transfer_account'], role=[ent_env.role2_name], + node='Enterprise 1') + self.assertTrue(any('You do not have the required privilege' in msg for msg in log.output)) + + # No removal command sent - Role2 should still hold the privilege + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_remove_privilege_with_authorization(self): + """Admin with transfer_account privilege CAN remove it from another role (regression test)""" + params = get_connected_params() + api.query_enterprise(params) + + # Role2 already holds transfer_account + params.enterprise['role_privileges'].append({ + 'role_id': ent_env.role2_id, + 'managed_node_id': ent_env.node1_id, + 'privilege': 'transfer_account' + }) + # Give User1 transfer_account via Admin role + params.enterprise['role_users'].append({ + 'role_id': ent_env.role_admin_id, + 'enterprise_user_id': ent_env.user1_id + }) + + cmd = enterprise.EnterpriseRoleCommand() + TestEnterprise.expected_commands = ['managed_node_privilege_remove'] + cmd.execute(params, remove_privilege=['transfer_account'], role=[ent_env.role2_name], + node='Enterprise 1') + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_require_account_share_enforcement_denied_delegated_admin(self): + """KC-1412: Delegated admin (non-root) cannot set require_account_share enforcement (CVE fix)""" + params = get_connected_params() + api.query_enterprise(params) + + # Node2 is a sub-node (has parent_id), so user in that node is not root admin + # Modify to make Role1 manage Node2 instead + params.enterprise['managed_nodes'] = [ + { + 'role_id': ent_env.role1_id, + 'managed_node_id': ent_env.node2_id, + 'cascade_node_management': True, + } + ] + + cmd = enterprise.EnterpriseRoleCommand() + # Try to set require_account_share enforcement by role name + # This should be denied because the user is not a root admin (managing non-root node) + # KC-1412 fix should reject this with a warning and return early + with self.assertLogs(level=logging.WARNING): + cmd.execute(params, enforcements=[f'require_account_share:Admin Role'], + role=[ent_env.role1_name]) + + # No command should be sent to server (returned early due to root admin check) + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_require_account_share_enforcement_allowed_root_admin(self): + """KC-1412: Root admin CAN set require_account_share enforcement (regression test)""" + params = get_connected_params() + api.query_enterprise(params) + + # Role1 manages Node1 (root node, no parent_id) - user is root admin + # Set enforcement to Admin Role which has transfer_account privilege + cmd = enterprise.EnterpriseRoleCommand() + TestEnterprise.expected_commands = ['role_enforcement_add'] + cmd.execute(params, enforcements=['require_account_share:Admin Role'], + role=[ent_env.role1_name]) + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_other_enforcements_work_delegated_admin(self): + """KC-1412: Delegated admin CAN set non-sensitive enforcements (regression test)""" + params = get_connected_params() + api.query_enterprise(params) + + cmd = enterprise.EnterpriseRoleCommand() + # Non-sensitive enforcement should work for delegated admin + TestEnterprise.expected_commands = ['role_enforcement_add'] + cmd.execute(params, enforcements=['require_two_factor:True'], + role=[ent_env.role1_name]) + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_cascade_privilege_grant(self): + """KC-1435: Privilege grant on child node respects cascade from parent (regression test)""" + params = get_connected_params() + api.query_enterprise(params) + + # Set up: Admin role has transfer_account on Node1 (root) + # Add admin role's transfer_account privilege to parent node + params.enterprise['role_privileges'].append({ + 'role_id': ent_env.role_admin_id, + 'managed_node_id': ent_env.node1_id, + 'privilege': 'transfer_account' + }) + # Add User1 to admin role so they have transfer_account via cascade + params.enterprise['role_users'].append({ + 'role_id': ent_env.role_admin_id, + 'enterprise_user_id': ent_env.user1_id + }) + # Role2 manages Node2 (child of Node1) with cascade enabled + params.enterprise['managed_nodes'].append({ + 'role_id': ent_env.role2_id, + 'managed_node_id': ent_env.node2_id, + 'cascade_node_management': False, + }) + + cmd = enterprise.EnterpriseRoleCommand() + TestEnterprise.expected_commands = ['managed_node_privilege_add'] + # Attempt to grant transfer_account on the child node — should succeed because + # the user's transfer_account privilege on parent Node1 cascades down to Node2 + cmd.execute(params, add_privilege=['transfer_account'], role=[ent_env.role2_name], + node='Sub node 1') + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_mixed_case_username(self): + """KC-1435: Username matching is case-insensitive (regression test)""" + params = get_connected_params() + # Mock user with mixed-case username + params.user = 'User@TEST.COM' + api.query_enterprise(params) + + # Update the test data user to match in lowercase + params.enterprise['users'][0]['username'] = 'user@test.com' + + # Add admin role to User1 for transfer_account + params.enterprise['role_users'].append({ + 'role_id': ent_env.role_admin_id, + 'enterprise_user_id': ent_env.user1_id + }) + + cmd = enterprise.EnterpriseRoleCommand() + TestEnterprise.expected_commands = ['managed_node_privilege_add'] + # Should succeed despite mixed-case username in params.user + cmd.execute(params, add_privilege=['transfer_account'], role=[ent_env.role2_name], + node='Enterprise 1') + self.assertEqual(len(TestEnterprise.expected_commands), 0) + + def test_enterprise_role_enforcement_removal_restricted_non_root(self): + """KC-1435: Delegated admin cannot remove require_account_share enforcement (CVE fix)""" + params = get_connected_params() + api.query_enterprise(params) + + # Set up Role1 to manage Node2 (non-root) + params.enterprise['managed_nodes'] = [ + { + 'role_id': ent_env.role1_id, + 'managed_node_id': ent_env.node2_id, + 'cascade_node_management': True, + } + ] + # Add existing enforcement on Role2 + params.enterprise['role_enforcements'] = [ + { + 'role_id': ent_env.role2_id, + 'enforcements': { + 'require_account_share': 'Admin Role' + } + } + ] + + cmd = enterprise.EnterpriseRoleCommand() + # Attempt to remove the enforcement — should be denied by KC-1435 fix + # because delegated admin managing non-root node cannot modify it + with self.assertLogs(level=logging.WARNING): + cmd.execute(params, enforcements=['require_account_share'], + role=[ent_env.role2_name]) + + # No command should be sent (denied by KC-1435 fix) + self.assertEqual(len(TestEnterprise.expected_commands), 0) + def test_enterprise_team(self): params = get_connected_params() api.query_enterprise(params) diff --git a/unit-tests/test_record_logging_sanitization.py b/unit-tests/test_record_logging_sanitization.py new file mode 100644 index 000000000..56dcc1864 --- /dev/null +++ b/unit-tests/test_record_logging_sanitization.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Integration tests verifying that sensitive record data is masked in actual logging paths. +Tests cover sanitize_command_fields and end-to-end logging for all affected record types. +""" + +import unittest +from keepercommander.service.decorators.logging import sanitize_command_fields, sanitize_debug_data + + +class TestCommandFieldSanitization(unittest.TestCase): + """Test sanitization of command fields in all supported formats.""" + + def test_bare_licensenumber_masked(self): + """Test bare licenseNumber field is masked.""" + command = "record-add -rt softwareLicense licenseNumber=LICENSE_SECRET --force" + result = sanitize_command_fields(command) + self.assertNotIn("LICENSE_SECRET", result) + self.assertIn("licenseNumber=***", result) + + def test_bare_encryptednote_masked(self): + """Test bare encryptedNote field is masked.""" + command = "record-add -rt encryptedNotes encryptedNote=NOTE_SECRET --force" + result = sanitize_command_fields(command) + self.assertNotIn("NOTE_SECRET", result) + self.assertIn("encryptedNote=***", result) + + def test_notes_field_masked(self): + """Test note field (in encryptedNotes) is masked.""" + command = "record-add -rt encryptedNotes note=NOTE_FIELD_SECRET --force" + result = sanitize_command_fields(command) + self.assertNotIn("NOTE_FIELD_SECRET", result) + self.assertIn("note=***", result) + + def test_notes_option_preserved(self): + """Test --notes option is preserved (not a sensitive field).""" + command = 'record-add --title MyRecord --notes "General notes here" --force' + result = sanitize_command_fields(command) + self.assertIn("--notes", result) + self.assertIn("General notes here", result) + + def test_prefixed_bankaccount_fields_masked(self): + """Test prefixed bankAccount fields are masked.""" + command = "record-add -rt bankAccount f.bankAccount.routingNumber=123456789 f.bankAccount.accountNumber=9876543210" + result = sanitize_command_fields(command) + self.assertNotIn("123456789", result) + self.assertNotIn("9876543210", result) + self.assertIn("f.bankAccount.routingNumber=***", result) + self.assertIn("f.bankAccount.accountNumber=***", result) + + def test_prefixed_paymentcard_fields_masked(self): + """Test prefixed paymentCard fields are masked.""" + command = "record-add -rt paymentCard f.paymentCard.cardNumber=4111111111111111 f.paymentCard.cardSecurityCode=123" + result = sanitize_command_fields(command) + self.assertNotIn("4111111111111111", result) + self.assertNotIn("123", result) + self.assertIn("f.paymentCard.cardNumber=***", result) + self.assertIn("f.paymentCard.cardSecurityCode=***", result) + + def test_prefixed_keypair_fields_masked(self): + """Test prefixed keyPair fields are masked.""" + command = "record-add -rt sshKeys f.keyPair.privateKey=SECRET_PRIVATE_KEY f.keyPair.publicKey=SECRET_PUBLIC_KEY" + result = sanitize_command_fields(command) + self.assertNotIn("SECRET_PRIVATE_KEY", result) + self.assertNotIn("SECRET_PUBLIC_KEY", result) + self.assertIn("f.keyPair.privateKey=***", result) + self.assertIn("f.keyPair.publicKey=***", result) + + def test_custom_field_licenseNumber_masked(self): + """Test custom field with licenseNumber type is masked.""" + command = "record-add -rt login c.licenseNumber.MyLicense=LICENSE_CUSTOM_SECRET" + result = sanitize_command_fields(command) + self.assertNotIn("LICENSE_CUSTOM_SECRET", result) + self.assertIn("c.licenseNumber.MyLicense=***", result) + + def test_custom_field_encryptedNote_masked(self): + """Test custom field with encryptedNote type is masked.""" + command = "record-add -rt login c.encryptedNote.MyNote=NOTE_CUSTOM_SECRET" + result = sanitize_command_fields(command) + self.assertNotIn("NOTE_CUSTOM_SECRET", result) + self.assertIn("c.encryptedNote.MyNote=***", result) + + def test_custom_field_password_label_masked(self): + """Test custom field password with label is masked.""" + command = "record-add f.password.DBPassword=SECRET_DB_PASS" + result = sanitize_command_fields(command) + self.assertNotIn("SECRET_DB_PASS", result) + self.assertIn("f.password.DBPassword=***", result) + + def test_custom_field_secret_label_masked(self): + """Test custom field secret with label is masked.""" + command = "record-add c.secret.APIKey=SECRET_API_KEY_VALUE" + result = sanitize_command_fields(command) + self.assertNotIn("SECRET_API_KEY_VALUE", result) + self.assertIn("c.secret.APIKey=***", result) + + def test_bare_bankaccount_field_masked(self): + """Test bare bankAccount field is masked.""" + command = "record-add bankAccount=SECRET_DATA" + result = sanitize_command_fields(command) + self.assertNotIn("SECRET_DATA", result) + self.assertIn("bankAccount=***", result) + + def test_bare_paymentcard_field_masked(self): + """Test bare paymentCard field is masked.""" + command = "record-add paymentCard=SECRET_DATA" + result = sanitize_command_fields(command) + self.assertNotIn("SECRET_DATA", result) + self.assertIn("paymentCard=***", result) + + def test_bare_keypair_field_masked(self): + """Test bare keyPair field is masked.""" + command = "record-add keyPair=SECRET_DATA" + result = sanitize_command_fields(command) + self.assertNotIn("SECRET_DATA", result) + self.assertIn("keyPair=***", result) + + def test_mixed_command_all_masked(self): + """Test mixed command with multiple sensitive fields all masked.""" + command = "record-add -rt softwareLicense --title MyLicense licenseNumber=LICENSE_NUM f.encryptedNote=NOTE_DATA" + result = sanitize_command_fields(command) + self.assertNotIn("LICENSE_NUM", result) + self.assertNotIn("NOTE_DATA", result) + self.assertIn("licenseNumber=***", result) + self.assertIn("f.encryptedNote=***", result) + + def test_mixed_case_field_names(self): + """Test that mixed-case field names are handled correctly.""" + commands = [ + "record-add EncryptedNote=CASE_SECRET", + "record-add LicenseNumber=CASE_SECRET", + "record-add BankAccount=CASE_SECRET", + "record-add f.Password.Label=CASE_SECRET", + "record-add c.Secret.Label=CASE_SECRET", + ] + for cmd in commands: + result = sanitize_command_fields(cmd) + self.assertNotIn("CASE_SECRET", result, f"Failed for: {cmd}") + self.assertIn("***", result, f"Not sanitized for: {cmd}") + + def test_non_sensitive_fields_preserved(self): + """Test that non-sensitive fields and options are preserved.""" + command = "record-add -rt bankAccount --title MyBank --folder MyFolder f.name=John" + result = sanitize_command_fields(command) + self.assertIn("--title", result) + self.assertIn("MyBank", result) + self.assertIn("--folder", result) + self.assertIn("MyFolder", result) + self.assertIn("f.name=John", result) + + +class TestDebugDataSanitization(unittest.TestCase): + """Test sanitize_debug_data for JSON and other formats.""" + + def test_sanitize_licensenumber_in_json(self): + """Test licenseNumber in JSON is masked.""" + json_str = '{"licenseNumber": "LICENSE_SECRET_123"}' + result = sanitize_debug_data(json_str) + self.assertNotIn("LICENSE_SECRET_123", result) + + def test_sanitize_encryptednote_in_json(self): + """Test encryptedNote in JSON is masked.""" + json_str = '{"encryptedNote": "NOTE_SECRET_456"}' + result = sanitize_debug_data(json_str) + self.assertNotIn("NOTE_SECRET_456", result) + + def test_sanitize_note_in_json(self): + """Test note field in JSON is masked.""" + json_str = '{"note": "SENSITIVE_NOTE_789"}' + result = sanitize_debug_data(json_str) + self.assertNotIn("SENSITIVE_NOTE_789", result) + + def test_bare_licensing_command_in_debug(self): + """Test bare licenseNumber command is masked in debug output.""" + debug_str = "Executing: record-add licenseNumber=LICENSE_DEBUG_SECRET" + result = sanitize_debug_data(debug_str) + self.assertNotIn("LICENSE_DEBUG_SECRET", result) + + def test_bare_encryptednote_command_in_debug(self): + """Test bare encryptedNote command is masked in debug output.""" + debug_str = "Executing: record-add encryptedNote=NOTE_DEBUG_SECRET" + result = sanitize_debug_data(debug_str) + self.assertNotIn("NOTE_DEBUG_SECRET", result) + + +class TestRecordTypes(unittest.TestCase): + """End-to-end tests for each sensitive record type.""" + + def test_bankaccount_all_formats(self): + """Test bankAccount in all command formats.""" + commands = [ + "record-add -rt bankAccount f.bankAccount.routingNumber=123456789 f.bankAccount.accountNumber=9876543210", + "record-add bankAccount=DATA_SECRET", + "record-update REC_UID bankAccount=DATA_SECRET", + ] + for cmd in commands: + result = sanitize_command_fields(cmd) + self.assertNotIn("123456789", result, f"Failed for: {cmd}") + self.assertNotIn("9876543210", result, f"Failed for: {cmd}") + self.assertNotIn("DATA_SECRET", result, f"Failed for: {cmd}") + + def test_paymentcard_all_formats(self): + """Test paymentCard in all command formats.""" + commands = [ + "record-add -rt paymentCard f.paymentCard.cardNumber=4111111111111111 f.paymentCard.cardSecurityCode=123", + "record-add paymentCard=DATA_SECRET", + "record-update REC_UID paymentCard=DATA_SECRET", + ] + for cmd in commands: + result = sanitize_command_fields(cmd) + self.assertNotIn("4111111111111111", result, f"Failed for: {cmd}") + self.assertNotIn("123", result, f"Failed for: {cmd}") + self.assertNotIn("DATA_SECRET", result, f"Failed for: {cmd}") + + def test_sshkeys_all_formats(self): + """Test sshKeys (keyPair) in all command formats.""" + commands = [ + "record-add -rt sshKeys f.keyPair.privateKey=PRIVATE_SECRET f.keyPair.publicKey=PUBLIC_SECRET", + "record-add keyPair=DATA_SECRET", + "record-update REC_UID keyPair=DATA_SECRET", + ] + for cmd in commands: + result = sanitize_command_fields(cmd) + self.assertNotIn("PRIVATE_SECRET", result, f"Failed for: {cmd}") + self.assertNotIn("PUBLIC_SECRET", result, f"Failed for: {cmd}") + self.assertNotIn("DATA_SECRET", result, f"Failed for: {cmd}") + + def test_softwarelicense_all_formats(self): + """Test softwareLicense in all command formats.""" + commands = [ + "record-add -rt softwareLicense licenseNumber=LICENSE_SECRET", + "record-add -rt softwareLicense f.licenseNumber=LICENSE_SECRET", + "record-add licenseNumber=LICENSE_SECRET", + "record-update REC_UID licenseNumber=LICENSE_SECRET", + ] + for cmd in commands: + result = sanitize_command_fields(cmd) + self.assertNotIn("LICENSE_SECRET", result, f"Failed for: {cmd}") + + def test_encryptednotes_all_formats(self): + """Test encryptedNotes in all command formats.""" + commands = [ + "record-add -rt encryptedNotes encryptedNote=NOTE_SECRET", + "record-add -rt encryptedNotes f.encryptedNote=NOTE_SECRET", + "record-add encryptedNote=NOTE_SECRET", + "record-update REC_UID encryptedNote=NOTE_SECRET", + "record-add -rt encryptedNotes note=NOTE_SECRET", + "record-update REC_UID note=NOTE_SECRET", + ] + for cmd in commands: + result = sanitize_command_fields(cmd) + self.assertNotIn("NOTE_SECRET", result, f"Failed for: {cmd}") + + +if __name__ == '__main__': + unittest.main()