From ec1c1bad128c9645b41e57bc5c64a00d0def5fe0 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Thu, 7 May 2026 16:11:20 +0000 Subject: [PATCH 01/16] feat(dev): load resource views; - Load resource views from ckanapi dump. --- ckanapi/cli/load.py | 115 +++++++++++++++++++++++++++++++++----------- 1 file changed, 88 insertions(+), 27 deletions(-) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 6c1a9b9..43ca3a9 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -110,6 +110,18 @@ def line_reader(): return 3 +def reply(action, error, response, out): + """ + format messages to be sent back to parent process + """ + out.write(compact_json([ + datetime.now().isoformat(), + action, + error, + response]) + b'\n') + out.flush() + + def load_things_worker(ckan, thing, arguments, stdin=None, stdout=None): """ @@ -144,23 +156,12 @@ def load_things_worker(ckan, thing, arguments, 'related_show','related_create','related_update'), }[thing] - def reply(action, error, response): - """ - format messages to be sent back to parent process - """ - stdout.write(compact_json([ - datetime.now().isoformat(), - action, - error, - response]) + b'\n') - stdout.flush() - for line in iter(stdin.readline, b''): try: obj = json.loads(line.decode('utf-8')) except UnicodeDecodeError as e: obj = None - reply('read', 'UnicodeDecodeError', str(e)) + reply('read', 'UnicodeDecodeError', str(e), stdout) continue requests_kwargs = None @@ -184,7 +185,7 @@ def reply(action, error, response): except NotFound: pass except NotAuthorized as e: - reply('show', 'NotAuthorized', str(e)) + reply('show', 'NotAuthorized', str(e), stdout) continue name = obj.get('name') if not existing and name: @@ -194,7 +195,7 @@ def reply(action, error, response): except NotFound: pass except NotAuthorized as e: - reply('show', 'NotAuthorized', str(e)) + reply('show', 'NotAuthorized', str(e), stdout) continue if existing: @@ -203,7 +204,7 @@ def reply(action, error, response): # FIXME: compare and reply when 'unchanged'? if not existing and arguments['--update-only']: - reply('show', 'NotFound', [obj.get('id'), obj.get('name')]) + reply('show', 'NotFound', [obj.get('id'), obj.get('name')], stdout) continue act = 'update' if existing else 'create' @@ -212,10 +213,14 @@ def reply(action, error, response): r = ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) else: - r = ckan.call_action(thing_create, obj) - if thing == 'datasets' and 'resources' in obj:# check if it is needed to upload resources when creating/updating packages - _upload_resources(ckan,obj,arguments) - elif thing in ['groups','organizations'] and 'image_display_url' in obj: #load images for groups and organizations + r = ckan.call_action(thing_create, obj, + requests_kwargs=requests_kwargs) + if thing == 'datasets' and 'resources' in obj: + if arguments['--upload-resources']: # check if it is needed to upload resources when creating/updating packages + _upload_resources(ckan, obj, arguments) + if arguments['--resource-views']: # check if it is needed to create resource views when creating/updating packages + _load_resource_views(ckan, obj, arguments, stdout) + elif thing in ['groups','organizations'] and 'image_display_url' in obj: # load images for groups and organizations if arguments['--upload-logo']: users = obj['users'] obj = _upload_logo(ckan,obj) @@ -224,15 +229,15 @@ def reply(action, error, response): ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) except ValidationError as e: - reply(act, 'ValidationError', e.error_dict) + reply(act, 'ValidationError', e.error_dict, stdout) except SearchIndexError as e: - reply(act, 'SearchIndexError', str(e)) + reply(act, 'SearchIndexError', str(e), stdout) except NotAuthorized as e: - reply(act, 'NotAuthorized', str(e)) + reply(act, 'NotAuthorized', str(e), stdout) except NotFound: - reply(act, 'NotFound', obj) + reply(act, 'NotFound', obj, stdout) else: - reply(act, None, r.get('name',r.get('id'))) + reply(act, None, r.get('name',r.get('id')), stdout) def _worker_command_line(thing, arguments): """ @@ -274,10 +279,9 @@ def _copy_from_existing_for_update(obj, existing, thing): if 'users' not in obj and 'users' in existing: obj['users'] = existing['users'] -def _upload_resources(ckan,obj,arguments): + +def _upload_resources(ckan, obj, arguments): resources = obj['resources'] - if not arguments['--upload-resources']: - return requests_kwargs = None if arguments['--insecure']: requests_kwargs = {'verify': False} @@ -293,6 +297,63 @@ def _upload_resources(ckan,obj,arguments): requests_kwargs=requests_kwargs) +def _load_resource_views(ckan, obj, arguments, stdout): + """ + Loads resource views + """ + resources = obj['resources'] + requests_kwargs = None + if arguments['--insecure']: + requests_kwargs = {'verify': False} + thing_show, thing_create, thing_update = ( + 'resource_view_show', 'resource_view_create', 'resource_view_update') + for resource in resources: + if not resource.get('resource_views'): + continue + resource_views = resource['resource_views'] + for view in resource_views: + existing = None + if not arguments['--create-only']: + # use either id or name to locate existing records + view_id = view.get('id') + if view_id: + try: + existing = ckan.call_action(thing_show, + {'id': view_id}, + requests_kwargs=requests_kwargs) + except NotFound: + pass + except NotAuthorized as e: + reply('show', 'NotAuthorized', str(e), stdout) + continue + + if existing: + _copy_from_existing_for_update(view, existing, 'resource_view') + + if not existing and arguments['--update-only']: + reply('show', 'NotFound', [view.get('id')], stdout) + continue + + act = 'update' if existing else 'create' + try: + if existing: + r = ckan.call_action(thing_update, view, + requests_kwargs=requests_kwargs) + else: + r = ckan.call_action(thing_create, view, + requests_kwargs=requests_kwargs) + except ValidationError as e: + reply(act, 'ValidationError', e.error_dict, stdout) + except SearchIndexError as e: + reply(act, 'SearchIndexError', str(e), stdout) + except NotAuthorized as e: + reply(act, 'NotAuthorized', str(e), stdout) + except NotFound: + reply(act, 'NotFound', view, stdout) + else: + reply(act, None, r.get('name', r.get('id')), stdout) + + def _upload_logo(ckan,obj_orig): obj = obj_orig.copy() for key in obj_orig.keys(): From d56a1ddf2c6f77dc7cbe7b51fd3fec7042e01716 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Thu, 7 May 2026 16:22:55 +0000 Subject: [PATCH 02/16] feat(dev): load ds fields; - Load resource datastore fields from ckanapi dump. --- ckanapi/cli/load.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 43ca3a9..9ac85b0 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -220,6 +220,8 @@ def load_things_worker(ckan, thing, arguments, _upload_resources(ckan, obj, arguments) if arguments['--resource-views']: # check if it is needed to create resource views when creating/updating packages _load_resource_views(ckan, obj, arguments, stdout) + if arguments['--datastore-fields']: # check if it is needed to update datastore resource fields when creating/updating packages + _load_datastore_resource_fields(ckan, obj, arguments, stdout) elif thing in ['groups','organizations'] and 'image_display_url' in obj: # load images for groups and organizations if arguments['--upload-logo']: users = obj['users'] @@ -354,6 +356,39 @@ def _load_resource_views(ckan, obj, arguments, stdout): reply(act, None, r.get('name', r.get('id')), stdout) +def _load_datastore_resource_fields(ckan, obj, arguments, stdout): + """ + Load datastore tables for Resources + """ + resources = obj['resources'] + requests_kwargs = None + if arguments['--insecure']: + requests_kwargs = {'verify': False} + thing_create = 'datastore_create' + act = 'create' + for resource in resources: + if not resource.get('datastore_fields'): + continue + args = { + 'resource_id': resource['id'], + 'fields': resource['datastore_fields'], + 'force': True + } + try: + r = ckan.call_action(thing_create, args, + requests_kwargs=requests_kwargs) + except ValidationError as e: + reply(act, 'ValidationError', e.error_dict, stdout) + except SearchIndexError as e: + reply(act, 'SearchIndexError', str(e), stdout) + except NotAuthorized as e: + reply(act, 'NotAuthorized', str(e), stdout) + except NotFound: + reply(act, 'NotFound', args, stdout) + else: + reply(act, None, r.get('name', r.get('id')), stdout) + + def _upload_logo(ckan,obj_orig): obj = obj_orig.copy() for key in obj_orig.keys(): From 4b813aac9c664e426166b6ddf77294f22db95ea2 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Thu, 7 May 2026 17:55:33 +0000 Subject: [PATCH 03/16] feat(dev): cli help; - Update the cli help text. --- ckanapi/cli/main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index 140708b..307845d 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -22,7 +22,7 @@ [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi load (groups | organizations) [--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX] - [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwzU] + [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-dqwzRU] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi load (users | related) [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] @@ -43,7 +43,8 @@ -c --config=CONFIG CKAN configuration file for local actions, defaults to $CKAN_INI or development.ini -d --datastore-fields export datastore field information along with - resource metadata as datastore_fields lists + resource metadata as datastore_fields lists (dump). + load datastore field information for resources (load). --include-private include private datasets in the dump --include-drafts include draft datasets in the dump --include-deleted include deleted datasets in the dump @@ -70,7 +71,8 @@ -q --quiet don't display progress messages -r --remote=URL URL of CKAN server for remote actions -R --resource-views export resource views information along with - resource metadata as resource_views lists + resource metadata as resource_views lists (dump). + create/update resource views for resources (load). -s --start-record=START start from record number START, where the first record is number 1 [default: 1] -u --ckan-user=USER perform actions as user with this name, uses the From a736a7004156cc7166436507018ebe4a0cc1d3fc Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Thu, 7 May 2026 18:03:03 +0000 Subject: [PATCH 04/16] feat(dev): cli help; - Update the cli help text. --- ckanapi/cli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index 307845d..5dcd13b 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -18,11 +18,11 @@ [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] ckanapi load datasets [--upload-resources] [-I JSONL_INPUT] [-s START] [-m MAX] - [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz] + [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-dqwzR] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi load (groups | organizations) [--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX] - [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-dqwzRU] + [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwzU] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi load (users | related) [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE] From 2b5b03d72c68160d417679ea8f5c75500929c0a2 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Thu, 7 May 2026 19:30:42 +0000 Subject: [PATCH 05/16] feat(dev): cli help; - Update the cli help text. --- ckanapi/cli/load.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 9ac85b0..1982fd3 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -262,6 +262,8 @@ def b(name): + b('--update-only') + b('--upload-resources') + b('--upload-logo') + + b('--datastore-fields') + + b('--resource-views') ) From eb6b493ff9053020600c1f47ba60f38ca9828426 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 11 May 2026 15:13:50 +0000 Subject: [PATCH 06/16] feat(dev): cli help; - Update the cli help text. --- ckanapi/cli/load.py | 202 +++++++++++++++++++++++--------------------- ckanapi/cli/main.py | 10 ++- 2 files changed, 114 insertions(+), 98 deletions(-) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 1982fd3..be935c0 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -110,18 +110,6 @@ def line_reader(): return 3 -def reply(action, error, response, out): - """ - format messages to be sent back to parent process - """ - out.write(compact_json([ - datetime.now().isoformat(), - action, - error, - response]) + b'\n') - out.flush() - - def load_things_worker(ckan, thing, arguments, stdin=None, stdout=None): """ @@ -156,12 +144,23 @@ def load_things_worker(ckan, thing, arguments, 'related_show','related_create','related_update'), }[thing] + def reply(action, error, response): + """ + format messages to be sent back to parent process + """ + stdout.write(compact_json([ + datetime.now().isoformat(), + action, + error, + response]) + b'\n') + stdout.flush() + for line in iter(stdin.readline, b''): try: obj = json.loads(line.decode('utf-8')) except UnicodeDecodeError as e: obj = None - reply('read', 'UnicodeDecodeError', str(e), stdout) + reply('read', 'UnicodeDecodeError', str(e)) continue requests_kwargs = None @@ -185,7 +184,7 @@ def load_things_worker(ckan, thing, arguments, except NotFound: pass except NotAuthorized as e: - reply('show', 'NotAuthorized', str(e), stdout) + reply('show', 'NotAuthorized', str(e)) continue name = obj.get('name') if not existing and name: @@ -195,7 +194,7 @@ def load_things_worker(ckan, thing, arguments, except NotFound: pass except NotAuthorized as e: - reply('show', 'NotAuthorized', str(e), stdout) + reply('show', 'NotAuthorized', str(e)) continue if existing: @@ -204,11 +203,20 @@ def load_things_worker(ckan, thing, arguments, # FIXME: compare and reply when 'unchanged'? if not existing and arguments['--update-only']: - reply('show', 'NotFound', [obj.get('id'), obj.get('name')], stdout) + reply('show', 'NotFound', [obj.get('id'), obj.get('name')]) continue act = 'update' if existing else 'create' try: + # do not send resource_views & datastore_fields to package actions + resource_views = [] + datastore_fields = {} + if thing == 'datasets' and obj.get('resources'): + for r in obj['resources']: + resource_views += r.pop('resource_views', []) + if r.get('datastore_fields'): + if r['id'] not in datastore_fields: + datastore_fields[r['id']] = r.pop('datastore_fields', []) if existing: r = ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) @@ -218,11 +226,11 @@ def load_things_worker(ckan, thing, arguments, if thing == 'datasets' and 'resources' in obj: if arguments['--upload-resources']: # check if it is needed to upload resources when creating/updating packages _upload_resources(ckan, obj, arguments) - if arguments['--resource-views']: # check if it is needed to create resource views when creating/updating packages - _load_resource_views(ckan, obj, arguments, stdout) - if arguments['--datastore-fields']: # check if it is needed to update datastore resource fields when creating/updating packages - _load_datastore_resource_fields(ckan, obj, arguments, stdout) - elif thing in ['groups','organizations'] and 'image_display_url' in obj: # load images for groups and organizations + if arguments['--resource-views'] and resource_views: # check if it is needed to create resource views when creating/updating packages + created_views, updated_views, skipped_views = _load_resource_views(ckan, resource_views, arguments) + if arguments['--datastore-fields'] and datastore_fields: # check if it is needed to update datastore resource fields when creating/updating packages + created_tables, skipped_tables = _load_datastore_resource_fields(ckan, datastore_fields, arguments) + if thing in ['groups','organizations'] and 'image_display_url' in obj: # load images for groups and organizations if arguments['--upload-logo']: users = obj['users'] obj = _upload_logo(ckan,obj) @@ -231,15 +239,28 @@ def load_things_worker(ckan, thing, arguments, ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) except ValidationError as e: - reply(act, 'ValidationError', e.error_dict, stdout) + reply(act, 'ValidationError', e.error_dict) except SearchIndexError as e: - reply(act, 'SearchIndexError', str(e), stdout) + reply(act, 'SearchIndexError', str(e)) except NotAuthorized as e: - reply(act, 'NotAuthorized', str(e), stdout) + reply(act, 'NotAuthorized', str(e)) except NotFound: - reply(act, 'NotFound', obj, stdout) + reply(act, 'NotFound', obj) else: - reply(act, None, r.get('name',r.get('id')), stdout) + log_obj = {} + if arguments['--resource-views'] and resource_views: + if created_views: + log_obj['created_resource_views'] = created_views + if updated_views: + log_obj['updated_resource_views'] = updated_views + if skipped_views: + log_obj['skipped_resource_views'] = skipped_views + if arguments['--datastore-fields'] and datastore_fields: + if created_tables: + log_obj['created_datastore_tables'] = created_tables + if skipped_tables: + log_obj['skipped_datastore_tables'] = skipped_tables + reply(act, None, log_obj if log_obj else r.get('name', r.get('id'))) def _worker_command_line(thing, arguments): """ @@ -301,94 +322,85 @@ def _upload_resources(ckan, obj, arguments): requests_kwargs=requests_kwargs) -def _load_resource_views(ckan, obj, arguments, stdout): +def _load_resource_views(ckan, resource_views, arguments): """ Loads resource views """ - resources = obj['resources'] + created = [] + updated = [] + skipped = [] requests_kwargs = None if arguments['--insecure']: requests_kwargs = {'verify': False} - thing_show, thing_create, thing_update = ( - 'resource_view_show', 'resource_view_create', 'resource_view_update') - for resource in resources: - if not resource.get('resource_views'): + for view in resource_views: + existing = None + if not arguments['--create-only']: + if view.get('id'): + try: + existing = ckan.call_action('resource_view_show', + {'id': view['id']}, + requests_kwargs=requests_kwargs) + except NotFound: + pass + + if existing: + _copy_from_existing_for_update(view, existing, 'resource_view') + + if not existing and arguments['--update-only']: + skipped.append(view.get('id', view.get('view_type'))) continue - resource_views = resource['resource_views'] - for view in resource_views: - existing = None - if not arguments['--create-only']: - # use either id or name to locate existing records - view_id = view.get('id') - if view_id: - try: - existing = ckan.call_action(thing_show, - {'id': view_id}, - requests_kwargs=requests_kwargs) - except NotFound: - pass - except NotAuthorized as e: - reply('show', 'NotAuthorized', str(e), stdout) - continue - if existing: - _copy_from_existing_for_update(view, existing, 'resource_view') + if existing: + # exceptions handled in load_things_worker + ckan.call_action('resource_view_update', view, + requests_kwargs=requests_kwargs) + updated.append(view.get('id', view.get('view_type'))) + else: + # exceptions handled in load_things_worker + ckan.call_action('resource_view_create', view, + requests_kwargs=requests_kwargs) + created.append(view.get('id', view.get('view_type'))) - if not existing and arguments['--update-only']: - reply('show', 'NotFound', [view.get('id')], stdout) - continue + return created, updated, skipped - act = 'update' if existing else 'create' - try: - if existing: - r = ckan.call_action(thing_update, view, - requests_kwargs=requests_kwargs) - else: - r = ckan.call_action(thing_create, view, - requests_kwargs=requests_kwargs) - except ValidationError as e: - reply(act, 'ValidationError', e.error_dict, stdout) - except SearchIndexError as e: - reply(act, 'SearchIndexError', str(e), stdout) - except NotAuthorized as e: - reply(act, 'NotAuthorized', str(e), stdout) - except NotFound: - reply(act, 'NotFound', view, stdout) - else: - reply(act, None, r.get('name', r.get('id')), stdout) - -def _load_datastore_resource_fields(ckan, obj, arguments, stdout): +def _load_datastore_resource_fields(ckan, datastore_fields, arguments): """ Load datastore tables for Resources """ - resources = obj['resources'] + created = [] + skipped = [] requests_kwargs = None if arguments['--insecure']: requests_kwargs = {'verify': False} - thing_create = 'datastore_create' - act = 'create' - for resource in resources: - if not resource.get('datastore_fields'): - continue - args = { - 'resource_id': resource['id'], - 'fields': resource['datastore_fields'], - 'force': True - } + for rid, ds_fields in datastore_fields.items(): + existing = None try: - r = ckan.call_action(thing_create, args, - requests_kwargs=requests_kwargs) - except ValidationError as e: - reply(act, 'ValidationError', e.error_dict, stdout) - except SearchIndexError as e: - reply(act, 'SearchIndexError', str(e), stdout) - except NotAuthorized as e: - reply(act, 'NotAuthorized', str(e), stdout) + existing = ckan.call_action('datastore_search', + {'resource_id': rid, 'limit': 0}, + requests_kwargs=requests_kwargs) except NotFound: - reply(act, 'NotFound', args, stdout) - else: - reply(act, None, r.get('name', r.get('id')), stdout) + pass + + if not existing: + # FIXME: only making new datastore tables. + # is it possible to safely update them via + # --datastore-fields dump when there is XLoader/DataPusher + skipped.append(rid) + continue + + # exceptions handled in load_things_worker + ckan.call_action( + 'datastore_create', + { + 'resource_id': rid, + 'fields': ds_fields, + 'force': True + }, + requests_kwargs=requests_kwargs) + created.append(rid) + + return created, skipped def _upload_logo(ckan,obj_orig): diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index 5dcd13b..da5f346 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -12,13 +12,17 @@ (ID_OR_NAME ... | [-I JSONL_INPUT] [-s START] [-m MAX]) [-p PROCESSES] [-l LOG_FILE] [-qwz] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] - ckanapi dump (datasets | groups | organizations | users | related) + ckanapi dump datasets (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY]) - [-p PROCESSES] [-dqwzRU --include-private --include-drafts --include-deleted] + [-p PROCESSES] [-qwz --include-private --include-drafts --include-deleted --datastore-fields --resource-views] + [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] + ckanapi dump (groups | organizations | users | related) + (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY]) + [-p PROCESSES] [-qwzU] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] ckanapi load datasets [--upload-resources] [-I JSONL_INPUT] [-s START] [-m MAX] - [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-dqwzR] + [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz --datastore-fields --resource-views] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi load (groups | organizations) [--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX] From e3f3d3523ad9f6ff830433455d905ab4c5bb4897 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 11 May 2026 15:19:12 +0000 Subject: [PATCH 07/16] feat(dev): cli help; - Update the cli help text. --- ckanapi/tests/test_cli_load.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index 7926940..d315c82 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -66,6 +66,7 @@ def test_create_with_no_resources(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -82,6 +83,7 @@ def test_create_with_corrupted_resources(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five","resources":[{"id":"123"}]}\n'), stdout=self.stdout) @@ -98,6 +100,7 @@ def test_create_with_complete_resources(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO( b'{"name": "45","title":"Forty-five",' @@ -116,6 +119,7 @@ def test_create_only(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -132,6 +136,7 @@ def test_create_empty_dict(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{}\n'), stdout=self.stdout) @@ -147,6 +152,7 @@ def test_create_bad_option(self): '--create-only': False, '--update-only': True, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -162,6 +168,7 @@ def test_update_with_no_resources(self): '--create-only': False, '--update-only': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -178,6 +185,7 @@ def test_update_with_corrupted_resources(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten","resources":[{"id":"123"}]}\n'), stdout=self.stdout) @@ -194,6 +202,7 @@ def test_update_with_complete_resources(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO( b'{"name": "30ish","title":"3.4 times ten",' @@ -212,6 +221,7 @@ def test_update_only(self): '--update-only': True, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -228,6 +238,7 @@ def test_update_bad_option(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -244,6 +255,7 @@ def test_update_unauthorized(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"name": "seekrit", "title": "Things"}\n'), stdout=self.stdout) @@ -260,6 +272,7 @@ def test_update_group(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"id": "ab","title":"a balloon"}\n'), stdout=self.stdout) @@ -276,6 +289,7 @@ def test_update_organization_two(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO( b'{"name": "cd", "title": "Go"}\n' @@ -300,6 +314,7 @@ def test_update_organization_with_users_unchanged(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"id": "used", "title": "here"}\n'), stdout=self.stdout) @@ -316,6 +331,7 @@ def test_update_organization_with_users_cleared(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, }, stdin=BytesIO(b'{"id": "unused", "users": []}\n'), stdout=self.stdout) @@ -345,6 +361,7 @@ def test_parent_load_two(self): '--upload-resources': False, '--upload-logo': False, '--insecure': False, + '--resource-views': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -380,6 +397,7 @@ def test_parent_load_start_max(self): '--upload-resources': False, '--upload-logo': False, '--insecure': False, + '--resource-views': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -418,6 +436,7 @@ def test_parent_parallel_limit(self): '--upload-resources': False, '--upload-logo': False, '--insecure': False, + '--resource-views': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( From ce44207c8922e42c2360181e00ad39b4e9ae1006 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 11 May 2026 15:22:12 +0000 Subject: [PATCH 08/16] feat(test): coverage; - New arg. --- ckanapi/tests/test_cli_load.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index d315c82..9678161 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -67,6 +67,7 @@ def test_create_with_no_resources(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -84,6 +85,7 @@ def test_create_with_corrupted_resources(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five","resources":[{"id":"123"}]}\n'), stdout=self.stdout) @@ -101,6 +103,7 @@ def test_create_with_complete_resources(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO( b'{"name": "45","title":"Forty-five",' @@ -120,6 +123,7 @@ def test_create_only(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -137,6 +141,7 @@ def test_create_empty_dict(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{}\n'), stdout=self.stdout) @@ -153,6 +158,7 @@ def test_create_bad_option(self): '--update-only': True, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -169,6 +175,7 @@ def test_update_with_no_resources(self): '--update-only': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -186,6 +193,7 @@ def test_update_with_corrupted_resources(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten","resources":[{"id":"123"}]}\n'), stdout=self.stdout) @@ -203,6 +211,7 @@ def test_update_with_complete_resources(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO( b'{"name": "30ish","title":"3.4 times ten",' @@ -222,6 +231,7 @@ def test_update_only(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -239,6 +249,7 @@ def test_update_bad_option(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -256,6 +267,7 @@ def test_update_unauthorized(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "seekrit", "title": "Things"}\n'), stdout=self.stdout) @@ -273,6 +285,7 @@ def test_update_group(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"id": "ab","title":"a balloon"}\n'), stdout=self.stdout) @@ -290,6 +303,7 @@ def test_update_organization_two(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO( b'{"name": "cd", "title": "Go"}\n' @@ -315,6 +329,7 @@ def test_update_organization_with_users_unchanged(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"id": "used", "title": "here"}\n'), stdout=self.stdout) @@ -332,6 +347,7 @@ def test_update_organization_with_users_cleared(self): '--upload-resources': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"id": "unused", "users": []}\n'), stdout=self.stdout) @@ -362,6 +378,7 @@ def test_parent_load_two(self): '--upload-logo': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -398,6 +415,7 @@ def test_parent_load_start_max(self): '--upload-logo': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -437,6 +455,7 @@ def test_parent_parallel_limit(self): '--upload-logo': False, '--insecure': False, '--resource-views': False, + '--datastore-fields': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( From dd74f2847d640dc58c3d8488a0664a00519c021b Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 11 May 2026 15:25:54 +0000 Subject: [PATCH 09/16] feat(test): coverage; - New arg. --- ckanapi/cli/load.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index be935c0..11d4270 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -214,9 +214,8 @@ def reply(action, error, response): if thing == 'datasets' and obj.get('resources'): for r in obj['resources']: resource_views += r.pop('resource_views', []) - if r.get('datastore_fields'): - if r['id'] not in datastore_fields: - datastore_fields[r['id']] = r.pop('datastore_fields', []) + # FIXME: assuming passed resources have an id + datastore_fields[r['id']] = r.pop('datastore_fields', []) if existing: r = ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) @@ -374,6 +373,8 @@ def _load_datastore_resource_fields(ckan, datastore_fields, arguments): if arguments['--insecure']: requests_kwargs = {'verify': False} for rid, ds_fields in datastore_fields.items(): + if not ds_fields: + continue existing = None try: existing = ckan.call_action('datastore_search', From 0a4bbf32743497ff6ab49e3302aaa49fe3c973c3 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 11 May 2026 15:33:22 +0000 Subject: [PATCH 10/16] feat(test): coverage; - New arg. --- ckanapi/cli/load.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 11d4270..b2691fb 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -223,12 +223,14 @@ def reply(action, error, response): r = ckan.call_action(thing_create, obj, requests_kwargs=requests_kwargs) if thing == 'datasets' and 'resources' in obj: + # NOTE: order is important as Resource uploads may be dependant on DS Fields (XLoader/DataPusher), + # and Resource Views may be dependant on DS and Upload. + if arguments['--datastore-fields'] and datastore_fields: # check if it is needed to update datastore resource fields when creating/updating packages + created_tables, skipped_tables = _load_datastore_resource_fields(ckan, datastore_fields, arguments) if arguments['--upload-resources']: # check if it is needed to upload resources when creating/updating packages _upload_resources(ckan, obj, arguments) if arguments['--resource-views'] and resource_views: # check if it is needed to create resource views when creating/updating packages created_views, updated_views, skipped_views = _load_resource_views(ckan, resource_views, arguments) - if arguments['--datastore-fields'] and datastore_fields: # check if it is needed to update datastore resource fields when creating/updating packages - created_tables, skipped_tables = _load_datastore_resource_fields(ckan, datastore_fields, arguments) if thing in ['groups','organizations'] and 'image_display_url' in obj: # load images for groups and organizations if arguments['--upload-logo']: users = obj['users'] @@ -383,23 +385,22 @@ def _load_datastore_resource_fields(ckan, datastore_fields, arguments): except NotFound: pass - if not existing: - # FIXME: only making new datastore tables. - # is it possible to safely update them via - # --datastore-fields dump when there is XLoader/DataPusher - skipped.append(rid) - continue - - # exceptions handled in load_things_worker - ckan.call_action( - 'datastore_create', - { - 'resource_id': rid, - 'fields': ds_fields, - 'force': True - }, - requests_kwargs=requests_kwargs) - created.append(rid) + try: + ckan.call_action( + 'datastore_create', + { + 'resource_id': rid, + 'fields': ds_fields, + 'force': True + }, + requests_kwargs=requests_kwargs) + created.append(rid) + except ValidationError as e: + if not existing: + # exceptions handled in load_things_worker + # raise normal exception for non-existing tables + raise e + skipped.append('%s: %s' % (rid, str(e))) return created, skipped From de6bf3a1fb16907b8c010ec62979a11c54fc69e0 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Thu, 21 May 2026 15:09:19 -0400 Subject: [PATCH 11/16] feat(tests): load resource views; - Test coverage for resource view loading. --- ckanapi/cli/load.py | 9 ++- ckanapi/cli/main.py | 2 + ckanapi/tests/test_cli_dump.py | 1 - ckanapi/tests/test_cli_load.py | 132 +++++++++++++++++++++++++++++---- 4 files changed, 124 insertions(+), 20 deletions(-) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index b2691fb..2ed3009 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -214,7 +214,8 @@ def reply(action, error, response): if thing == 'datasets' and obj.get('resources'): for r in obj['resources']: resource_views += r.pop('resource_views', []) - # FIXME: assuming passed resources have an id + # NOTE: will only work with existing Resource IDs in the input, + # documented in the command help. datastore_fields[r['id']] = r.pop('datastore_fields', []) if existing: r = ckan.call_action(thing_update, obj, @@ -248,7 +249,7 @@ def reply(action, error, response): except NotFound: reply(act, 'NotFound', obj) else: - log_obj = {} + log_obj = {'id': r.get('id'), 'name': r.get('name')} if arguments['--resource-views'] and resource_views: if created_views: log_obj['created_resource_views'] = created_views @@ -261,7 +262,7 @@ def reply(action, error, response): log_obj['created_datastore_tables'] = created_tables if skipped_tables: log_obj['skipped_datastore_tables'] = skipped_tables - reply(act, None, log_obj if log_obj else r.get('name', r.get('id'))) + reply(act, None, log_obj) def _worker_command_line(thing, arguments): """ @@ -400,7 +401,7 @@ def _load_datastore_resource_fields(ckan, datastore_fields, arguments): # exceptions handled in load_things_worker # raise normal exception for non-existing tables raise e - skipped.append('%s: %s' % (rid, str(e))) + skipped.append('%s: %s' % (rid, e)) return created, skipped diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index da5f346..1d7a442 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -49,6 +49,7 @@ -d --datastore-fields export datastore field information along with resource metadata as datastore_fields lists (dump). load datastore field information for resources (load). + Requires existing Resource IDs in the JSONL. --include-private include private datasets in the dump --include-drafts include draft datasets in the dump --include-deleted include deleted datasets in the dump @@ -77,6 +78,7 @@ -R --resource-views export resource views information along with resource metadata as resource_views lists (dump). create/update resource views for resources (load). + Requires existing Resource IDs in the JSONL. -s --start-record=START start from record number START, where the first record is number 1 [default: 1] -u --ckan-user=USER perform actions as user with this name, uses the diff --git a/ckanapi/tests/test_cli_dump.py b/ckanapi/tests/test_cli_dump.py index 95534fa..7276442 100644 --- a/ckanapi/tests/test_cli_dump.py +++ b/ckanapi/tests/test_cli_dump.py @@ -453,7 +453,6 @@ def _worker_pool_with_data(self, cmd, processes, job_iter): for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')): yield [[], i, v] - def _worker_pool_with_resource_views(self, cmd, proccesses, job_iter): worker_stdin = BytesIO(b''.join(v for i, v in job_iter)) worker_stdout = BytesIO() diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index 9678161..71af28b 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -17,12 +17,15 @@ def call_action(self, name, data_dict, requests_kwargs=None): raise ValidationError({'users': 'should be unchanged'}) if data_dict['id'] == 'unused' and data_dict.get('users') != []: raise ValidationError({'users': 'should be cleared'}) + if name == 'resource_view_show' and data_dict['id'] == '123': + raise NotFound('no resource view with ID 123') try: return { 'package_show': { '12': {'title': "Twelve"}, '30ish': {'id': '34', 'title': "Thirty-four"}, '34': {'id': '34', 'title': "Thirty-four"}, + '46': {'id': '46', 'name': '46', 'title': 'Forty-six'}, }, 'group_show': { 'ab': {'title': "ABBA"}, @@ -34,10 +37,21 @@ def call_action(self, name, data_dict, requests_kwargs=None): }, 'package_create': { None: {'name': 'something-new'}, + '46': {'id': '46', 'name': '46'}, }, 'package_update': { '34': {'name': 'something-updated'}, + '46': {'id': '46', 'name': '46'}, }, + 'resource_view_show': { + '456': {'description': 'Test view', 'package_id': '46', 'resource_id': '456'} + }, + 'resource_view_create': { + '123': {'description': 'Test view', 'package_id': '46', 'resource_id': '123'}, + }, + 'resource_view_update': { + '456': {'description': 'Test view', 'package_id': '46', 'resource_id': '456'}, + }, 'group_update': { 'ab': {'name': 'group-updated'}, }, @@ -50,7 +64,13 @@ def call_action(self, name, data_dict, requests_kwargs=None): None: {'name': 'org-created'}, }, }[name][data_dict.get('id')] - except KeyError: + except KeyError as e: + print(' ') + print('DEBUGGING:2') + print(' ') + print(name) + print(e) + print(' ') raise NotFound() @@ -76,7 +96,7 @@ def test_create_with_no_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': None, 'name': 'something-new'}) def test_create_with_corrupted_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -94,7 +114,7 @@ def test_create_with_corrupted_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': None, 'name': 'something-new'}) def test_create_with_complete_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -114,7 +134,48 @@ def test_create_with_complete_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': None, 'name': 'something-new'}) + + def test_create_with_resource_views(self): + """ + Creating a dataset with Resources that have views should create + the resource views. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '123', + 'url': 'http://example.com', + 'datastore_active': True, + 'resource_views': [{ + 'description': 'Test view', + 'filterable': True, + 'id': '123', + 'resource_id': '123', + 'responsive': True, + 'show_fields': ['_id'] + }] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': False, + '--upload-resources': False, + '--insecure': False, + '--resource-views': True, + '--datastore-fields': False, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'update') + self.assertEqual(error, None) + self.assertEqual(data, {'id': '46', 'name': '46', 'created_resource_views': ['123']}) def test_create_only(self): load_things_worker(self.ckan, 'datasets', { @@ -132,7 +193,7 @@ def test_create_only(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': None, 'name': 'something-new'}) def test_create_empty_dict(self): load_things_worker(self.ckan, 'datasets', { @@ -150,7 +211,7 @@ def test_create_empty_dict(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'something-new') + self.assertEqual(data, {'id': None, 'name': 'something-new'}) def test_create_bad_option(self): load_things_worker(self.ckan, 'datasets', { @@ -184,7 +245,7 @@ def test_update_with_no_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': None, 'name': 'something-updated'}) def test_update_with_corrupted_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -202,7 +263,7 @@ def test_update_with_corrupted_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': None, 'name': 'something-updated'}) def test_update_with_complete_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -222,7 +283,48 @@ def test_update_with_complete_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': None, 'name': 'something-updated'}) + + def test_update_with_resource_views(self): + """ + Updating a dataset with Resources that have views should update + the resource views. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '123', + 'url': 'http://example.com', + 'datastore_active': True, + 'resource_views': [{ + 'description': 'Test view', + 'filterable': True, + 'id': '456', + 'resource_id': '456', + 'responsive': True, + 'show_fields': ['_id'] + }] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': False, + '--upload-resources': False, + '--insecure': False, + '--resource-views': True, + '--datastore-fields': False, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'update') + self.assertEqual(error, None) + self.assertEqual(data, {'id': '46', 'name': '46', 'updated_resource_views': ['456']}) def test_update_only(self): load_things_worker(self.ckan, 'datasets', { @@ -240,7 +342,7 @@ def test_update_only(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'something-updated') + self.assertEqual(data, {'id': None, 'name': 'something-updated'}) def test_update_bad_option(self): load_things_worker(self.ckan, 'datasets', { @@ -294,7 +396,7 @@ def test_update_group(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'group-updated') + self.assertEqual(data, {'id': None, 'name': 'group-updated'}) def test_update_organization_two(self): load_things_worker(self.ckan, 'organizations', { @@ -316,11 +418,11 @@ def test_update_organization_two(self): timstamp, action, error, data = json.loads(r1.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'org-updated') + self.assertEqual(data, {'id': None, 'name': 'org-updated'}) timstamp, action, error, data = json.loads(r2.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, 'org-created') + self.assertEqual(data, {'id': None, 'name': 'org-created'}) def test_update_organization_with_users_unchanged(self): load_things_worker(self.ckan, 'organizations', { @@ -338,7 +440,7 @@ def test_update_organization_with_users_unchanged(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'users-unchanged') + self.assertEqual(data, {'id': None, 'name': 'users-unchanged'}) def test_update_organization_with_users_cleared(self): load_things_worker(self.ckan, 'organizations', { @@ -356,7 +458,7 @@ def test_update_organization_with_users_cleared(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, 'users-cleared') + self.assertEqual(data, {'id': None, 'name': 'users-cleared'}) def test_parent_load_two(self): load_things(self.ckan, 'datasets', { From 8f72ade73b73c7eca638cff1b063a5fbf5a06e04 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Thu, 21 May 2026 16:04:10 -0400 Subject: [PATCH 12/16] feat(tests): load datastore fields; - Test coverage for datastore field loading. --- ckanapi/cli/load.py | 59 +++--- ckanapi/tests/test_cli_load.py | 333 +++++++++++++++++++++++++++++---- 2 files changed, 329 insertions(+), 63 deletions(-) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 2ed3009..4e3f27c 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -209,14 +209,16 @@ def reply(action, error, response): act = 'update' if existing else 'create' try: # do not send resource_views & datastore_fields to package actions - resource_views = [] + resource_views = {} datastore_fields = {} if thing == 'datasets' and obj.get('resources'): for r in obj['resources']: - resource_views += r.pop('resource_views', []) # NOTE: will only work with existing Resource IDs in the input, # documented in the command help. - datastore_fields[r['id']] = r.pop('datastore_fields', []) + if arguments['--resource-views']: + resource_views[r['id']] = r.pop('resource_views', []) + if arguments['--datastore-fields']: + datastore_fields[r['id']] = r.pop('datastore_fields', []) if existing: r = ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) @@ -334,34 +336,35 @@ def _load_resource_views(ckan, resource_views, arguments): requests_kwargs = None if arguments['--insecure']: requests_kwargs = {'verify': False} - for view in resource_views: - existing = None - if not arguments['--create-only']: - if view.get('id'): - try: - existing = ckan.call_action('resource_view_show', - {'id': view['id']}, - requests_kwargs=requests_kwargs) - except NotFound: - pass + for _rid, views in resource_views.items(): + for view in views: + existing = None + if not arguments['--create-only']: + if view.get('id'): + try: + existing = ckan.call_action('resource_view_show', + {'id': view['id']}, + requests_kwargs=requests_kwargs) + except NotFound: + pass - if existing: - _copy_from_existing_for_update(view, existing, 'resource_view') + if existing: + _copy_from_existing_for_update(view, existing, 'resource_view') - if not existing and arguments['--update-only']: - skipped.append(view.get('id', view.get('view_type'))) - continue + if not existing and arguments['--update-only']: + skipped.append(view.get('id', view.get('view_type'))) + continue - if existing: - # exceptions handled in load_things_worker - ckan.call_action('resource_view_update', view, - requests_kwargs=requests_kwargs) - updated.append(view.get('id', view.get('view_type'))) - else: - # exceptions handled in load_things_worker - ckan.call_action('resource_view_create', view, - requests_kwargs=requests_kwargs) - created.append(view.get('id', view.get('view_type'))) + if existing: + # exceptions handled in load_things_worker + ckan.call_action('resource_view_update', view, + requests_kwargs=requests_kwargs) + updated.append(view.get('id', view.get('view_type'))) + else: + # exceptions handled in load_things_worker + ckan.call_action('resource_view_create', view, + requests_kwargs=requests_kwargs) + created.append(view.get('id', view.get('view_type'))) return created, updated, skipped diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index 71af28b..b8c1c6a 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -19,6 +19,10 @@ def call_action(self, name, data_dict, requests_kwargs=None): raise ValidationError({'users': 'should be cleared'}) if name == 'resource_view_show' and data_dict['id'] == '123': raise NotFound('no resource view with ID 123') + if name == 'datastore_search' and (data_dict['resource_id'] == '123' or data_dict['resource_id'] == '111'): + raise NotFound('no resource datastore with ID 123') + if name == 'datastore_create' and (data_dict['resource_id'] == '789' or data_dict['resource_id'] == '111'): + raise ValidationError({'pg_error': 'no db connection'}) try: return { 'package_show': { @@ -36,11 +40,11 @@ def call_action(self, name, data_dict, requests_kwargs=None): 'unused': {'users': ['people']}, }, 'package_create': { - None: {'name': 'something-new'}, + None: {'id': 'some-generated-uuid', 'name': 'something-new'}, '46': {'id': '46', 'name': '46'}, }, 'package_update': { - '34': {'name': 'something-updated'}, + '34': {'id': '34', 'name': 'something-updated'}, '46': {'id': '46', 'name': '46'}, }, 'resource_view_show': { @@ -52,25 +56,27 @@ def call_action(self, name, data_dict, requests_kwargs=None): 'resource_view_update': { '456': {'description': 'Test view', 'package_id': '46', 'resource_id': '456'}, }, + 'datastore_search': { + '456': {'resource_id': '456', 'fields': [{'id': 'test_field1', 'type': 'text'}, {'id': 'test_field2', 'type': 'text'}]}, + '789': {'resource_id': '789', 'fields': [{'id': 'test_field1', 'type': 'text'}, {'id': 'test_field2', 'type': 'text'}]}, + }, + 'datastore_create': { + '123': {'resource_id': '123', 'fields': [{'id': 'test_field1', 'type': 'text'}, {'id': 'test_field2', 'type': 'text'}]}, + '456': {'resource_id': '456', 'fields': [{'id': 'test_field1', 'type': 'text'}, {'id': 'test_field2', 'type': 'text'}]}, + }, 'group_update': { - 'ab': {'name': 'group-updated'}, + 'ab': {'id': 'ab', 'name': 'group-updated'}, }, 'organization_update': { - 'cd': {'name': 'org-updated'}, - 'used': {'name': 'users-unchanged'}, - 'unused': {'name': 'users-cleared'}, + 'cd': {'id': 'cd', 'name': 'org-updated'}, + 'used': {'id': 'used', 'name': 'users-unchanged'}, + 'unused': {'id': 'unused', 'name': 'users-cleared'}, }, 'organization_create': { - None: {'name': 'org-created'}, + None: {'id': 'some-generated-uuid', 'name': 'org-created'}, }, - }[name][data_dict.get('id')] + }[name][data_dict.get('id', data_dict.get('resource_id'))] except KeyError as e: - print(' ') - print('DEBUGGING:2') - print(' ') - print(name) - print(e) - print(' ') raise NotFound() @@ -96,7 +102,7 @@ def test_create_with_no_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'something-new'}) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_with_corrupted_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -114,7 +120,7 @@ def test_create_with_corrupted_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'something-new'}) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_with_complete_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -134,11 +140,11 @@ def test_create_with_complete_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'something-new'}) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_with_resource_views(self): """ - Creating a dataset with Resources that have views should create + A dataset with Resources that have views should create the resource views. """ payload = { @@ -161,7 +167,7 @@ def test_create_with_resource_views(self): }] } load_things_worker(self.ckan, 'datasets', { - '--create-only': False, + '--create-only': True, '--update-only': False, '--upload-resources': False, '--insecure': False, @@ -173,9 +179,83 @@ def test_create_with_resource_views(self): response = self.stdout.getvalue() self.assertEqual(response[-1:], b'\n') timstamp, action, error, data = json.loads(response.decode('UTF-8')) - self.assertEqual(action, 'update') + self.assertEqual(action, 'create') + self.assertEqual(error, None) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new', 'created_resource_views': ['123']}) + + def test_create_with_resource_datastore_fields(self): + """ + A dataset with Resources that have datastore fields should create + the datastore table with the fields. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '123', + 'url': 'http://example.com', + 'datastore_active': True, + 'datastore_fields': [ + {'id': 'test_field1', 'type': 'text'}, + {'id': 'test_field2', 'type': 'text'}, + ] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': True, + '--update-only': False, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': True, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, {'id': '46', 'name': '46', 'created_resource_views': ['123']}) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new', 'created_datastore_tables': ['123']}) + + def test_create_with_bad_resource_datastore_fields(self): + """ + A dataset with Resources that have datastore fields that exist + but throw ValidationErrors should skip them. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '789', + 'url': 'http://example.com', + 'datastore_active': True, + 'datastore_fields': [ + {'id': 'test_field1', 'type': 'text'}, + {'id': 'test_field2', 'type': 'text'}, + ] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': True, + '--update-only': False, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': True, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'create') + self.assertEqual(error, None) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new', 'skipped_datastore_tables': ["789: None - {'pg_error': 'no db connection'}"]}) def test_create_only(self): load_things_worker(self.ckan, 'datasets', { @@ -193,7 +273,7 @@ def test_create_only(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'something-new'}) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_empty_dict(self): load_things_worker(self.ckan, 'datasets', { @@ -211,7 +291,7 @@ def test_create_empty_dict(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'something-new'}) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) def test_create_bad_option(self): load_things_worker(self.ckan, 'datasets', { @@ -245,7 +325,7 @@ def test_update_with_no_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'something-updated'}) + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_with_corrupted_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -263,7 +343,7 @@ def test_update_with_corrupted_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'something-updated'}) + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_with_complete_resources(self): load_things_worker(self.ckan, 'datasets', { @@ -283,11 +363,11 @@ def test_update_with_complete_resources(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'something-updated'}) + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_with_resource_views(self): """ - Updating a dataset with Resources that have views should update + A dataset with Resources that have views should update the resource views. """ payload = { @@ -296,7 +376,7 @@ def test_update_with_resource_views(self): 'resources': [{ 'name': 'resource1', 'format': 'csv', - 'id': '123', + 'id': '456', 'url': 'http://example.com', 'datastore_active': True, 'resource_views': [{ @@ -311,7 +391,7 @@ def test_update_with_resource_views(self): } load_things_worker(self.ckan, 'datasets', { '--create-only': False, - '--update-only': False, + '--update-only': True, '--upload-resources': False, '--insecure': False, '--resource-views': True, @@ -326,6 +406,189 @@ def test_update_with_resource_views(self): self.assertEqual(error, None) self.assertEqual(data, {'id': '46', 'name': '46', 'updated_resource_views': ['456']}) + def test_update_with_new_resource_views(self): + """ + A dataset with Resources that have NEW views should not + be able to create the resource views, just skip them. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '456', + 'url': 'http://example.com', + 'datastore_active': True, + 'resource_views': [{ + 'description': 'Test view', + 'filterable': True, + 'id': '123', + 'resource_id': '456', + 'responsive': True, + 'show_fields': ['_id'] + }] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': True, + '--datastore-fields': False, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'update') + self.assertEqual(error, None) + self.assertEqual(data, {'id': '46', 'name': '46', 'skipped_resource_views': ['123']}) + + def test_update_with_resource_datastore_fields(self): + """ + A dataset with Resources that have datastore fields should create + the datastore table with the fields. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '456', + 'url': 'http://example.com', + 'datastore_active': True, + 'datastore_fields': [ + {'id': 'test_field1', 'type': 'text'}, + {'id': 'test_field2', 'type': 'text'}, + ] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': True, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'update') + self.assertEqual(error, None) + self.assertEqual(data, {'id': '46', 'name': '46', 'created_datastore_tables': ['456']}) + + def test_update_with_bad_resource_datastore_fields(self): + """ + A dataset with Resources that have datastore fields that do not exist + should throw ValidationErrors should raise the error as normal. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '111', + 'url': 'http://example.com', + 'datastore_active': True, + 'datastore_fields': [ + {'id': 'test_field1', 'type': 'text'}, + {'id': 'test_field2', 'type': 'text'}, + ] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': True, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'update') + self.assertEqual(error, 'ValidationError') + self.assertEqual(data, {'pg_error': 'no db connection'}) + + def test_update_with_resource_datastore_fields_no_resource_id(self): + """ + A dataset with Resources that have datastore fields but no ID + should raise a KeyError. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'url': 'http://example.com', + 'datastore_active': True, + 'datastore_fields': [ + {'id': 'test_field1', 'type': 'text'}, + {'id': 'test_field2', 'type': 'text'}, + ] + }] + } + with self.assertRaises(KeyError) as ke: + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': True, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + self.assertEqual(str(ke.exception), "'id'") + + def test_update_with_resource_views_no_resource_id(self): + """ + A dataset with Resources that have views but no ID + should raise a KeyError. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'url': 'http://example.com', + 'datastore_active': True, + 'resource_views': [{ + 'description': 'Test view', + 'filterable': True, + 'id': '123', + 'resource_id': '456', + 'responsive': True, + 'show_fields': ['_id'] + }] + }] + } + with self.assertRaises(KeyError) as ke: + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': True, + '--datastore-fields': False, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + self.assertEqual(str(ke.exception), "'id'") + def test_update_only(self): load_things_worker(self.ckan, 'datasets', { '--create-only': False, @@ -342,7 +605,7 @@ def test_update_only(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'something-updated'}) + self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) def test_update_bad_option(self): load_things_worker(self.ckan, 'datasets', { @@ -396,7 +659,7 @@ def test_update_group(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'group-updated'}) + self.assertEqual(data, {'id': 'ab', 'name': 'group-updated'}) def test_update_organization_two(self): load_things_worker(self.ckan, 'organizations', { @@ -418,11 +681,11 @@ def test_update_organization_two(self): timstamp, action, error, data = json.loads(r1.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'org-updated'}) + self.assertEqual(data, {'id': 'cd', 'name': 'org-updated'}) timstamp, action, error, data = json.loads(r2.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'org-created'}) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'org-created'}) def test_update_organization_with_users_unchanged(self): load_things_worker(self.ckan, 'organizations', { @@ -440,7 +703,7 @@ def test_update_organization_with_users_unchanged(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'users-unchanged'}) + self.assertEqual(data, {'id': 'used', 'name': 'users-unchanged'}) def test_update_organization_with_users_cleared(self): load_things_worker(self.ckan, 'organizations', { @@ -458,7 +721,7 @@ def test_update_organization_with_users_cleared(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'update') self.assertEqual(error, None) - self.assertEqual(data, {'id': None, 'name': 'users-cleared'}) + self.assertEqual(data, {'id': 'unused', 'name': 'users-cleared'}) def test_parent_load_two(self): load_things(self.ckan, 'datasets', { From 24e20658f4069f3bff40c91f7b378afad855d19b Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Thu, 21 May 2026 16:06:49 -0400 Subject: [PATCH 13/16] feat(tests): load datastore fields; - Test coverage for datastore field loading. --- ckanapi/tests/test_cli_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index b8c1c6a..8b6920e 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -255,7 +255,7 @@ def test_create_with_bad_resource_datastore_fields(self): timstamp, action, error, data = json.loads(response.decode('UTF-8')) self.assertEqual(action, 'create') self.assertEqual(error, None) - self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new', 'skipped_datastore_tables': ["789: None - {'pg_error': 'no db connection'}"]}) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new', 'skipped_datastore_tables': ["789: {'pg_error': 'no db connection'}"]}) def test_create_only(self): load_things_worker(self.ckan, 'datasets', { From 955a39f024c93d3d4419de9cf996d3ac2ab14869 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Fri, 22 May 2026 13:53:13 -0400 Subject: [PATCH 14/16] feat(tests): load datastore fields; - Test coverage for datastore field loading. --- ckanapi/cli/dump.py | 3 ++- ckanapi/cli/load.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ckanapi/cli/dump.py b/ckanapi/cli/dump.py index 9c6870a..c4f3743 100644 --- a/ckanapi/cli/dump.py +++ b/ckanapi/cli/dump.py @@ -246,5 +246,6 @@ def populate_res_views(ckan, res): return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass if not views: return # return if the resource views list is empty - res['resource_views'] = views + rem = ['resource_id', 'package_id'] # remove unneeded key/values + res['resource_views'] = [{k: val for k, val in v.items() if k not in rem} for v in views] diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index 4e3f27c..a2c271c 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -336,9 +336,10 @@ def _load_resource_views(ckan, resource_views, arguments): requests_kwargs = None if arguments['--insecure']: requests_kwargs = {'verify': False} - for _rid, views in resource_views.items(): + for rid, views in resource_views.items(): for view in views: existing = None + view['resource_id'] = rid if not arguments['--create-only']: if view.get('id'): try: From ed9e10361b22d7a9d62cac69a672cdcd1cce4b47 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Fri, 22 May 2026 13:54:47 -0400 Subject: [PATCH 15/16] feat(tests): load datastore fields; - Test coverage for datastore field loading. --- ckanapi/tests/test_cli_dump.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ckanapi/tests/test_cli_dump.py b/ckanapi/tests/test_cli_dump.py index 7276442..b90779b 100644 --- a/ckanapi/tests/test_cli_dump.py +++ b/ckanapi/tests/test_cli_dump.py @@ -347,8 +347,6 @@ def test_resource_views(self): 'description': 'Test view', 'filterable': True, 'id': 'd902fafc-5717-4dd0-87f2-7a6fc96989d9', - 'package_id': 'dp', - 'resource_id': 'd902fafc-5717-4dd0-87f2-7a6fc96989b7', 'responsive': True, 'show_fields': ['_id'] }] From daab7f7b1e803c931fc40b8bf03dac5b2c9284b0 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Fri, 22 May 2026 14:20:26 -0400 Subject: [PATCH 16/16] feat(tests): load datastore fields; - Test coverage for datastore field loading. --- ckanapi/cli/load.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index a2c271c..98b5320 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -252,14 +252,14 @@ def reply(action, error, response): reply(act, 'NotFound', obj) else: log_obj = {'id': r.get('id'), 'name': r.get('name')} - if arguments['--resource-views'] and resource_views: + if thing == 'datasets' and arguments['--resource-views'] and resource_views: if created_views: log_obj['created_resource_views'] = created_views if updated_views: log_obj['updated_resource_views'] = updated_views if skipped_views: log_obj['skipped_resource_views'] = skipped_views - if arguments['--datastore-fields'] and datastore_fields: + if thing == 'datasets' and arguments['--datastore-fields'] and datastore_fields: if created_tables: log_obj['created_datastore_tables'] = created_tables if skipped_tables: