diff --git a/ckanapi/cli/dump.py b/ckanapi/cli/dump.py index da826b8..ad7de53 100644 --- a/ckanapi/cli/dump.py +++ b/ckanapi/cli/dump.py @@ -249,7 +249,8 @@ def populate_res_views(ckan, res): return # with localckan we'll get the real CKAN exception not a CKANAPIError subclass if not views: return # return if the resource views list is empty - res['resource_views'] = views + rem = ['resource_id', 'package_id'] # remove unneeded key/values + res['resource_views'] = [{k: val for k, val in v.items() if k not in rem} for v in views] def populate_api_tokens(ckan, user): diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py index fcc06da..c7eb91a 100644 --- a/ckanapi/cli/load.py +++ b/ckanapi/cli/load.py @@ -209,14 +209,33 @@ def reply(action, error, response): act = 'update' if existing else 'create' try: api_token_list = obj.pop('api_token_list', None) # do not send api_token_list to user actions + # do not send resource_views & datastore_fields to package actions + resource_views = {} + datastore_fields = {} + if thing == 'datasets' and obj.get('resources'): + for r in obj['resources']: + # NOTE: will only work with existing Resource IDs in the input, + # documented in the command help. + if arguments['--resource-views']: + resource_views[r['id']] = r.pop('resource_views', []) + if arguments['--datastore-fields']: + datastore_fields[r['id']] = r.pop('datastore_fields', []) if existing: r = ckan.call_action(thing_update, obj, requests_kwargs=requests_kwargs) else: - r = ckan.call_action(thing_create, obj) - if thing == 'datasets' and 'resources' in obj:# check if it is needed to upload resources when creating/updating packages - _upload_resources(ckan,obj,arguments) - if thing in ['groups','organizations'] and 'image_display_url' in obj: #load images for groups and organizations + r = ckan.call_action(thing_create, obj, + requests_kwargs=requests_kwargs) + if thing == 'datasets' and 'resources' in obj: + # NOTE: order is important as Resource uploads may be dependant on DS Fields (XLoader/DataPusher), + # and Resource Views may be dependant on DS and Upload. + if arguments['--datastore-fields'] and datastore_fields: # check if it is needed to update datastore resource fields when creating/updating packages + created_tables, skipped_tables = _load_datastore_resource_fields(ckan, datastore_fields, arguments) + if arguments['--upload-resources']: # check if it is needed to upload resources when creating/updating packages + _upload_resources(ckan, obj, arguments) + if arguments['--resource-views'] and resource_views: # check if it is needed to create resource views when creating/updating packages + created_views, updated_views, skipped_views = _load_resource_views(ckan, resource_views, arguments) + if thing in ['groups','organizations'] and 'image_display_url' in obj: # load images for groups and organizations if arguments['--upload-logo']: users = obj['users'] obj = _upload_logo(ckan,obj) @@ -238,9 +257,20 @@ def reply(action, error, response): log_obj = {'id': r.get('id'), 'name': r.get('name')} if thing == 'users' and arguments['--api-tokens'] and api_token_list and created_tokens: log_obj['created_tokens'] = created_tokens + if thing == 'datasets' and arguments['--resource-views'] and resource_views: + if created_views: + log_obj['created_resource_views'] = created_views + if updated_views: + log_obj['updated_resource_views'] = updated_views + if skipped_views: + log_obj['skipped_resource_views'] = skipped_views + if thing == 'datasets' and arguments['--datastore-fields'] and datastore_fields: + if created_tables: + log_obj['created_datastore_tables'] = created_tables + if skipped_tables: + log_obj['skipped_datastore_tables'] = skipped_tables reply(act, None, log_obj) - def _worker_command_line(thing, arguments): """ Create a worker command line suitable for Popen with only the @@ -263,6 +293,8 @@ def b(name): + b('--upload-resources') + b('--upload-logo') + b('--api-tokens') + + b('--datastore-fields') + + b('--resource-views') ) @@ -282,10 +314,9 @@ def _copy_from_existing_for_update(obj, existing, thing): if 'users' not in obj and 'users' in existing: obj['users'] = existing['users'] -def _upload_resources(ckan,obj,arguments): + +def _upload_resources(ckan, obj, arguments): resources = obj['resources'] - if not arguments['--upload-resources']: - return requests_kwargs = None if arguments['--insecure']: requests_kwargs = {'verify': False} @@ -301,6 +332,90 @@ def _upload_resources(ckan,obj,arguments): requests_kwargs=requests_kwargs) +def _load_resource_views(ckan, resource_views, arguments): + """ + Loads resource views + """ + created = [] + updated = [] + skipped = [] + requests_kwargs = None + if arguments['--insecure']: + requests_kwargs = {'verify': False} + for rid, views in resource_views.items(): + for view in views: + existing = None + view['resource_id'] = rid + if not arguments['--create-only']: + if view.get('id'): + try: + existing = ckan.call_action('resource_view_show', + {'id': view['id']}, + requests_kwargs=requests_kwargs) + except NotFound: + pass + + if existing: + _copy_from_existing_for_update(view, existing, 'resource_view') + + if not existing and arguments['--update-only']: + skipped.append(view.get('id', view.get('view_type'))) + continue + + if existing: + # exceptions handled in load_things_worker + ckan.call_action('resource_view_update', view, + requests_kwargs=requests_kwargs) + updated.append(view.get('id', view.get('view_type'))) + else: + # exceptions handled in load_things_worker + ckan.call_action('resource_view_create', view, + requests_kwargs=requests_kwargs) + created.append(view.get('id', view.get('view_type'))) + + return created, updated, skipped + + +def _load_datastore_resource_fields(ckan, datastore_fields, arguments): + """ + Load datastore tables for Resources + """ + created = [] + skipped = [] + requests_kwargs = None + if arguments['--insecure']: + requests_kwargs = {'verify': False} + for rid, ds_fields in datastore_fields.items(): + if not ds_fields: + continue + existing = None + try: + existing = ckan.call_action('datastore_search', + {'resource_id': rid, 'limit': 0}, + requests_kwargs=requests_kwargs) + except NotFound: + pass + + try: + ckan.call_action( + 'datastore_create', + { + 'resource_id': rid, + 'fields': ds_fields, + 'force': True + }, + requests_kwargs=requests_kwargs) + created.append(rid) + except ValidationError as e: + if not existing: + # exceptions handled in load_things_worker + # raise normal exception for non-existing tables + raise e + skipped.append('%s: %s' % (rid, e)) + + return created, skipped + + def _upload_logo(ckan,obj_orig): obj = obj_orig.copy() for key in obj_orig.keys(): diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py index 9b3e308..4fef28f 100644 --- a/ckanapi/cli/main.py +++ b/ckanapi/cli/main.py @@ -12,9 +12,13 @@ (ID_OR_NAME ... | [-I JSONL_INPUT] [-s START] [-m MAX]) [-p PROCESSES] [-l LOG_FILE] [-qwz] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] - ckanapi dump (datasets | groups | organizations | related) + ckanapi dump datasets (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY]) - [-p PROCESSES] [-dqwzRU --include-private --include-drafts --include-deleted] + [-p PROCESSES] [-qwz --include-private --include-drafts --include-deleted --datastore-fields --resource-views] + [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] + ckanapi dump (groups | organizations | users | related) + (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY]) + [-p PROCESSES] [-qwzU] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] ckanapi dump users (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY]) @@ -22,7 +26,7 @@ [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]] ckanapi load datasets [--upload-resources] [-I JSONL_INPUT] [-s START] [-m MAX] - [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz] + [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz --datastore-fields --resource-views] [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]] ckanapi load (groups | organizations) [--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX] @@ -51,7 +55,9 @@ -c --config=CONFIG CKAN configuration file for local actions, defaults to $CKAN_INI or development.ini -d --datastore-fields export datastore field information along with - resource metadata as datastore_fields lists + resource metadata as datastore_fields lists (dump). + load datastore field information for resources (load). + Requires existing Resource IDs in the JSONL. --include-private include private datasets in the dump --include-drafts include draft datasets in the dump --include-deleted include deleted datasets in the dump @@ -78,7 +84,9 @@ -q --quiet don't display progress messages -r --remote=URL URL of CKAN server for remote actions -R --resource-views export resource views information along with - resource metadata as resource_views lists + resource metadata as resource_views lists (dump). + create/update resource views for resources (load). + Requires existing Resource IDs in the JSONL. -s --start-record=START start from record number START, where the first record is number 1 [default: 1] -u --ckan-user=USER perform actions as user with this name, uses the diff --git a/ckanapi/tests/test_cli_dump.py b/ckanapi/tests/test_cli_dump.py index c9c7c89..25ac9f6 100644 --- a/ckanapi/tests/test_cli_dump.py +++ b/ckanapi/tests/test_cli_dump.py @@ -369,8 +369,6 @@ def test_resource_views(self): 'description': 'Test view', 'filterable': True, 'id': 'd902fafc-5717-4dd0-87f2-7a6fc96989d9', - 'package_id': 'dp', - 'resource_id': 'd902fafc-5717-4dd0-87f2-7a6fc96989b7', 'responsive': True, 'show_fields': ['_id'] }] diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py index 9c1a42d..ec534fd 100644 --- a/ckanapi/tests/test_cli_load.py +++ b/ckanapi/tests/test_cli_load.py @@ -17,12 +17,19 @@ def call_action(self, name, data_dict, requests_kwargs=None): raise ValidationError({'users': 'should be unchanged'}) if data_dict['id'] == 'unused' and data_dict.get('users') != []: raise ValidationError({'users': 'should be cleared'}) + if name == 'resource_view_show' and data_dict['id'] == '123': + raise NotFound('no resource view with ID 123') + if name == 'datastore_search' and (data_dict['resource_id'] == '123' or data_dict['resource_id'] == '111'): + raise NotFound('no resource datastore with ID 123') + if name == 'datastore_create' and (data_dict['resource_id'] == '789' or data_dict['resource_id'] == '111'): + raise ValidationError({'pg_error': 'no db connection'}) try: return { 'package_show': { '12': {'title': "Twelve"}, '30ish': {'id': '34', 'title': "Thirty-four"}, '34': {'id': '34', 'title': "Thirty-four"}, + '46': {'id': '46', 'name': '46', 'title': 'Forty-six'}, }, 'group_show': { 'ab': {'title': "ABBA"}, @@ -34,10 +41,29 @@ def call_action(self, name, data_dict, requests_kwargs=None): }, 'package_create': { None: {'id': 'some-generated-uuid', 'name': 'something-new'}, + '46': {'id': '46', 'name': '46'}, }, 'package_update': { '34': {'id': '34', 'name': 'something-updated'}, + '46': {'id': '46', 'name': '46'}, }, + 'resource_view_show': { + '456': {'description': 'Test view', 'package_id': '46', 'resource_id': '456'} + }, + 'resource_view_create': { + '123': {'description': 'Test view', 'package_id': '46', 'resource_id': '123'}, + }, + 'resource_view_update': { + '456': {'description': 'Test view', 'package_id': '46', 'resource_id': '456'}, + }, + 'datastore_search': { + '456': {'resource_id': '456', 'fields': [{'id': 'test_field1', 'type': 'text'}, {'id': 'test_field2', 'type': 'text'}]}, + '789': {'resource_id': '789', 'fields': [{'id': 'test_field1', 'type': 'text'}, {'id': 'test_field2', 'type': 'text'}]}, + }, + 'datastore_create': { + '123': {'resource_id': '123', 'fields': [{'id': 'test_field1', 'type': 'text'}, {'id': 'test_field2', 'type': 'text'}]}, + '456': {'resource_id': '456', 'fields': [{'id': 'test_field1', 'type': 'text'}, {'id': 'test_field2', 'type': 'text'}]}, + }, 'group_update': { 'ab': {'id': 'ab', 'name': 'group-updated'}, }, @@ -60,8 +86,8 @@ def call_action(self, name, data_dict, requests_kwargs=None): 'id': 'this-is-a-token' } }, - }[name][data_dict.get('id')] - except KeyError: + }[name][data_dict.get('id', data_dict.get('resource_id'))] + except KeyError as e: raise NotFound() @@ -77,6 +103,8 @@ def test_create_with_no_resources(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -93,6 +121,8 @@ def test_create_with_corrupted_resources(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five","resources":[{"id":"123"}]}\n'), stdout=self.stdout) @@ -109,6 +139,8 @@ def test_create_with_complete_resources(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO( b'{"name": "45","title":"Forty-five",' @@ -121,12 +153,129 @@ def test_create_with_complete_resources(self): self.assertEqual(error, None) self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new'}) + def test_create_with_resource_views(self): + """ + A dataset with Resources that have views should create + the resource views. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '123', + 'url': 'http://example.com', + 'datastore_active': True, + 'resource_views': [{ + 'description': 'Test view', + 'filterable': True, + 'id': '123', + 'resource_id': '123', + 'responsive': True, + 'show_fields': ['_id'] + }] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': True, + '--update-only': False, + '--upload-resources': False, + '--insecure': False, + '--resource-views': True, + '--datastore-fields': False, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'create') + self.assertEqual(error, None) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new', 'created_resource_views': ['123']}) + + def test_create_with_resource_datastore_fields(self): + """ + A dataset with Resources that have datastore fields should create + the datastore table with the fields. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '123', + 'url': 'http://example.com', + 'datastore_active': True, + 'datastore_fields': [ + {'id': 'test_field1', 'type': 'text'}, + {'id': 'test_field2', 'type': 'text'}, + ] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': True, + '--update-only': False, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': True, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'create') + self.assertEqual(error, None) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new', 'created_datastore_tables': ['123']}) + + def test_create_with_bad_resource_datastore_fields(self): + """ + A dataset with Resources that have datastore fields that exist + but throw ValidationErrors should skip them. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '789', + 'url': 'http://example.com', + 'datastore_active': True, + 'datastore_fields': [ + {'id': 'test_field1', 'type': 'text'}, + {'id': 'test_field2', 'type': 'text'}, + ] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': True, + '--update-only': False, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': True, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'create') + self.assertEqual(error, None) + self.assertEqual(data, {'id': 'some-generated-uuid', 'name': 'something-new', 'skipped_datastore_tables': ["789: {'pg_error': 'no db connection'}"]}) + def test_create_only(self): load_things_worker(self.ckan, 'datasets', { '--create-only': True, '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -143,6 +292,8 @@ def test_create_empty_dict(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{}\n'), stdout=self.stdout) @@ -158,6 +309,8 @@ def test_create_bad_option(self): '--create-only': False, '--update-only': True, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'), stdout=self.stdout) @@ -173,6 +326,8 @@ def test_update_with_no_resources(self): '--create-only': False, '--update-only': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -189,6 +344,8 @@ def test_update_with_corrupted_resources(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten","resources":[{"id":"123"}]}\n'), stdout=self.stdout) @@ -205,6 +362,8 @@ def test_update_with_complete_resources(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO( b'{"name": "30ish","title":"3.4 times ten",' @@ -217,12 +376,238 @@ def test_update_with_complete_resources(self): self.assertEqual(error, None) self.assertEqual(data, {'id': '34', 'name': 'something-updated'}) + def test_update_with_resource_views(self): + """ + A dataset with Resources that have views should update + the resource views. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '456', + 'url': 'http://example.com', + 'datastore_active': True, + 'resource_views': [{ + 'description': 'Test view', + 'filterable': True, + 'id': '456', + 'resource_id': '456', + 'responsive': True, + 'show_fields': ['_id'] + }] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': True, + '--datastore-fields': False, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'update') + self.assertEqual(error, None) + self.assertEqual(data, {'id': '46', 'name': '46', 'updated_resource_views': ['456']}) + + def test_update_with_new_resource_views(self): + """ + A dataset with Resources that have NEW views should not + be able to create the resource views, just skip them. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '456', + 'url': 'http://example.com', + 'datastore_active': True, + 'resource_views': [{ + 'description': 'Test view', + 'filterable': True, + 'id': '123', + 'resource_id': '456', + 'responsive': True, + 'show_fields': ['_id'] + }] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': True, + '--datastore-fields': False, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'update') + self.assertEqual(error, None) + self.assertEqual(data, {'id': '46', 'name': '46', 'skipped_resource_views': ['123']}) + + def test_update_with_resource_datastore_fields(self): + """ + A dataset with Resources that have datastore fields should create + the datastore table with the fields. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '456', + 'url': 'http://example.com', + 'datastore_active': True, + 'datastore_fields': [ + {'id': 'test_field1', 'type': 'text'}, + {'id': 'test_field2', 'type': 'text'}, + ] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': True, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'update') + self.assertEqual(error, None) + self.assertEqual(data, {'id': '46', 'name': '46', 'created_datastore_tables': ['456']}) + + def test_update_with_bad_resource_datastore_fields(self): + """ + A dataset with Resources that have datastore fields that do not exist + should throw ValidationErrors should raise the error as normal. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'id': '111', + 'url': 'http://example.com', + 'datastore_active': True, + 'datastore_fields': [ + {'id': 'test_field1', 'type': 'text'}, + {'id': 'test_field2', 'type': 'text'}, + ] + }] + } + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': True, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + response = self.stdout.getvalue() + self.assertEqual(response[-1:], b'\n') + timstamp, action, error, data = json.loads(response.decode('UTF-8')) + self.assertEqual(action, 'update') + self.assertEqual(error, 'ValidationError') + self.assertEqual(data, {'pg_error': 'no db connection'}) + + def test_update_with_resource_datastore_fields_no_resource_id(self): + """ + A dataset with Resources that have datastore fields but no ID + should raise a KeyError. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'url': 'http://example.com', + 'datastore_active': True, + 'datastore_fields': [ + {'id': 'test_field1', 'type': 'text'}, + {'id': 'test_field2', 'type': 'text'}, + ] + }] + } + with self.assertRaises(KeyError) as ke: + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': False, + '--datastore-fields': True, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + self.assertEqual(str(ke.exception), "'id'") + + def test_update_with_resource_views_no_resource_id(self): + """ + A dataset with Resources that have views but no ID + should raise a KeyError. + """ + payload = { + 'name': '46', + 'title': 'Forty-six', + 'resources': [{ + 'name': 'resource1', + 'format': 'csv', + 'url': 'http://example.com', + 'datastore_active': True, + 'resource_views': [{ + 'description': 'Test view', + 'filterable': True, + 'id': '123', + 'resource_id': '456', + 'responsive': True, + 'show_fields': ['_id'] + }] + }] + } + with self.assertRaises(KeyError) as ke: + load_things_worker(self.ckan, 'datasets', { + '--create-only': False, + '--update-only': True, + '--upload-resources': False, + '--insecure': False, + '--resource-views': True, + '--datastore-fields': False, + }, + stdin=BytesIO(json.dumps(payload).encode()), + stdout=self.stdout) + self.assertEqual(str(ke.exception), "'id'") + def test_update_only(self): load_things_worker(self.ckan, 'datasets', { '--create-only': False, '--update-only': True, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -239,6 +624,8 @@ def test_update_bad_option(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'), stdout=self.stdout) @@ -255,6 +642,8 @@ def test_update_unauthorized(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"name": "seekrit", "title": "Things"}\n'), stdout=self.stdout) @@ -271,6 +660,8 @@ def test_update_group(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"id": "ab","title":"a balloon"}\n'), stdout=self.stdout) @@ -287,6 +678,8 @@ def test_update_organization_two(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO( b'{"name": "cd", "title": "Go"}\n' @@ -311,6 +704,8 @@ def test_update_organization_with_users_unchanged(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"id": "used", "title": "here"}\n'), stdout=self.stdout) @@ -327,6 +722,8 @@ def test_update_organization_with_users_cleared(self): '--update-only': False, '--upload-resources': False, '--insecure': False, + '--resource-views': False, + '--datastore-fields': False, }, stdin=BytesIO(b'{"id": "unused", "users": []}\n'), stdout=self.stdout) @@ -357,6 +754,8 @@ def test_parent_load_two(self): '--upload-logo': False, '--insecure': False, '--api-tokens': False, + '--resource-views': False, + '--datastore-fields': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -393,6 +792,8 @@ def test_parent_load_start_max(self): '--upload-logo': False, '--insecure': False, '--api-tokens': False, + '--resource-views': False, + '--datastore-fields': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO( @@ -432,6 +833,8 @@ def test_parent_parallel_limit(self): '--upload-logo': False, '--insecure': False, '--api-tokens': False, + '--resource-views': False, + '--datastore-fields': False, }, worker_pool=self._mock_worker_pool, stdin=BytesIO(