From a9d4e8475de64611eec4308fd7b0fd3b8022617d Mon Sep 17 00:00:00 2001 From: i-just Date: Thu, 9 Apr 2026 11:46:08 +0200 Subject: [PATCH 001/172] importing elements WIP --- composer.json | 1 + .../icons/light/arrow-down-to-bracket.svg | 1 + resources/icons/light/arrow-up-to-bracket.svg | 1 + .../import/_importer-types/base-importer.twig | 71 +++ .../_importer-types/element-importer.twig | 79 +++ resources/templates/import/configs/_edit.twig | 114 ++++ resources/templates/import/configs/index.twig | 140 +++++ resources/templates/import/index.twig | 28 + resources/templates/import/runs/_edit.twig | 130 ++++ resources/templates/import/runs/index.twig | 79 +++ routes/actions.php | 16 + routes/cp.php | 16 + src/Component/Contracts/Importable.php | 17 + src/Cp/Navigation.php | 25 + ...0000_00_00_000009_create_import_tables.php | 49 ++ src/Database/Migrations/Install.php | 25 + src/Database/Table.php | 4 + src/Element/Concerns/Draftable.php | 6 + src/Element/Concerns/HasCanonical.php | 3 + src/Element/Concerns/HasStatuses.php | 2 + src/Element/Concerns/Structurable.php | 2 + src/Element/Element.php | 10 +- src/Entry/Elements/Entry.php | 15 + .../Import/ImportConfigController.php | 248 ++++++++ .../Import/ImportRunController.php | 199 ++++++ src/Import/Commands/Element.php | 101 ++++ src/Import/Data/ImportRun.php | 149 +++++ src/Import/DataTypes/Csv.php | 36 ++ src/Import/DataTypes/DataTypeInterface.php | 13 + src/Import/DataTypes/Json.php | 28 + src/Import/DataTypes/Xml.php | 33 + src/Import/Events/DataImported.php | 18 + src/Import/Events/DataImporting.php | 21 + src/Import/Events/ImportConfigSaved.php | 18 + src/Import/Events/ImportConfigSaving.php | 21 + src/Import/Events/ImportRunDispatched.php | 18 + src/Import/Events/ImportRunDispatching.php | 21 + src/Import/Events/ImportRunSaved.php | 18 + src/Import/Events/ImportRunSaving.php | 21 + src/Import/Events/RegisterDataTypes.php | 29 + src/Import/Events/RegisterImporterTypes.php | 12 + src/Import/Import.php | 564 ++++++++++++++++++ src/Import/ImportServiceProvider.php | 18 + src/Import/Importers/BaseImporter.php | 292 +++++++++ src/Import/Importers/ElementImporter.php | 175 ++++++ src/Import/Jobs/Import.php | 78 +++ src/Import/Jobs/ImportPipeline.php | 45 ++ src/Import/Models/ImportConfig.php | 28 + src/Import/Models/ImportRun.php | 28 + .../Transformers/ElementTransformer.php | 86 +++ src/Import/Transformers/EntryTransformer.php | 53 ++ src/Providers/CraftServiceProvider.php | 2 + src/Support/Attributes/Importable.php | 15 + src/Support/Facades/Import.php | 36 ++ src/User/UserPermissions.php | 45 ++ yii2-adapter/legacy/web/assets/cp/dist/cp.js | 2 +- .../legacy/web/assets/cp/dist/cp.js.map | 2 +- .../legacy/web/assets/cp/dist/css/cp.css | 2 +- .../legacy/web/assets/cp/dist/css/cp.css.map | 2 +- .../legacy/web/assets/cp/src/css/_fld.scss | 6 +- .../web/assets/cp/src/css/_preview.scss | 6 +- .../legacy/web/assets/cp/src/css/_tokens.scss | 22 +- .../web/assets/cp/src/js/CraftTooltip.js | 3 +- .../legacy/web/assets/cp/src/js/UI.js | 16 +- .../web/assets/fileupload/webpack.config.js | 4 +- .../assets/iframeresizer/webpack.config.js | 4 +- .../jquerytouchevents/webpack.config.js | 4 +- yii2-adapter/legacy/web/twig/variables/Cp.php | 25 + 68 files changed, 3370 insertions(+), 31 deletions(-) create mode 100644 resources/icons/light/arrow-down-to-bracket.svg create mode 100644 resources/icons/light/arrow-up-to-bracket.svg create mode 100644 resources/templates/import/_importer-types/base-importer.twig create mode 100644 resources/templates/import/_importer-types/element-importer.twig create mode 100644 resources/templates/import/configs/_edit.twig create mode 100644 resources/templates/import/configs/index.twig create mode 100644 resources/templates/import/index.twig create mode 100644 resources/templates/import/runs/_edit.twig create mode 100644 resources/templates/import/runs/index.twig create mode 100644 src/Component/Contracts/Importable.php create mode 100644 src/Database/Migrations/0000_00_00_000009_create_import_tables.php create mode 100644 src/Http/Controllers/Import/ImportConfigController.php create mode 100644 src/Http/Controllers/Import/ImportRunController.php create mode 100644 src/Import/Commands/Element.php create mode 100644 src/Import/Data/ImportRun.php create mode 100644 src/Import/DataTypes/Csv.php create mode 100644 src/Import/DataTypes/DataTypeInterface.php create mode 100644 src/Import/DataTypes/Json.php create mode 100644 src/Import/DataTypes/Xml.php create mode 100644 src/Import/Events/DataImported.php create mode 100644 src/Import/Events/DataImporting.php create mode 100644 src/Import/Events/ImportConfigSaved.php create mode 100644 src/Import/Events/ImportConfigSaving.php create mode 100644 src/Import/Events/ImportRunDispatched.php create mode 100644 src/Import/Events/ImportRunDispatching.php create mode 100644 src/Import/Events/ImportRunSaved.php create mode 100644 src/Import/Events/ImportRunSaving.php create mode 100644 src/Import/Events/RegisterDataTypes.php create mode 100644 src/Import/Events/RegisterImporterTypes.php create mode 100644 src/Import/Import.php create mode 100644 src/Import/ImportServiceProvider.php create mode 100644 src/Import/Importers/BaseImporter.php create mode 100644 src/Import/Importers/ElementImporter.php create mode 100644 src/Import/Jobs/Import.php create mode 100644 src/Import/Jobs/ImportPipeline.php create mode 100644 src/Import/Models/ImportConfig.php create mode 100644 src/Import/Models/ImportRun.php create mode 100644 src/Import/Transformers/ElementTransformer.php create mode 100644 src/Import/Transformers/EntryTransformer.php create mode 100644 src/Support/Attributes/Importable.php create mode 100644 src/Support/Facades/Import.php diff --git a/composer.json b/composer.json index 4b3ba25fd19..bddae715171 100644 --- a/composer.json +++ b/composer.json @@ -50,6 +50,7 @@ "laravel/wayfinder": "^0.1.12", "league/commonmark": "^2.8", "league/flysystem-path-prefixing": "^3.31", + "league/fractal": "^0.21.0", "league/uri": "^7.0", "moneyphp/money": "^4.0", "phpdocumentor/reflection-docblock": "^5.3", diff --git a/resources/icons/light/arrow-down-to-bracket.svg b/resources/icons/light/arrow-down-to-bracket.svg new file mode 100644 index 00000000000..cd265e83586 --- /dev/null +++ b/resources/icons/light/arrow-down-to-bracket.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/resources/icons/light/arrow-up-to-bracket.svg b/resources/icons/light/arrow-up-to-bracket.svg new file mode 100644 index 00000000000..534c63b883b --- /dev/null +++ b/resources/icons/light/arrow-up-to-bracket.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/resources/templates/import/_importer-types/base-importer.twig b/resources/templates/import/_importer-types/base-importer.twig new file mode 100644 index 00000000000..69712896857 --- /dev/null +++ b/resources/templates/import/_importer-types/base-importer.twig @@ -0,0 +1,71 @@ +{% import '_includes/forms.twig' as forms %} + +{% set readOnly = readOnly ?? false %} +{% set static = static ?? false %} + +{% if (import.isEditable() == false) or static == true %} + {% set readOnly = true %} +{% endif %} + +{% set errors = errors ?? [] %} + +{{ forms.textField({ + label: "Data File"|t('app'), + instructions: "The absolute path to the file containing the data you want to import."|t('app'), + id: 'file', + name: 'file', + class: 'code', + value: old('settings.file', import.file), + required: true, + errors: sessionErrors.get('settings.file'), + data: {'error-key': 'settings.file'}, + disabled: readOnly, + placeholder: '@root/resources/my-data.json', + static: static, +}) }} + +{{ forms.textField({ + label: "Model Class"|t('app'), + instructions: "The class name (with namespace) of the model you’d like to import the data into."|t('app'), + id: 'className', + name: 'className', + class: 'code', + autocorrect: false, + autocapitalize: false, + maxlength: 255, + value: old('settings.className', import.className), + required: false, + errors: sessionErrors.get('settings.className'), + data: {'error-key': 'settings.className'}, + disabled: readOnly, + placeholder: 'my\\namespace\\Model', + static: static, +}) }} + +{#{{ forms.textField({ + label: "Transformer"|t('app'), + instructions: "The class name (with namespace) of the transformer you’d like to use."|t('app'), + id: 'transformer', + name: 'transformer', + class: 'code', + autocorrect: false, + autocapitalize: false, + maxlength: 255, + value: import ? old('transformer', import.transformer) : null, + required: false, + errors: sessionErrors.get('transformer'), + data: {'error-key': 'transformer'}, + disabled: readOnly, + placeholder: 'CraftCms\\Cms\Import\\Transformers\\EntryTransformer', + static: static, +}) }}#} + +
+

mapping stuff goes here, or maybe we want a separate screen?

+

+ if we're editing something that already exists, this is prepopulated with what's already saved;
+ if it's a new config (or the map in the config is empty), you use the UI to create your mapping;
+ which means you can create a new config, leave mapping empty, then go edit the mapping in the php file, + if that's easier for you and then go back here to tweak +

+
diff --git a/resources/templates/import/_importer-types/element-importer.twig b/resources/templates/import/_importer-types/element-importer.twig new file mode 100644 index 00000000000..dca7e5dcf62 --- /dev/null +++ b/resources/templates/import/_importer-types/element-importer.twig @@ -0,0 +1,79 @@ +{% import '_includes/forms.twig' as forms %} + +{% set readOnly = readOnly ?? false %} +{% set static = static ?? false %} + +{% if static == true %} + {% set readOnly = true %} +{% endif %} + +{% set errors = errors ?? [] %} + +{{ forms.textField({ + label: "Data File"|t('app'), + instructions: "The absolute path to the file containing the data you want to import."|t('app'), + id: 'file', + name: 'file', + class: 'code', + value: old('settings.file', import.file), + required: true, + errors: sessionErrors.get('settings.file'), + data: {'error-key': 'settings.file'}, + disabled: readOnly, + placeholder: '@root/resources/my-data.json', + static: static, +}) }} + +{{ forms.selectField({ + id: 'site', + name: 'site', + label: 'Site'|t('app'), + instructions: 'The site you want to import the data into'|t('app'), + options: availableSites, + value: old('settings.site', import.site) ?? primarySite.handle, + errors: sessionErrors.get('settings.site'), + data: {'error-key': 'settings.site'}, + disabled: readOnly, + static: static, +}) }} + +{{ forms.selectField({ + id: 'elementType', + name: 'elementType', + label: 'Element Type'|t('app'), + instructions: 'The element type this import is for.'|t('app'), + options: availableElementTypes, + value: old('settings.elementType') ?? defaultElementType, + errors: sessionErrors.get('settings.className'), + data: {'error-key': 'settings.elementType'}, + disabled: readOnly, + static: static, +}) }} + +{#{{ forms.textField({ + label: "Transformer"|t('app'), + instructions: "The class name (with namespace) of the transformer you’d like to use."|t('app'), + id: 'transformer', + name: 'transformer', + class: 'code', + autocorrect: false, + autocapitalize: false, + maxlength: 255, + value: import ? old('settings.transformer', import.transformer) : null, + required: false, + errors: sessionErrors.get('settings.transformer'), + data: {'error-key': 'settings.transformer'}, + disabled: readOnly, + placeholder: 'CraftCms\\Cms\Import\\Transformers\\ElementTransformer', + static: static, +}) }}#} + +
+

mapping stuff goes here, or maybe we want a separate screen?

+

+ if we're editing something that already exists, this is prepopulated with what's already saved;
+ if it's a new config (or the map in the config is empty), you use the UI to create your mapping;
+ which means you can create a new config, leave mapping empty, then go edit the mapping in the php file, + if that's easier for you and then go back here to tweak +

+
diff --git a/resources/templates/import/configs/_edit.twig b/resources/templates/import/configs/_edit.twig new file mode 100644 index 00000000000..10e7adb4991 --- /dev/null +++ b/resources/templates/import/configs/_edit.twig @@ -0,0 +1,114 @@ +{% import '_includes/forms.twig' as forms %} + +{% set readOnly = readOnly ?? false %} +{% set static = static ?? false %} + +{% if import is null %} + {{ forms.selectField({ + id: 'type', + name: 'type', + label: 'Importer Type'|t('app'), + options: importerTypes, + value: old('type'), + errors: sessionErrors.get('type'), + data: {'error-key': 'type'}, + disabled: readOnly, + static: static, + }) }} + +
+{% else %} + {% if static == true %} + {% set readOnly = true %} + {% endif %} + + {% set errors = errors ?? [] %} + + {% if not readOnly and import.uid is defined %} + {{ hiddenInput('uid', import.uid ) }} + {% endif %} + + {{ hiddenInput('type', className(import) ) }} + + {{ forms.textField({ + first: true, + label: "Name"|t('app'), + instructions: "What this import config will be called in the control panel."|t('app'), + id: 'name', + name: 'name', + value: old('name', import.name), + required: true, + errors: sessionErrors.get('name'), + autofocus: true, + data: {'error-key': 'name'}, + disabled: readOnly, + static: static, + }) }} + + {{ forms.textField({ + label: "Handle"|t('app'), + instructions: "How you’ll refer to this import config in the code."|t('app'), + id: 'handle', + name: 'handle', + class: 'code', + autocorrect: false, + autocapitalize: false, + maxlength: 64, + value: old('handle', import.handle), + required: true, + errors: sessionErrors.get('handle'), + data: {'error-key': 'handle'}, + disabled: readOnly, + static: static, + }) }} + + {{ forms.textareaField({ + label: 'Description'|t('app'), + instructions: 'A description of what this import config is for.'|t('app'), + id: 'description', + class: 'nicetext', + name: 'description', + value: old('description', import.description ?? null), + errors: sessionErrors.get('description'), + data: {'error-key': 'description'}, + disabled: readOnly, + static: static, + }) }} + +
+ {% namespace 'settings' %} + {% autoescape false %} + {{ readOnly ? import.getReadOnlySettingsHtml() : import.getSettingsHtml() }} + {% endautoescape %} + {% endnamespace %} +
+ + {% if not (import.handle ?? false) %} + {% js %} + new Craft.HandleGenerator('#{{ 'name'|namespaceInputId|e('js') }}', '#{{ 'handle'|namespaceInputId|e('js') }}'); + {% endjs %} + {% endif %} +{% endif %} + +{% js %} + $('#type').on('change', function() { + let data = { + type: $(this).val(), + }; + let $container = $('#settings'); + + Craft.sendActionRequest('POST', 'import/configs/render-settings', { + data: data, + }) + .then(async (response) => { + let $settings = $(response.data.settingsHtml || ''); + $container.html('').append($settings); + Craft.initUiElements($container); + await Craft.appendHeadHtml(response.data.headHtml); + await Craft.appendBodyHtml(response.data.bodyHtml); + }) + .catch((e) => { + Craft.cp.displayError(e?.response?.data?.message); + }); + }); +{% endjs %} diff --git a/resources/templates/import/configs/index.twig b/resources/templates/import/configs/index.twig new file mode 100644 index 00000000000..39dad4f916e --- /dev/null +++ b/resources/templates/import/configs/index.twig @@ -0,0 +1,140 @@ +{% extends "_layouts/cp" %} +{% set title = "Import configs"|t('app') %} + +{% set crumbs = [ + { label: "Import"|t('app'), url: url('import') } +] %} + +{% block actionButton %} + {% if not readOnly and currentUser.can('editImportConfigs') %} + {% set newImportUrl = url('import/configs/new') %} + {{ "New import config"|t('app') }} + {% endif %} +{% endblock %} + +{% if readOnly %} + {% set contentNotice = readOnlyNotice() %} +{% endif %} + +{% block content %} + {% if editableImportConfigs is empty %} +

{{ "No editable import configs yet."|t('app') }}

+ {% else %} + + + + + + + + + + + {% for config in editableImportConfigs %} + {#{% set config = importConfig.getConfig() %}#} + + + + + + + + + + {% endfor %} +
{{ "Name"|t('app') }}{{ "File"|t('app') }}{{ "Site"|t('app') }}{{ "Element import?"|t('app') }}{{ "Class name"|t('app') }}{{ "Map"|t('app') }}{{ "Actions"|t('app') }}
+ {{ config.name }} +
{{ config.handle }} + 		{{ config.file }}{{ config.site }} + {% if config.isElementImport() %} + {{ iconSvg('check', altText: 'yes') }} + {% else %} + {{ iconSvg('xmark', altText: 'no') }} + {% endif %} + {{ config.className ?? '' }} + {{ dump(config.map) }} + + {% set actionItems = [ + { + label: 'Edit'|t('app'), + url: url('import/configs/' ~ config.handle), + }, + ] %} + + {% if not readOnly and currentUser.can('deleteImportConfigs') %} + {% set actionItems = actionItems|merge([ + { + label: 'Delete'|t('app'), + action: 'import/configs/delete', + params: {uid: config.uid}, + destructive: true, + confirm: 'Are you sure you want to delete “{name}”?'|t('app', { + name: config.name, + }), + }, + ]) %} + {% endif %} + + {{ disclosureMenu(actionItems, { + buttonAttributes: { + class: ['action-btn', 'hairline'], + hiddenLabel: 'Actions'|t('app'), + }, + }) }} +
+ {% endif %} + +
+ + {% if nonEditableImportConfigs is empty %} +

{{ "No non-editable import configs yet."|t('app') }}

+ {% else %} + + + + + + + + + + + {% for config in nonEditableImportConfigs %} + {#{% set config = importConfig.getConfig() %}#} + + + + + + + + + + {% endfor %} +
{{ "Name"|t('app') }}{{ "Site"|t('app') }}{{ "Element import?"|t('app') }}{{ "Class name"|t('app') }}{{ "Transformer"|t('app') }}{{ "Map"|t('app') }}{{ "Actions"|t('app') }}
+ {{ config.name }} +
{{ config.handle }} + 		{{ config.site }} + {% if config.isElementImport() %} + {{ iconSvg('check', altText: 'yes') }} + {% else %} + {{ iconSvg('xmark', altText: 'no') }} + {% endif %} + {{ config.className ?? '' }}{{ className(config.transformer) }}{{ dump(config.map) }} + {% set actionItems = [ + { + label: 'Run'|t('app'), + action: 'import/configs/run', + params: {handle: config.handle}, + }, + ] %} + + {{ disclosureMenu(actionItems, { + buttonAttributes: { + class: ['action-btn', 'hairline'], + hiddenLabel: 'Actions'|t('app'), + }, + }) }} +
+ {% endif %} +{% endblock %} diff --git a/resources/templates/import/index.twig b/resources/templates/import/index.twig new file mode 100644 index 00000000000..533628191d0 --- /dev/null +++ b/resources/templates/import/index.twig @@ -0,0 +1,28 @@ +{% extends "_layouts/cp" %} +{% set title = "Import"|t('app') %} + +{% block content %} + {% if not currentUser.can('viewImportConfigs') and not currentUser.can('viewImportRuns') %} +

{{ 'You don’t have access to view import configs or runs.'|t('app') }}

+ {% endif %} + +{% endblock %} diff --git a/resources/templates/import/runs/_edit.twig b/resources/templates/import/runs/_edit.twig new file mode 100644 index 00000000000..170d470f2a2 --- /dev/null +++ b/resources/templates/import/runs/_edit.twig @@ -0,0 +1,130 @@ +{% import '_includes/forms.twig' as forms %} + +{% set readOnly = readOnly ?? false %} +{% set static = static ?? false %} + +{% set errors = errors ?? [] %} + +{% if not readOnly and run.uid %} + {{ hiddenInput('uid', run.uid ) }} +{% endif %} + +{{ forms.textField({ + first: true, + label: "Name"|t('app'), + id: 'name', + name: 'name', + value: old('name', run.name), + required: true, + errors: sessionErrors.get('name'), + autofocus: true, + data: {'error-key': 'name'}, + disabled: readOnly, + static: static, +}) }} + +{{ forms.textField({ + label: "Handle"|t('app'), + id: 'handle', + name: 'handle', + class: 'code', + autocorrect: false, + autocapitalize: false, + maxlength: 64, + value: old('handle', run.handle), + required: true, + errors: sessionErrors.get('handle'), + data: {'error-key': 'handle'}, + disabled: readOnly, + static: static, +}) }} + +{{ forms.textareaField({ + label: 'Description'|t('app'), + instructions: 'A description of what this import run is for.'|t('app'), + id: 'description', + class: 'nicetext', + name: 'description', + value: old('description', run.description ?? null), + errors: sessionErrors.get('description'), + data: {'error-key': 'description'}, + disabled: readOnly, + static: static, +}) }} + +{{ forms.editableTableField({ + label: "Steps"|t('app'), + instructions: "Define steps for your import."|t('app'), + id: 'steps', + name: 'steps', + cols: { + config: { + type: 'select', + heading: "Config"|t('app'), + options: configs ?? [], + class: ['config-cell'], + }, + file: { + heading: "File"|t('app'), + type: 'singleline', + class: ['file-cell'], + }, + batchSize: { + type: 'number', + heading: "Custom batch size"|t('app'), + info: "By default, each step will be run in batches containing up to 100 items. You can provide a different size, if you wish. Set to 0 to disable batching."|t('app'), + } + }|filter, + rows: run.steps, + fullWidth: true, + allowAdd: true, + allowDelete: true, + allowReorder: true, + errors: sessionErrors.get('steps'), + data: {'error-key': 'steps'}, + static: readOnly or static, +}) }} + + +{% if not (run.handle ?? false) %} + {% js %} + new Craft.HandleGenerator('#{{ 'name'|namespaceInputId|e('js') }}', '#{{ 'handle'|namespaceInputId|e('js') }}'); + {% endjs %} +{% endif %} + +{% js %} + let stepsTable = $('#steps').data('editable-table'); + + if (stepsTable) { + stepsTable.on('addRow', function(event) { + updateRow(event.$tr); + }); + } + + let $stepRows = $('#steps').children('tbody').children(); + + $stepRows.each(function () { + updateRow($(this)); + }); + + function updateRow($row) { + let $configSelect = $row.children('.config-cell').find('select'); + let $fileCell = $row.children('.file-cell'); + + toggleFile($configSelect, $fileCell); + + $configSelect.on('change', function() { + toggleFile($(this), $fileCell); + }); + } + + function toggleFile($select, $fileCell) { + let $selectedOption = $select.find(":selected"); + + if (Garnish.hasAttr($selectedOption, 'data-editable') || $selectedOption.data('editable') == 'true') { + $fileCell.addClass('disabled').find('textarea').attr({'tabindex': '-1', 'readonly': 'readonly'}); + } else { + $fileCell.removeClass('disabled').find('textarea').attr('tabindex', '0').removeAttr('readonly'); + } + } +{% endjs %} diff --git a/resources/templates/import/runs/index.twig b/resources/templates/import/runs/index.twig new file mode 100644 index 00000000000..64306429dd5 --- /dev/null +++ b/resources/templates/import/runs/index.twig @@ -0,0 +1,79 @@ +{% extends "_layouts/cp" %} +{% set title = "Import runs"|t('app') %} + +{% set crumbs = [ + { label: "Import"|t('app'), url: url('import') } +] %} + +{% block actionButton %} + {% if not readOnly and currentUser.can('editImportRuns') %} + {% set newImportUrl = url('import/runs/new') %} + {{ "New import run"|t('app') }} + {% endif %} +{% endblock %} + +{% if readOnly %} + {% set contentNotice = readOnlyNotice() %} +{% endif %} + +{% block content %} + {% if runs is empty %} +

{{ "No import runs exist yet."|t('app') }}

+ {% else %} + + + + + + {% for run in runs %} + + + + + {% endfor %} +
{{ "Name"|t('app') }}{{ "Actions"|t('app') }}
+ {{ run.name }} + + {% set actionItems = [ + { + label: 'Edit'|t('app'), + url: url('import/runs/' ~ run.handle), + }, + ] %} + + {% if currentUser.can('triggerImportRuns') %} + {% set actionItems = actionItems|merge([ + { + label: 'Start this run'|t('app'), + action: 'import/run', + params: {uid: run.uid}, + confirm: 'Are you sure you want to start “{name}” run?'|t('app', { + name: run.name, + }), + }, + ]) %} + {% endif %} + + {% if not readOnly and currentUser.can('deleteImportRuns') %} + {% set actionItems = actionItems|merge([ + { + label: 'Delete'|t('app'), + action: 'import/runs/delete', + params: {uid: run.uid}, + destructive: true, + confirm: 'Are you sure you want to delete “{name}”?'|t('app', { + name: run.name, + }), + } + ]) %} + {% endif %} + + {{ disclosureMenu(actionItems, { + buttonAttributes: { + class: ['action-btn', 'hairline'], + hiddenLabel: 'Actions'|t('app'), + }, + }) }} +
+ {% endif %} +{% endblock %} diff --git a/routes/actions.php b/routes/actions.php index 67cfef0f96d..74ad96b3d1b 100644 --- a/routes/actions.php +++ b/routes/actions.php @@ -35,6 +35,8 @@ use CraftCms\Cms\Http\Controllers\Gql\SchemasController as GqlSchemasController; use CraftCms\Cms\Http\Controllers\Gql\TokensController as GqlTokensController; use CraftCms\Cms\Http\Controllers\IconController; +use CraftCms\Cms\Http\Controllers\Import\ImportConfigController; +use CraftCms\Cms\Http\Controllers\Import\ImportRunController; use CraftCms\Cms\Http\Controllers\InstallController; use CraftCms\Cms\Http\Controllers\MigrateController; use CraftCms\Cms\Http\Controllers\PluginsController; @@ -270,6 +272,20 @@ }); }); + // Import + Route::middleware('can:editImportConfigs')->group(function () { + Route::post('import/configs/render-settings', [ImportConfigController::class, 'renderSettings']); + Route::post('import/configs/save', [ImportConfigController::class, 'store']); + }); + Route::middleware('can:deleteImportConfigs')->post('import/configs/delete', [ImportConfigController::class, 'destroy']); + + Route::middleware('can:editImportRuns')->post('import/runs/save', [ImportRunController::class, 'store']); + Route::middleware('can:deleteImportRuns')->post('import/runs/delete', [ImportRunController::class, 'destroy']); + Route::middleware('can:triggerImportRuns')->group(function () { + Route::post('import/run', [ImportRunController::class, 'run']); + Route::post('import/configs/run', [ImportConfigController::class, 'run']); + }); + // Migrations Route::post('utilities/apply-new-migrations', MigrationsController::class); diff --git a/routes/cp.php b/routes/cp.php index 2051dbd73fa..6481d2980e9 100644 --- a/routes/cp.php +++ b/routes/cp.php @@ -17,6 +17,8 @@ use CraftCms\Cms\Http\Controllers\Gql\IndexController as GqlIndexController; use CraftCms\Cms\Http\Controllers\Gql\SchemasController; use CraftCms\Cms\Http\Controllers\Gql\TokensController; +use CraftCms\Cms\Http\Controllers\Import\ImportConfigController; +use CraftCms\Cms\Http\Controllers\Import\ImportRunController; use CraftCms\Cms\Http\Controllers\InstallController; use CraftCms\Cms\Http\Controllers\PluginsController; use CraftCms\Cms\Http\Controllers\PluginStore\PluginStoreController; @@ -91,6 +93,20 @@ Route::view('content/{page}/{sectionHandle}', 'entries.index')->where('page', '[^\/]+'); Route::get('content/{section}/new', CreateEntryController::class); + /** + * Import + */ + Route::view('import', 'craftcms::import/index'); + Route::middleware('can:viewImportConfigs')->group(function () { + Route::get('import/configs', [ImportConfigController::class, 'index']); + Route::middleware('can:editImportConfigs')->get('import/configs/new', [ImportConfigController::class, 'create']); + Route::get('import/configs/{handle}', [ImportConfigController::class, 'edit']); + }); + Route::middleware('can:viewImportRuns')->group(function () { + Route::get('import/runs', [ImportRunController::class, 'index']); + Route::middleware('can:editImportRuns')->get('import/runs/new', [ImportRunController::class, 'create']); + Route::get('import/runs/{handle}', [ImportRunController::class, 'edit']); + }); /** * Users */ diff --git a/src/Component/Contracts/Importable.php b/src/Component/Contracts/Importable.php new file mode 100644 index 00000000000..444845530d3 --- /dev/null +++ b/src/Component/Contracts/Importable.php @@ -0,0 +1,17 @@ +can('viewImportConfigs') || $user?->can('viewImportRuns')) { + $subNavItems = []; + + if ($user?->can('viewImportConfigs')) { + $subNavItems['configs'] = [ + 'label' => t('Configs'), + 'url' => 'import/configs', + ]; + } + + if ($user?->can('viewImportRuns')) { + $subNavItems['runs'] = [ + 'label' => t('Runs'), + 'url' => 'import/runs', + ]; + } + + $navItems[] = [ + 'label' => t('Import'), + 'url' => 'import', + 'icon' => 'arrow-up-to-bracket', + 'subnav' => $subNavItems, + ]; + } + // Add any Plugin nav items $plugins = $this->plugins->getAllPlugins(); diff --git a/src/Database/Migrations/0000_00_00_000009_create_import_tables.php b/src/Database/Migrations/0000_00_00_000009_create_import_tables.php new file mode 100644 index 00000000000..a704a73f5e0 --- /dev/null +++ b/src/Database/Migrations/0000_00_00_000009_create_import_tables.php @@ -0,0 +1,49 @@ +integer('id', true); + $table->string('type'); + $table->string('name'); + $table->string('handle'); + $table->text('description')->nullable(); + $table->mediumText('settings')->nullable(); + $table->dateTime('dateCreated'); + $table->dateTime('dateUpdated'); + $table->dateTime('dateDeleted')->nullable()->default(null); + $table->char('uid', 36)->default('0'); + }); + } + + if (! Schema::hasTable(Table::IMPORT_RUNS)) { + Schema::create(Table::IMPORT_RUNS, function (Blueprint $table) { + $table->integer('id', true); + $table->string('name'); + $table->string('handle'); + $table->text('description')->nullable(); + $table->text('steps'); + $table->dateTime('dateCreated'); + $table->dateTime('dateUpdated'); + $table->dateTime('dateDeleted')->nullable()->default(null); + $table->char('uid', 36)->default('0'); + }); + } + } + + public function down(): void + { + Schema::dropIfExists(Table::IMPORT_CONFIGS); + Schema::dropIfExists(Table::IMPORT_RUNS); + } +}; diff --git a/src/Database/Migrations/Install.php b/src/Database/Migrations/Install.php index 5db600a3891..eb2a7c2dd20 100644 --- a/src/Database/Migrations/Install.php +++ b/src/Database/Migrations/Install.php @@ -474,6 +474,31 @@ public function createTables(): void $table->char('uid', 36)->default('0'); }); + Schema::create('import_configs', function (Blueprint $table) { + $table->integer('id', true); + $table->string('type'); + $table->string('name'); + $table->string('handle'); + $table->text('description')->nullable(); + $table->mediumText('settings')->nullable(); + $table->dateTime('dateCreated'); + $table->dateTime('dateUpdated'); + $table->dateTime('dateDeleted')->nullable()->default(null); + $table->char('uid', 36)->default('0'); + }); + + Schema::create('import_runs', function (Blueprint $table) { + $table->integer('id', true); + $table->string('name'); + $table->string('handle'); + $table->text('description')->nullable(); + $table->text('steps'); + $table->dateTime('dateCreated'); + $table->dateTime('dateUpdated'); + $table->dateTime('dateDeleted')->nullable()->default(null); + $table->char('uid', 36)->default('0'); + }); + /** @todo Change when Larastan is updated */ Schema::create('info', function (Blueprint $table) { $table->integer('id', true); diff --git a/src/Database/Table.php b/src/Database/Table.php index 6751cb581c2..fd717ab145b 100644 --- a/src/Database/Table.php +++ b/src/Database/Table.php @@ -65,6 +65,10 @@ public const string IMAGETRANSFORMS = 'imagetransforms'; + public const string IMPORT_CONFIGS = 'import_configs'; + + public const string IMPORT_RUNS = 'import_runs'; + public const string INFO = 'info'; public const string MIGRATIONS = 'migrations'; diff --git a/src/Element/Concerns/Draftable.php b/src/Element/Concerns/Draftable.php index 2f1612df3c6..821f6628ce8 100644 --- a/src/Element/Concerns/Draftable.php +++ b/src/Element/Concerns/Draftable.php @@ -6,6 +6,7 @@ use CraftCms\Cms\Database\Table; use CraftCms\Cms\Element\Events\AuthorizeCreateDrafts; +use CraftCms\Cms\Support\Attributes\Importable; use CraftCms\Cms\User\Elements\User as UserElement; use Illuminate\Support\Facades\DB; @@ -23,6 +24,7 @@ trait Draftable /** * @var int|null The ID of the draft’s row in the `drafts` table */ + #[Importable('draftId')] public ?int $draftId = null; /** @@ -35,6 +37,7 @@ trait Draftable /** * @var bool Whether this is a provisional draft. */ + #[Importable('isProvisionalDraft')] public bool $isProvisionalDraft = false; /** @@ -45,16 +48,19 @@ trait Draftable /** * @var int|null The creator’s ID */ + #[Importable('draftCreatorId')] public ?int $draftCreatorId = null; /** * @var string|null The draft name */ + #[Importable('draftName')] public ?string $draftName = null; /** * @var string|null The draft notes */ + #[Importable('draftNotes')] public ?string $draftNotes = null; /** diff --git a/src/Element/Concerns/HasCanonical.php b/src/Element/Concerns/HasCanonical.php index 5405c6284b7..34cf2469110 100644 --- a/src/Element/Concerns/HasCanonical.php +++ b/src/Element/Concerns/HasCanonical.php @@ -7,6 +7,7 @@ use craft\base\ElementInterface; use craft\base\NestedElementInterface; use craft\elements\db\NestedElementQueryInterface; +use CraftCms\Cms\Support\Attributes\Importable; use DateTime; use yii\base\NotSupportedException; @@ -56,6 +57,7 @@ trait HasCanonical * @see getIsCanonical() * @see getIsDerivative() */ + #[Importable('canonicalId')] private ?int $_canonicalId = null; /** @@ -71,6 +73,7 @@ trait HasCanonical /** * @see getCanonicalUid() */ + #[Importable('canonicalUid')] private ?string $_canonicalUid = null; public function getId(): ?int diff --git a/src/Element/Concerns/HasStatuses.php b/src/Element/Concerns/HasStatuses.php index 74fdb0de9dd..022cbfa6b40 100644 --- a/src/Element/Concerns/HasStatuses.php +++ b/src/Element/Concerns/HasStatuses.php @@ -4,6 +4,7 @@ namespace CraftCms\Cms\Element\Concerns; +use CraftCms\Cms\Support\Attributes\Importable; use CraftCms\Cms\Twig\Attributes\AllowedInSandbox; use function CraftCms\Cms\t; @@ -48,6 +49,7 @@ trait HasStatuses * @see getEnabledForSite() * @see setEnabledForSite() */ + #[Importable('enabledForSite')] private array|bool $_enabledForSite = true; /** diff --git a/src/Element/Concerns/Structurable.php b/src/Element/Concerns/Structurable.php index a40b792b4f3..0163b7eecef 100644 --- a/src/Element/Concerns/Structurable.php +++ b/src/Element/Concerns/Structurable.php @@ -11,6 +11,7 @@ use CraftCms\Cms\Element\Events\BeforeMoveInStructure; use CraftCms\Cms\Element\Queries\Contracts\ElementQueryInterface; use CraftCms\Cms\Element\Queries\ElementQuery; +use CraftCms\Cms\Support\Attributes\Importable; use CraftCms\Cms\Support\Facades\ElementCaches; use CraftCms\Cms\Support\Typecast; @@ -51,6 +52,7 @@ trait Structurable private ElementInterface|false|null $_prevElement = null; + #[Importable('parentId')] private int|false|null $_parentId = null; private ElementInterface|false|null $_parent = null; diff --git a/src/Element/Element.php b/src/Element/Element.php index 5984a8bc3d1..ac1033296a9 100644 --- a/src/Element/Element.php +++ b/src/Element/Element.php @@ -9,8 +9,10 @@ use craft\base\Component; use craft\base\ElementInterface; use CraftCms\Cms\Cms; +use CraftCms\Cms\Component\Contracts\Importable; use CraftCms\Cms\Element\Validation\ElementRules; use CraftCms\Cms\FieldLayout\LayoutElements\BaseField; +use CraftCms\Cms\Import\Transformers\ElementTransformer; use CraftCms\Cms\Support\Facades\Sites; use CraftCms\Cms\Support\Str; use CraftCms\Cms\Support\Utils; @@ -34,7 +36,7 @@ * Element is the base class for classes representing elements in terms of objects. */ #[Ruleset(ElementRules::class)] -abstract class Element extends Component implements ElementInterface +abstract class Element extends Component implements ElementInterface, Importable { use ArrayableTrait { toArray as traitToArray; @@ -723,4 +725,10 @@ public function safeAttributes(): array { return array_keys($this->getRuleset()->rules()); } + + #[Override] + public static function getDefaultTransformer(): ?string + { + return ElementTransformer::class; + } } diff --git a/src/Entry/Elements/Entry.php b/src/Entry/Elements/Entry.php index 2c860a3affe..06baca33253 100644 --- a/src/Entry/Elements/Entry.php +++ b/src/Entry/Elements/Entry.php @@ -54,6 +54,7 @@ use CraftCms\Cms\FieldLayout\FieldLayout; use CraftCms\Cms\FieldLayout\LayoutElements\entries\EntryTitleField; use CraftCms\Cms\Gql\Interfaces\Elements\Entry as EntryInterface; +use CraftCms\Cms\Import\Transformers\EntryTransformer; use CraftCms\Cms\Section\Data\Section; use CraftCms\Cms\Section\Data\SectionSiteSettings; use CraftCms\Cms\Section\Enums\DefaultPlacement; @@ -62,6 +63,7 @@ use CraftCms\Cms\Site\Data\Site; use CraftCms\Cms\Structure\Enums\Mode; use CraftCms\Cms\Support\Arr; +use CraftCms\Cms\Support\Attributes\Importable; use CraftCms\Cms\Support\DateTimeHelper; use CraftCms\Cms\Support\Facades\DeltaRegistry; use CraftCms\Cms\Support\Facades\ElementActions; @@ -787,6 +789,7 @@ protected static function prepElementQueryForTableAttribute(ElementQueryInterfac * {{ entry.sectionId }} * ``` */ + #[Importable('sectionId')] public ?int $sectionId = null; /** @@ -807,6 +810,7 @@ protected static function prepElementQueryForTableAttribute(ElementQueryInterfac * ``` */ #[AllowedInSandbox] + #[Importable('postDate')] public ?DateTime $postDate = null; /** @@ -824,6 +828,7 @@ protected static function prepElementQueryForTableAttribute(ElementQueryInterfac * ``` */ #[AllowedInSandbox] + #[Importable('expiryDate')] public ?DateTime $expiryDate = null; /** @@ -834,6 +839,7 @@ protected static function prepElementQueryForTableAttribute(ElementQueryInterfac /** * @var self::STATUS_LIVE|self::STATUS_PENDING|self::STATUS_EXPIRED */ + #[Importable('status')] private string $status; /** @@ -859,6 +865,7 @@ protected static function prepElementQueryForTableAttribute(ElementQueryInterfac * * @since 5.7.0 */ + #[Importable('placeInStructure')] public bool $placeInStructure = false; /** @@ -867,6 +874,7 @@ protected static function prepElementQueryForTableAttribute(ElementQueryInterfac * @see getAuthorIds() * @see setAuthorIds() */ + #[Importable('authorIds')] private array $_authorIds; /** @@ -890,6 +898,7 @@ protected static function prepElementQueryForTableAttribute(ElementQueryInterfac * * @see getType() */ + #[Importable('typeId')] private ?int $_typeId = null; private ?int $_oldTypeId = null; @@ -2900,4 +2909,10 @@ protected function partialTemplatePathCandidates(): array return $templates; } + + #[Override] + public static function getDefaultTransformer(): ?string + { + return EntryTransformer::class; + } } diff --git a/src/Http/Controllers/Import/ImportConfigController.php b/src/Http/Controllers/Import/ImportConfigController.php new file mode 100644 index 00000000000..f762fac4cb7 --- /dev/null +++ b/src/Http/Controllers/Import/ImportConfigController.php @@ -0,0 +1,248 @@ +readOnly = ! $generalConfig->allowAdminChanges; + $this->cpTrigger = $generalConfig->cpTrigger; + } + + public function index(): View + { + return view('craftcms::import.configs.index', [ + 'readOnly' => $this->readOnly, + 'editableImportConfigs' => $this->importService->getEditableConfigs(), + 'nonEditableImportConfigs' => $this->importService->getNonEditableConfigs(), + ]); + } + + public function create(Request $request): CpScreenResponse + { + $old = $request->old() ?? $request->session()?->get('import'); + if (! empty($old)) { + $import = new ($old['type'])($old); + } else { + $type = $request->input('type'); + if ($type) { + $import = new $type; + } else { + $import = null; + } + } + + return $this->cpScreenResponse($import); + } + + public function renderSettings(Request $request): JsonResponse + { + $request->validate([ + 'type' => ['required', 'string'], + 'namespace' => ['nullable', 'string'], + ]); + + $type = $request->input('type'); + $import = new $type; + + $html = template('import/configs/_edit', [ + 'import' => $import, + 'namespace' => $request->input('namespace'), + ]); + + return new JsonResponse([ + 'settingsHtml' => $html, + 'headHtml' => $this->HtmlStack->headHtml(), + 'bodyHtml' => $this->HtmlStack->bodyHtml(), + ]); + } + + public function edit(Request $request, ?BaseImporter $import = null, ?string $handle = null): CpScreenResponse + { + $handle ??= $import->handle ?? $request->input('handle'); + + if (is_null($handle)) { + return $this->create($request); + } + + abort_if(is_null($found = $this->importService->getConfigByHandle($handle)), 404, 'Import config not found'); + abort_if(! $found->isEditable(), 400, "This import config is not editable: $found->handle"); + + if ($import === null) { + $import = $found; + } + + return $this->cpScreenResponse($import); + } + + public function store(Request $request): Response + { + $request->validate([ + 'uid' => ['nullable', 'string', 'max:36'], + ]); + + $importConfigUid = $request->input('uid'); + + if ($importConfigUid) { + abort_if(is_null($import = $this->importService->getConfigByUid($importConfigUid)), 400, "Invalid import config UID: $importConfigUid"); + } else { + $import = new ($request->input('type')); + } + + $request->validate($import::getRules()); + + $import->name($request->input('name', $import->name)); + $import->handle($request->input('handle', $import->handle)); + $import->description($request->input('description', $import->description)); + $import->file($request->input('settings.file', $import->file)); + $import->site($request->input('settings.site', $import->site)); + $import->className($request->input('settings.elementType')); + $import->transformer($request->input('settings.transformer', $import->transformer)); + $import->map($request->input('settings.map', $import->map)); + $import->matchCriteria($request->input('settings.matchCriteria', $import->matchCriteria)); + + if (! $this->importService->saveConfig($import)) { + // Flash::fail(t('Couldn’t save import config.')); + return $this->asModelFailure($import, t('Couldn’t save import config.'), 'import'); + } + + return $this->asModelSuccess( + $import, + t('Import config saved.'), + 'import', + ); + } + + public function destroy(Request $request): Response + { + $uid = $request->input('uid'); + + if (! $uid) { + throw ValidationException::withMessages([ + 'id' => t('uid is required.'), + ]); + } + + $config = $this->importService->getConfigByUid($uid); + + abort_if(is_null($config), 404, "Invalid import config UID: $uid"); + abort_if(! $config->editable, 400, "This import config is not editable, so it can’t be deleted via the Control Panel: $uid"); + + $this->importService->deleteConfig($config); + + return $this->asSuccess(t('“{name}” deleted.', [ + 'name' => $config->name, + ])); + } + + private function cpScreenResponse(?BaseImporter $import = null): CpScreenResponse + { + $currentUser = auth('craft')->user(); + + $templateVars = [ + 'readOnly' => $this->readOnly, + 'static' => ! $currentUser?->can('editImportConfigs'), + 'import' => $import, + ]; + + if ($import === null) { + $templateVars['importerTypes'] = array_map(fn ($type) => [ + 'label' => $type::displayName(), + 'value' => $type, + ], $this->importService->getAllImporterTypes()); + array_unshift($templateVars['importerTypes'], ['label' => t('Please select'), 'value' => null]); + } + + return new CpScreenResponse() + ->title(! isset($import?->uid) ? t('Create a new import config') : t('Edit {name} import config', ['name' => $import->name])) + ->addCrumb(t('Import'), 'import') + ->addCrumb(t('Configs'), 'import/configs') + ->contentTemplate('import/configs/_edit.twig', $templateVars) + ->unless( + $this->readOnly || ! $currentUser?->can('editImportConfigs'), + callback: function (CpScreenResponse $response) use ($import) { + $response + ->action('import/configs/save') + ->redirectUrl('import/configs') + ->addAltAction(t('Save and continue editing'), [ + 'redirect' => 'import/configs/{handle}', + 'shortcut' => true, + 'retainScroll' => true, + ]) + ->addAltAction(t('Delete'), [ + 'action' => 'import/configs/delete', + 'redirect' => 'import/configs', + 'destructive' => true, + 'confirm' => t('Are you sure you want to delete “{name}”?', [ + 'name' => $import?->name, + ]), + ]); + }, + default: function (CpScreenResponse $response) { + if ($this->readOnly) { + $response->noticeHtml(Cp::readOnlyNoticeHtml()); + } + }, + ); + } + + public function run(Request $request): Response + { + $handle = $request->input('handle'); + + abort_if(is_null($handle), 400, 'Import config handle is required.'); + abort_if(is_null($config = $this->importService->getConfigByHandle($handle)), 400, 'Import config not found.'); + + try { + $file = $config->file; + $filePath = BaseImporter::resolvedFilePath($file); + + // TEMP - start + $allData = $this->importService->getData($filePath); + $data = array_slice($allData, 0); + $dataCount = count($data); + + for ($i = 0; $i < $dataCount; $i++) { + $this->importService->importItem($config, $data[$i]); + } + // TEMP - end + } catch (Throwable $e) { + Log::warning("Import failed: {$e->getMessage()}"); + + return $this->asFailure(t('Import could not be started.')); + } + + return $this->asSuccess(t('Import started')); + } +} diff --git a/src/Http/Controllers/Import/ImportRunController.php b/src/Http/Controllers/Import/ImportRunController.php new file mode 100644 index 00000000000..e99c3f46942 --- /dev/null +++ b/src/Http/Controllers/Import/ImportRunController.php @@ -0,0 +1,199 @@ +readOnly = ! $generalConfig->allowAdminChanges; + } + + public function index(): View + { + return view('craftcms::import.runs.index', [ + 'readOnly' => $this->readOnly, + 'runs' => $this->importService->getImportRuns(), + ]); + } + + public function create(Request $request): CpScreenResponse + { + $old = $request->session()?->get('run'); + if (! empty($old)) { + $run = new ImportRun($old); + } else { + $run = new ImportRun; + } + + return $this->cpScreenResponse($run); + } + + public function edit(Request $request, ?ImportRun $run = null, ?string $handle = null): CpScreenResponse + { + $handle ??= $run->handle ?? $request->input('handle'); + + if (is_null($handle)) { + return $this->create($request); + } + + abort_if(is_null($found = $this->importService->getImportRunByHandle($handle)), 404, 'Import run not found'); + + $old = $request->session()?->get('run'); + if (! empty($old)) { + $run = new ImportRun($old); + } + + if ($run === null) { + $run = $found; + } + + return $this->cpScreenResponse($run); + } + + public function store(Request $request): Response + { + $runUid = $request->input('uid'); + + if ($runUid) { + abort_if(is_null($run = $this->importService->getImportRunByUid($runUid)), 400, "Invalid run UID: $runUid"); + } else { + $run = new ImportRun; + } + + $run->name = $request->input('name', $run->name); + $run->handle = $request->input('handle', $run->handle); + $run->description = $request->input('description', $run->description); + $run->steps = $request->input('steps', $run->steps); + + if (! $this->importService->saveRun($run)) { + return $this->asModelFailure($run, t('Couldn’t save import run.'), 'run'); + } + + return $this->asModelSuccess( + $run, + t('Import run saved.'), + 'run', + ); + } + + public function destroy(Request $request): Response + { + $uid = $request->input('uid'); + + if (! $uid) { + throw ValidationException::withMessages([ + 'id' => t('uid is required.'), + ]); + } + + $run = $this->importService->getImportRunByUid($uid); + + abort_if(is_null($run), 404, "Invalid import run UID: $uid"); + + $this->importService->deleteRun($run); + + return $this->asSuccess(t('“{name}” deleted.', [ + 'name' => $run->name, + ])); + } + + public function run(Request $request): Response + { + $uid = $request->input('uid'); + + abort_if(is_null($uid), 400, 'Import run uid is required.'); + abort_if(is_null($run = $this->importService->getImportRunByUid($uid)), 400, 'Import run not found.'); + + try { + /** @phpstan-ignore-next-line */ + $this->importService->dispatchImport($run); + } catch (Throwable $e) { + Log::warning("Import run failed: {$e->getMessage()}"); + + return $this->asFailure(t('Import could not be started.')); + } + + return $this->asSuccess(t('Import started')); + } + + private function cpScreenResponse(ImportRun $run): CpScreenResponse + { + $currentUser = auth('craft')->user(); + + return new CpScreenResponse() + ->title(! isset($run->uid) ? t('Create a new import run') : t('Edit {name} import run', ['name' => $run->name])) + ->addCrumb(t('Import'), 'import') + ->addCrumb(t('Runs'), 'import/runs') + ->contentTemplate('import/runs/_edit.twig', [ + 'run' => $run, + 'configs' => $this->importService->getAllConfigs() + ->map(fn ($config) => [ + 'label' => $config->name, + 'value' => $config->editable ? $config->uid : $config->handle, + 'data' => ['editable' => $config->editable], + ]) + ->prepend(['label' => t('Please select'), 'value' => null]) + ->all(), + 'readOnly' => $this->readOnly, + 'static' => ! $currentUser?->can('editImportRuns'), + ]) + ->unless( + $this->readOnly || ! $currentUser?->can('editImportRuns'), + callback: function (CpScreenResponse $response) use ($run) { + $response + ->action('import/runs/save') + ->redirectUrl('import/runs') + ->addAltAction(t('Save and continue editing'), [ + 'redirect' => 'import/runs/{handle}', + 'shortcut' => true, + 'retainScroll' => true, + ]) + ->addAltAction(t('Delete'), [ + 'action' => 'import/runs/delete', + 'redirect' => 'import/runs', + 'destructive' => true, + 'confirm' => t('Are you sure you want to delete “{name}”?', [ + 'name' => $run->name, + ]), + ]); + + if ($run->uid) { + $response->addAltAction(t('Start this run'), [ + 'action' => 'import/run', + 'redirect' => 'import/runs', + 'confirm' => t('Are you sure you want to start this import?'), + ]); + } + }, + default: function (CpScreenResponse $response) { + if ($this->readOnly) { + $response->noticeHtml(Cp::readOnlyNoticeHtml()); + } + }, + ); + } +} diff --git a/src/Import/Commands/Element.php b/src/Import/Commands/Element.php new file mode 100644 index 00000000000..67165cc2644 --- /dev/null +++ b/src/Import/Commands/Element.php @@ -0,0 +1,101 @@ +addIf(! $this->option('elementType'), fn ($form) => select( + label: 'Which element type you want to import into?', + options: collect((new Elements)->getAllElementTypes()) + ->all() + ), 'elementType') + // TODO: do we want to support URLs containing all the data (like in feed me where you can use rss feed) or just files? + ->addIf(! $this->option('file'), fn () => text( + label: 'The `@root`-relative path to the file containing the data you want to import', + required: true, + validate: [ + 'string', + ] + ), 'file') + ->addIf(! $this->option('site') && Sites::isMultiSite(), fn ($form) => select( + label: 'Which site you want to import into?', + options: Sites::getAllSites() + ->mapWithKeys(fn (Site $site) => [$site->handle => $site->name]) + ->all(), + default: Sites::getPrimarySite()->handle, + ), 'site') + // TODO: maybe change this to a select field and show all available transformers? but then we'd still have to allow for custom ones too + ->addIf(! $this->option('transformer'), fn () => text( + label: 'The transformer you want to use to manipulate the data on import', + validate: [ + 'string', + ] + ), 'transformer') + ->submit(); + + // important: don't change "?:" to "??" as it'll treat an empty string passed into --optionName as valid + $config['elementImport'] = true; + $config['className'] = $this->option('elementType') ?: $responses['elementType']; + $config['file'] = $this->option('file') ?: $responses['file']; + $config['site'] = $this->option('site') ?: $responses['site'] ?? Sites::getPrimarySite()->handle; + $config['transformer'] = $this->option('transformer') ?: $responses['transformer']; + + // $validator = Validator::make($config, ElementImporter::getRules()); + // + // if ($validator->fails()) { + // foreach ($validator->errors()->all() as $error) { + // $this->error($error); + // } + // return self::FAILURE; + // } + // + // $importConfig = new ElementImporter($config); + + $importConfig = new ElementImporter($config); + + $this->components->info("element type: `{$importConfig->className}`"); + $this->components->info("file: `{$importConfig->file}`"); + $this->components->info("site: `{$importConfig->site}`"); + $this->components->info('transformer: '.$importConfig->transformer::class); + + Import::import($importConfig); + + return self::SUCCESS; + } +} diff --git a/src/Import/Data/ImportRun.php b/src/Import/Data/ImportRun.php new file mode 100644 index 00000000000..287f314cdf2 --- /dev/null +++ b/src/Import/Data/ImportRun.php @@ -0,0 +1,149 @@ + [ + 'required', + 'string', + 'max:255', + ], + 'handle' => [ + 'required', + 'string', + 'max:255', + new HandleRule(['id', 'dateCreated', 'dateUpdated', 'uid', 'title']), + function ($attribute, $value, Closure $fail, Validator $validator) { + $found = Import::getImportRunByHandle($value); + if ($found !== null && $found->uid !== $validator->getValue('uid')) { + $fail(t('{attribute} "{value}" has already been taken.', [ + 'attribute' => $attribute, + 'value' => $value, + ])); + } + }, + ], + 'description' => [ + 'string', + 'nullable', + ], + 'steps' => [ + 'required', + ], + 'steps.*.config' => [ + Rule::in(array_merge(Import::getEditableConfigs()->pluck('uid')->toArray(), Import::getNonEditableConfigs()->keys()->all())), + ], + 'steps.*.file' => [ + function ($attribute, $value, Closure $fail, Validator $validator) { + $key = preg_match('/\d+/', $attribute, $matches) ? (int) $matches[0] : null; + $config = Import::getConfigByHandle($this->steps[$key]['config']) ?? Import::getConfigByUid($this->steps[$key]['config']); + if ($config && ! $config->editable) { + // if the config is not editable (file-based), + // then the file is required and has to be valid + return BaseImporter::validateFile($value, $attribute, $fail, $validator, 'steps'); + } + + // if config is editable, clear out the file value, just in case + if ($config && $config->editable) { + $this->steps[$key]['file'] = null; + } + + return true; + }, + ], + 'steps.*.batchSize' => [ + 'nullable', + 'integer', + 'min:0', + 'max:1000', + ], + ]; + } + + public function afterValidate(?Validator $validator = null): void + { + // move all the nested steps validation messages up top for now + // we might want to change this if/once the editable table is rewritten + if ($validator->errors()->has('steps.*')) { + $nestedErrors = $validator->errors()->get('steps.*'); + foreach ($nestedErrors as $key => $bag) { + $validator->errors()->add('steps', ...$bag); + $validator->errors()->forget($key); + } + } + } + + // public function getMessages(): array + // { + // return [ + // //'steps.*.file' => 'test234', // works + // //'steps.*.file.closure_validation_rule' => 'test567', // works + // ]; + // } + + public function getConfig(): array + { + return [ + 'name' => $this->name, + 'handle' => $this->handle, + 'steps' => $this->steps, + 'uid' => $this->uid, + ]; + } + + /** + * {@inheritdoc} + */ + public function getCpEditUrl(): ?string + { + if (! $this->handle || ! Auth::user()?->isAdmin()) { + return null; + } + + return UrlHelper::cpUrl("import/runs/$this->uid"); + } +} diff --git a/src/Import/DataTypes/Csv.php b/src/Import/DataTypes/Csv.php new file mode 100644 index 00000000000..92dceb82bd9 --- /dev/null +++ b/src/Import/DataTypes/Csv.php @@ -0,0 +1,36 @@ +setDelimiter(','); + $reader->setReadDataOnly(true); + // $reader->setSheetIndex(0); + + $spreadsheet = $reader->loadSpreadsheetFromString($data); + $sheet = $spreadsheet->getSheet(0); + $data = $sheet->toArray(); + $headings = array_shift($data); + $array = array_map(fn ($row) => array_combine($headings, $row), $data); + } catch (InvalidArgumentException $e) { + $error = 'Invalid CSV: '.$e->getMessage(); + + return ['success' => false, 'error' => $error]; + } + + return ['success' => true, 'data' => $array]; + } +} diff --git a/src/Import/DataTypes/DataTypeInterface.php b/src/Import/DataTypes/DataTypeInterface.php new file mode 100644 index 00000000000..5cf2d49d065 --- /dev/null +++ b/src/Import/DataTypes/DataTypeInterface.php @@ -0,0 +1,13 @@ +getMessage(); + + return ['success' => false, 'error' => $error]; + } + + return ['success' => true, 'data' => $array]; + } +} diff --git a/src/Import/DataTypes/Xml.php b/src/Import/DataTypes/Xml.php new file mode 100644 index 00000000000..6aec19b41df --- /dev/null +++ b/src/Import/DataTypes/Xml.php @@ -0,0 +1,33 @@ +getMessage()}"; + + return ['success' => false, 'error' => $error]; + } + + return ['success' => true, 'data' => $array]; + } +} diff --git a/src/Import/Events/DataImported.php b/src/Import/Events/DataImported.php new file mode 100644 index 00000000000..7ab472d6c54 --- /dev/null +++ b/src/Import/Events/DataImported.php @@ -0,0 +1,18 @@ +types->add(MyDataType::class); + * }); + * ``` + */ +class RegisterDataTypes +{ + public function __construct( + /** @var array> */ + public array $dataTypes, + ) {} +} diff --git a/src/Import/Events/RegisterImporterTypes.php b/src/Import/Events/RegisterImporterTypes.php new file mode 100644 index 00000000000..62e4395204b --- /dev/null +++ b/src/Import/Events/RegisterImporterTypes.php @@ -0,0 +1,12 @@ + Json::class, + 'csv' => Csv::class, + 'xml' => Xml::class, + ]; + + if (Event::hasListeners(RegisterDataTypes::class)) { + Event::dispatch($event = new RegisterDataTypes($dataTypes)); + + $dataTypes = $event->dataTypes; + } + + return $dataTypes; + } + + public function getAllImporterTypes(): array + { + $importers = [ + ElementImporter::class, + ]; + + if (Event::hasListeners(RegisterImporterTypes::class)) { + Event::dispatch($event = new RegisterImporterTypes($importers)); + + $importers = $event->importers; + } + + return $importers; + } + + // //// configs ////// + public function createImporter($config) + { + $importer = new $config['type']($config); + $importer->name($config['name']); + $importer->handle($config['handle']); + $importer->description($config['description']); + $settings = JsonSupport::decode($config['settings']); + foreach ($settings as $setting => $value) { + if (method_exists($importer, $setting)) { + $importer->{$setting}($value); + } + } + + return $importer; + } + + public function getAllConfigs(): LaravelCollection + { + if ($this->configs === null) { + $dbConfigs = $this->_importConfigQuery()->get()->all(); + $dbConfigs = array_map( + fn ($config) => $this->createImporter((array) $config + ['editable' => true]), + $dbConfigs + ); + + $fileConfigs = Config::get('craft.import') ?? []; + + foreach ($fileConfigs as &$fileConfig) { + $fileConfig = $fileConfig(); + + // TODO: this might be wrong - think about it; + // if there's no transformer set, use the default one + if ($fileConfig->transformer === null) { + $fileConfig->transformer(null); + } + } + + $this->configs = new LaravelCollection($dbConfigs + $fileConfigs) + ->keyBy(fn (BaseImporter $item, $key) => $item->handle ?? $key) + ->sortBy('name'); + } + + return $this->configs; + } + + public function getEditableConfigs(): LaravelCollection + { + return $this->getAllConfigs()->filter(fn ($config) => $config->isEditable()); + } + + public function getNonEditableConfigs(): LaravelCollection + { + return $this->getAllConfigs()->reject(fn ($config) => $config->isEditable()); + } + + public function getConfigByHandle(?string $handle, bool $editableOnly = false): ?BaseImporter + { + if (is_null($handle)) { + return null; + } + + if ($editableOnly) { + $configs = $this->getEditableConfigs(); + } else { + $configs = $this->getAllConfigs(); + } + + /** @var BaseImporter|null */ + return $configs->where('handle', $handle)->first(); + } + + public function getConfigByUid(string $uid, bool $editableOnly = false): ?BaseImporter + { + if ($editableOnly) { + $configs = $this->getEditableConfigs(); + } else { + $configs = $this->getAllConfigs(); + } + + /** @var BaseImporter|null */ + return $configs->where('uid', $uid)->first(); + } + + public function saveConfig(BaseImporter $import): bool + { + $isNewConfig = ! $import->uid; + + event($event = new ImportConfigSaving($import, $isNewConfig)); + + if (! $event->isValid) { + return false; + } + + $import = $event->import; + + if ($isNewConfig) { + $import->uid = Str::uuid7()->toString(); + } + + $configRecord = $this->_getImportConfigModel($import->uid); + + DB::beginTransaction(); + + try { + $configRecord->uid = $import->uid; + $configRecord->type = $import::class; + $configRecord->name = $import->name; + $configRecord->handle = $import->handle; + $configRecord->description = $import->description; + $settings = [ + 'file' => $import->file, + 'site' => $import->site->uid, + 'className' => $import->className, + 'transformer' => $import->transformer ? $import->transformer::class : null, + 'map' => $import->map, + 'matchCriteria' => $import->matchCriteria, + ]; + $configRecord->settings = $settings; + $configRecord->save(); + + DB::commit(); + } catch (Throwable $e) { + DB::rollBack(); + + throw $e; + } + + // invalidate caches + $this->configs = null; + + event(new ImportConfigSaved($import, $isNewConfig)); + + return true; + } + + public function deleteConfig(BaseImporter $import): void + { + $configRecord = $this->_getImportConfigModel($import->uid); + + if (! $configRecord->exists) { + return; + } + + DB::beginTransaction(); + + try { + DB::table(Table::IMPORT_CONFIGS)->softDelete($configRecord->id); + DB::commit(); + } catch (Throwable $e) { + DB::rollBack(); + throw $e; + } + + // invalidate caches + $this->configs = null; + } + + /** + * Returns an import config model for a given UID + */ + private function _getImportConfigModel(string $uid, bool $withTrashed = false): ImportConfigModel + { + return ImportConfigModel::withTrashed($withTrashed) + ->where('uid', $uid) + ->first() ?? new ImportConfigModel; + } + + private function _importConfigQuery(): Builder + { + return DB::table(Table::IMPORT_CONFIGS) + ->select([ + 'import_configs.type', + 'import_configs.name', + 'import_configs.handle', + 'import_configs.description', + 'import_configs.settings', + 'import_configs.uid', + ]) + ->orderBy('import_configs.name') + ->orderBy('import_configs.handle') + ->whereNull('import_configs.dateDeleted'); + } + + // //// runs ////// + public function getImportRuns(): LaravelCollection + { + if ($this->runs === null) { + $runs = $this->_importRunQuery()->get()->all(); + $runs = array_map(fn ($run) => new ImportRun($run), $runs); + + $this->runs = new LaravelCollection($runs)->keyBy('handle')->sortBy('name'); + } + + return $this->runs; + } + + public function getImportRunByHandle(?string $handle): ?ImportRun + { + if (is_null($handle)) { + return null; + } + + /** @var ImportRun|null */ + return $this->getImportRuns()->where('handle', $handle)->first(); + } + + public function getImportRunByUid(string $uid): ?ImportRun + { + /** @var ImportRun|null */ + return $this->getImportRuns()->where('uid', $uid)->first(); + } + + public function saveRun(ImportRun $run): bool + { + $isNewRun = ! $run->uid; + + event($event = new ImportRunSaving($run, $isNewRun)); + + if (! $event->isValid) { + return false; + } + + $run = $event->run; + + if (! $run->validate()) { + return false; + } + + if ($isNewRun) { + $run->uid = Str::uuid7()->toString(); + } + + $runRecord = $this->_getImportRunModel($run->uid); + + DB::beginTransaction(); + + try { + $runRecord->uid = $run->uid; + $runRecord->name = $run->name; + $runRecord->handle = $run->handle; + $runRecord->description = $run->description; + $runRecord->steps = $run->steps; + + $runRecord->save(); + + DB::commit(); + } catch (Throwable $e) { + DB::rollBack(); + + throw $e; + } + + // invalidate caches + $this->runs = null; + + event(new ImportRunSaved($run, $isNewRun)); + + return true; + } + + public function deleteRun(ImportRun $run): void + { + $runRecord = $this->_getImportRunModel($run->uid); + + if (! $runRecord->exists) { + return; + } + + DB::beginTransaction(); + + try { + DB::table(Table::IMPORT_RUNS)->softDelete($runRecord->id); + DB::commit(); + } catch (Throwable $e) { + DB::rollBack(); + throw $e; + } + + // invalidate caches + $this->runs = null; + } + + /** + * Returns an import config model for a given UID + */ + private function _getImportRunModel(string $uid, bool $withTrashed = false): ImportRunModel + { + return ImportRunModel::withTrashed($withTrashed) + ->where('uid', $uid) + ->first() ?? new ImportRunModel; + } + + private function _importRunQuery(): Builder + { + return DB::table(Table::IMPORT_RUNS) + ->select([ + 'import_runs.name', + 'import_runs.handle', + 'import_runs.description', + 'import_runs.steps', + 'import_runs.uid', + ]) + ->orderBy('import_runs.name') + ->orderBy('import_runs.handle') + ->whereNull('import_runs.dateDeleted'); + } + + // ///// import ////// + public function dispatchImport(ImportRun $run): bool + { + $steps = []; + + // for each step in the $run + foreach ($run->steps as $key => $step) { + $config = $this->getConfigByUid($step['config']) ?? $this->getConfigByHandle($step['config']); + $file = $config->file ?? $step['file']; + $filePath = BaseImporter::resolvedFilePath($file); + + // name for this batch of jobs + $steps[$key]['name'] = $config->name; + $steps[$key]['job'] = new ImportJob($step, $filePath, 0); + + } + + event($event = new ImportRunDispatching($steps, $run)); + + if (! $event->isValid) { + return false; + } + + $steps = $event->steps; + $run = $event->run; + + // TODO: think about scheduling batch pruning + + // we need to go through a single job because we want to name our chain + dispatch(new ImportPipeline($steps, $run)); + + event(new ImportRunDispatched($steps, $run)); + + return true; + } + + public function importItem(BaseImporter $config, array $data): void + { + event($event = new DataImporting($config, $data)); + + if (! $event->isValid) { + return; + } + + $data = $event->data; + + // figure out if we're adding or updating + $element = $this->getElement($config, $data); + + $item = $this->processData($config, $data, $element); + + $attributeHandles = $element->attributes(); + $fieldHandles = array_diff(array_keys($item), $attributeHandles); + $attributes = array_filter(array_filter($item, fn ($value, $key) => in_array($key, $attributeHandles), ARRAY_FILTER_USE_BOTH)); + $fields = array_filter($item, fn ($value, $key) => in_array($key, $fieldHandles), ARRAY_FILTER_USE_BOTH); + + $element->setAttributesFromRequest($attributes); + $element->setFieldValues($fields); + + \Craft::$app->getElements()->saveElement($element); + + event(new DataImported($config, $data)); + } + + private function getElement(BaseImporter $config, array $data): ElementInterface + { + // figure out if we're adding or editing + $element = new $config->className; + + // if null then return a brand new ElementInterface object with just the siteId set to the selected value + if ($config->matchCriteria === null) { + $element->siteId = $config->site->id; + + return $element; + } + + $query = $element::find(); + if (is_array($config->matchCriteria)) { + $criteria = []; + foreach ($config->matchCriteria as $key => $value) { + if (array_key_exists((string) $value, $data)) { + $criteria[$key] = $data[$value]; + } + } + + if (empty($criteria)) { + $element->siteId = $config->site?->id; + + return $element; + } + + \Craft::configure($query, $criteria); + // force the selected siteId + $query->siteId = $config->site?->id; + + return $query->one() ?? $element; + } + + return $element; + } + + public function getData(string $filePath): array + { + error_clear_last(); + $rawData = @file_get_contents($filePath); + $error = error_get_last(); + if ($error) { + throw new Exception($error['message']); + } + + if (! $rawData) { + throw new Exception('Unable to parse data.'); + } + + // process raw data based on the file type it came from + $data = $this->formatData($filePath, $rawData); + + if ($data === null || $data['success'] === false) { + throw new Exception($data['error'] ?? 'Unable to parse data.'); + } + + return $data['data']; + } + + private function formatData(string $filePath, string $rawData): ?array + { + $extension = File::extension($filePath); + $dataTypes = $this->getAllDataTypes(); + + if (! $dataTypes[$extension]) { + throw new Exception('Unsupported data type: '.$extension); + } + + try { + $data = $dataTypes[$extension]::format($rawData); + } catch (Throwable $e) { + Log::error($e->getMessage()); + + return null; + } + + return $data; + } + + private function processData(BaseImporter $config, array $data, mixed $element): array + { + // turn that data into a fractal collection + $resource = new Item($data, $config->transformer); + $resource->setMeta(['config' => $config, 'element' => $element]); + + // Load Fractal + $fractalManager = new Manager; + $fractalManager->setSerializer(new DataArraySerializer); + + // // Parse includes/excludes + // $fractalManager->parseIncludes($includes); + // $fractalManager->parseExcludes($excludes); + + $data = $fractalManager->createData($resource); + + // todo: ->toArray() freaks out if the transformer is null; not sure if that's expected or not + if ($config->transformer === null) { + return $data->getResource()->getData(); + } + + $data = $data->toArray(); + + return $data['data']; + } +} diff --git a/src/Import/ImportServiceProvider.php b/src/Import/ImportServiceProvider.php new file mode 100644 index 00000000000..86e2438aea6 --- /dev/null +++ b/src/Import/ImportServiceProvider.php @@ -0,0 +1,18 @@ +commands([ + Element::class, + ]); + } +} diff --git a/src/Import/Importers/BaseImporter.php b/src/Import/Importers/BaseImporter.php new file mode 100644 index 00000000000..22c3149af32 --- /dev/null +++ b/src/Import/Importers/BaseImporter.php @@ -0,0 +1,292 @@ + the column that should be used to match incoming data against existing elements; + * by default, [id => id] is used, meaning elements are matched on their ID, and we expect an 'id' key in the provided data + * null => don't match against existing elements; import all incoming data + * Closure => + */ + public protected(set) Closure|array|null $matchCriteria = null; + + public ?string $uid = null; + + public bool $editable = false; + + public function __construct(?array $config = null) + { + if (! empty($config)) { + $this->uid = $config['uid'] ?? null; + $this->editable = $config['editable'] ?? false; + } + } + + public static function displayName(): string + { + return t('Base Importer'); + } + + public function isEditable(): bool + { + return isset($this->uid); + } + + public function getSettingsHtml(): string + { + return $this->settingsHtml(false); + } + + public function getReadOnlySettingsHtml(): string + { + return $this->settingsHtml(true); + } + + protected function settingsHtml(bool $readOnly): string + { + return template('import/_importer-types/base-importer', [ + 'readOnly' => $readOnly, + 'import' => $this, + ]); + } + + public function isElementImport() + { + return is_subclass_of($this->className, ElementInterface::class); + } + + public static function getRules(): array + { + return [ + 'name' => [ + 'required', + 'string', + 'max:255', + ], + 'handle' => [ + 'required', + 'string', + 'max:255', + new HandleRule(['id', 'dateCreated', 'dateUpdated', 'uid', 'title']), + function ($attribute, $value, Closure $fail, Validator $validator) { + $found = Import::getConfigByHandle($value, true); + if ($found !== null && $found->uid !== $validator->getValue('uid')) { + $fail(t('{attribute} "{value}" has already been taken.', [ + 'attribute' => $attribute, + 'value' => $value, + ])); + } + }, + ], + 'settings.file' => [ + 'required', + 'string', + 'max:255', + fn ($attribute, $value, Closure $fail, Validator $validator) => self::validateFile($value, $attribute, $fail, $validator), + ], + 'settings.className' => [ + 'required', + 'string', + ], + // 'settings.transformer' => [ + // 'nullable', + // 'string', + // 'max:255', + // fn ($attribute, $value, Closure $fail, Validator $validator) => self::validateTransformer($value, $attribute, $validator), + // ], + 'settings.map' => ['array'], + ]; + } + + public function name(string $name): self + { + $this->name = $name; + + return $this; + } + + public function handle(?string $handle): self + { + $this->handle = $handle; + + return $this; + } + + public function description(?string $description): self + { + $this->description = $description; + + return $this; + } + + public function className(string $className): self + { + $this->className = $className; + + return $this; + } + + public function file(?string $file): self + { + $this->file = $file; + + return $this; + } + + public function transformer(?string $transformer): self + { + $this->transformer = $this->normalizeTransformer($transformer); + + return $this; + } + + public function map(array $map): self + { + $this->map = $map; + + return $this; + } + + public function matchCriteria(array $matchCriteria): self + { + $this->matchCriteria = $matchCriteria; + + return $this; + } + + public static function validateFile(mixed $value, string $attribute, Closure $fail, Validator $validator, ?string $attributeForMessage = null): bool + { + if (empty($value)) { + $fail(/* $attributeForMessage ?? */ $attribute, t('File must be provided.')); + // $validator->errors()->add($attributeForMessage ?? $attribute, t('File must be provided.')); + + return false; + } + + $filePath = self::resolvedFilePath($value); + if (! file_exists($filePath)) { + $fail($attribute, t('File “{filePath}” does not exist.', [ + 'filePath' => $filePath, + ])); + + return false; + } + + $file = new File($filePath); + $dataTypes = array_unique(array_filter(array_keys(Import::getAllDataTypes()))); + + // validate file type (e.g. csv, json, xml) + $newValidator = ValidatorFacade::make([ + 'file' => $file, + ], [ + 'file' => ['mimes:'.implode(',', $dataTypes)], + ]); + + if ($newValidator->fails()) { + $fail($attribute, t('Only files with these MIME types are allowed: {mimeTypes}.', [ + 'mimeTypes' => implode(', ', $dataTypes), + ])); + + return false; + } + + return true; + } + + public static function resolvedFilePath(?string $file): ?string + { + if (is_null($file)) { + return null; + } + + return str_starts_with($file, '@root/') ? Aliases::get($file) : Aliases::get('@root/'.$file); + } + + public function normalizeTransformer(string|null|TransformerAbstract $transformer): TransformerAbstract|Closure|null + { + if ($transformer instanceof TransformerAbstract) { + return $transformer; + } + + if (empty($transformer)) { + return null; + } + + if (preg_match('/^fn\s*\(\s*(?:\$(\w+)\s*)?\)\s*=>\s*(.+)/', $transformer, $match)) { + $var = $match[1]; + $php = sprintf('return %s;', Str::removeLeft(rtrim($match[2], ';'), 'return ')); + + return function (ElementInterface $element) use ($var, $php) { + if ($var) { + ${$var} = $element; + } + + return eval($php); + }; + } + + if (class_exists($transformer) && (new $transformer) instanceof TransformerAbstract) { + return new $transformer; + } + + return null; + } + + public static function validateTransformer(mixed $value, string $attribute, Closure $fail, Validator $validator): bool + { + // if it's empty - that's fine (we'll probably use the default ElementTransformer) + if (empty($value)) { + return true; + } + + // if it's an arrow function - ok + if (preg_match('/^fn\s*\(\s*(?:\$(\w+)\s*)?\)\s*=>\s*(.+)/', (string) $value)) { + return true; + } + + // if it's a string - the assumption is that it's a class name with namespace (just like with elementType) + // and we need to check if it exists and is compatible + if (class_exists($value) && (new $value) instanceof TransformerAbstract) { + return true; + } + + // no other options are valid + $fail($attribute, t('Transformer has to be empty, a valid class or a closure.')); + + return false; + } +} diff --git a/src/Import/Importers/ElementImporter.php b/src/Import/Importers/ElementImporter.php new file mode 100644 index 00000000000..caf79fd7e84 --- /dev/null +++ b/src/Import/Importers/ElementImporter.php @@ -0,0 +1,175 @@ +matchCriteria = ['id' => 'id']; + } + + #[Override] + public static function displayName(): string + { + return t('Element Importer'); + } + + #[Override] + protected function settingsHtml(bool $readOnly): string + { + $allElementTypes = Craft::$app->getElements()->getAllElementTypes(); + $availableElementTypes = array_map(fn ($type) => [ + 'label' => $type::displayName(), + 'value' => $type, + ], $allElementTypes); + + $defaultElementType = null; + if (in_array(Entry::class, $allElementTypes, true)) { + $defaultElementType = Entry::class; + } + + return template('import/_importer-types/element-importer', [ + 'readOnly' => $readOnly, + 'import' => $this, + 'availableElementTypes' => $availableElementTypes, + 'defaultElementType' => $defaultElementType, + 'availableSites' => Sites::getEditableSites() + ->map(fn ($item) => ['label' => $item->name, 'value' => $item->handle]) + ->all(), + ]); + } + + // #[Override] + // public function defaultName() + // { + // return t('Element Import'); + // } + + #[Override] + public static function getRules(): array + { + return array_merge(parent::getRules(), [ + 'settings.className' => fn ($attribute, $value, Closure $fail, Validator $validator) => self::validateElementType($value, $attribute, $fail, $validator), + 'settings.site' => [ + 'required', + 'string', + 'max:255', + fn ($attribute, $value, Closure $fail, Validator $validator) => self::validateSite($value, $attribute, $fail, $validator), + ], + ]); + } + + public static function validateElementType(mixed $value, string $attribute, Closure $fail, Validator $validator): bool + { + if (empty($value)) { + $fail($attribute, t('Element type must be provided.')); + + return false; + } + + $allElementTypes = (new Elements)->getAllElementTypes(); + if (! in_array($value, $allElementTypes)) { + $fail($attribute, t('Element type “{elementType}” is not a valid element type.', [ + 'elementType' => $value, + ])); + + return false; + } + + return true; + } + + public static function validateSite(mixed $value, string $attribute, Closure $fail, Validator $validator): bool + { + if (empty($value)) { + $fail($attribute, t('Site must be provided.')); + + return false; + } + + $allSites = Sites::getAllSites()->pluck('handle')->all(); + if (! in_array($value, $allSites)) { + $fail($attribute, t('“{site}” is not a valid site handle.', [ + 'site' => $value, + ])); + + return false; + } + + return true; + } + + public static function create(): self + { + return new self; + } + + #[Override] + public function className(string $className): self + { + $allElements = (new Elements)->getAllElementTypes(); + if (! in_array($className, $allElements)) { + throw new InvalidArgumentException("Class '{$className}' is not a valid element type."); + } + + $this->className = $className; + + return $this; + } + + public function site(string|int|Site|null $site): self + { + if ($site instanceof Site) { + $this->site = $site; + } elseif ($site === null) { + $this->site = Sites::getPrimarySite(); + } elseif (is_numeric($site)) { + $this->site = Sites::getAllSites()->firstWhere('id', $site); + } elseif (is_string($site)) { + $this->site = Sites::getAllSites()->firstWhere('handle', $site); + if ($this->site === null) { + $this->site = Sites::getAllSites()->firstWhere('uid', $site); + } + } + + return $this; + } + + #[Override] + public function transformer(?string $transformer): self + { + if ($transformer === null) { + // use the default for this element type + $transformer = $this->className::getDefaultTransformer(); + } + + return parent::transformer($transformer); + } + + // + // public function map(array $map): self + // { + // $this->map = $map; + // + // return $this; + // } +} diff --git a/src/Import/Jobs/Import.php b/src/Import/Jobs/Import.php new file mode 100644 index 00000000000..68f91d86d1d --- /dev/null +++ b/src/Import/Jobs/Import.php @@ -0,0 +1,78 @@ +batch()->cancelled()) { + return; + } + + $importService = app(ImportService::class); + $config = $importService->getConfigByUid($this->step['config']) ?? $importService->getConfigByHandle($this->step['config']); + + // get all the data + $allData = $importService->getData($this->filePath); + // discard the part at the start that was already processed + $data = array_slice($allData, $this->start); + // count how many items we have to process + $dataCount = count($data); + // figure out our batch limit + $batchLimit = $this->getBatchSize($this->step); + + // if batch limit is 0, it means this step's batch size was set to zero to disable batching of this step + // so we want to go through all the data in one go + if ($batchLimit === 0) { + $batchLimit = $dataCount; + } + + for ($i = 0; $i < $batchLimit; $i++) { + // if we have less data than the limit, break + if (! isset($data[$i])) { + break; + } + + // import data + $importService->importItem($config, $data[$i]); + } + + // if there's any data items left - add another job to the batch + if ($dataCount - $batchLimit > 0) { + $this->batch()->add(new Import($this->step, $this->filePath, ($this->start + $batchLimit))); + } + } + + private function getBatchSize(array $step): int + { + // if batch size was left empty, it was cast to a null, and we should use the default batch size + if ($step['batchSize'] === null) { + return $this->defaultBatchSize; + } + + // otherwise, return the number specified in the step + return (int) $step['batchSize']; + } +} diff --git a/src/Import/Jobs/ImportPipeline.php b/src/Import/Jobs/ImportPipeline.php new file mode 100644 index 00000000000..578be8ad6d5 --- /dev/null +++ b/src/Import/Jobs/ImportPipeline.php @@ -0,0 +1,45 @@ +steps as $step) { + $steps[] = Bus::batch([$step['job']])->name($step['name'] ?? 'Importing step data')->allowFailures(); + } + + Bus::chain($steps)->dispatch(); + } + + #[Override] + protected function defaultDescription(): string + { + return t("Importing “{$this->run->name}” data"); + } +} diff --git a/src/Import/Models/ImportConfig.php b/src/Import/Models/ImportConfig.php new file mode 100644 index 00000000000..05e0d537e5e --- /dev/null +++ b/src/Import/Models/ImportConfig.php @@ -0,0 +1,28 @@ + 'json', + 'elementImport' => 'boolean', + ]; + } +} diff --git a/src/Import/Models/ImportRun.php b/src/Import/Models/ImportRun.php new file mode 100644 index 00000000000..c0cb0e4a912 --- /dev/null +++ b/src/Import/Models/ImportRun.php @@ -0,0 +1,28 @@ + 'json', + ]; + } +} diff --git a/src/Import/Transformers/ElementTransformer.php b/src/Import/Transformers/ElementTransformer.php new file mode 100644 index 00000000000..890574c9dec --- /dev/null +++ b/src/Import/Transformers/ElementTransformer.php @@ -0,0 +1,86 @@ +getCurrentScope()->getResource()->getMeta()['element'] ?? null; + + // automatically include all Importable properties (e.g. sectionId, typeId for Entry); + if ($this->props === null) { + $config = $this->getCurrentScope()->getResource()->getMeta()['config']; + $class = new \ReflectionClass($config->className); + $properties = $class->getProperties(); + $properties = array_values(array_filter($properties, fn ($property) => ! empty($property->getAttributes(Importable::class)))); + $this->props = array_map(fn ($property) => [ + 'name' => $property->getAttributes(Importable::class)[0]->getArguments()[0], + 'defaultValue' => $property->getDefaultValue(), + ], $properties); + } + + $array = []; + foreach ($this->props as $prop) { + $array[$prop['name']] = $this->normalizePropertyValue($item, $prop); + } + + // // Get the serialized custom field values + // $fields = $element->getSerializedFieldValues(); + // + // // Get the element attributes that aren't custom fields + // /** @var Element $element */ + // $attributes = array_diff($element->attributes(), array_keys($fields)); + // + // // Return the element as an array merged with its serialized custom field values + // return array_merge($element->toArray($attributes), $fields); + + $fieldLayout = $element->getFieldLayout(); + if (! $fieldLayout) { + if (isset($array['typeId'])) { + $entryType = EntryTypes::getEntryTypeById($array['typeId']); + if ($entryType) { + $fieldLayout = $entryType->getFieldLayout(); + } + } + } + + if ($fieldLayout) { + $fieldHandles = array_filter( + array_map( + fn ($fieldLayoutElement) => $fieldLayoutElement->attribute(), + $fieldLayout->getAllElements() + ) + ); + + foreach ($fieldHandles as $fieldHandle) { + $array[$fieldHandle] = $item[$fieldHandle] ?? null; + } + } + + return $array; + } + + private function normalizePropertyValue(mixed $item, array $prop): mixed + { + $rawValue = $item[$prop['name']] ?? null; + + if ($rawValue !== null) { + if (method_exists($this, 'normalize'.ucfirst((string) $prop['name']))) { + return $this::{'normalize'.ucfirst((string) $prop['name'])}($rawValue); + } + + return $rawValue; + } + + return $prop['defaultValue'] ?? null; + } +} diff --git a/src/Import/Transformers/EntryTransformer.php b/src/Import/Transformers/EntryTransformer.php new file mode 100644 index 00000000000..63ed1d2f425 --- /dev/null +++ b/src/Import/Transformers/EntryTransformer.php @@ -0,0 +1,53 @@ +id; + } + } + + return null; + } + + protected function normalizeTypeId($value): ?int + { + if ($value === null) { + return null; + } + + if (is_int($value)) { + // $section = Sections::getSectionById($value); + return $value; + } + + if (is_string($value)) { + $type = EntryTypes::getEntryTypeByHandle($value); + if ($type) { + return $type->id; + } + } + + return null; + } +} diff --git a/src/Providers/CraftServiceProvider.php b/src/Providers/CraftServiceProvider.php index af3f27bc088..10ed545fae8 100644 --- a/src/Providers/CraftServiceProvider.php +++ b/src/Providers/CraftServiceProvider.php @@ -16,6 +16,7 @@ use CraftCms\Cms\Field\FieldsServiceProvider; use CraftCms\Cms\FieldLayout\FieldLayoutServiceProvider; use CraftCms\Cms\Gql\GqlServiceProvider; +use CraftCms\Cms\Import\ImportServiceProvider; use CraftCms\Cms\License\LicenseServiceProvider; use CraftCms\Cms\Plugin\PluginServiceProvider; use CraftCms\Cms\ProjectConfig\ProjectConfigServiceProvider; @@ -62,5 +63,6 @@ class CraftServiceProvider extends AggregateServiceProvider EntryServiceProvider::class, StructureServiceProvider::class, QueueServiceProvider::class, + ImportServiceProvider::class, ]; } diff --git a/src/Support/Attributes/Importable.php b/src/Support/Attributes/Importable.php new file mode 100644 index 00000000000..f356159c014 --- /dev/null +++ b/src/Support/Attributes/Importable.php @@ -0,0 +1,15 @@ +entryPermissions($this->allPermissions); $this->volumePermissions($this->allPermissions); $this->utilityPermissions($this->allPermissions); + $this->importPermissions($this->allPermissions); event($event = new RegisterUserPermissions($this->allPermissions)); @@ -743,6 +744,50 @@ private function utilityPermissions(Collection $permissions): void )); } + private function importPermissions(Collection $permissions): void + { + $permissions->add(new PermissionGroup( + heading: t('Import'), + permissions: collect([ + new Permission( + key: 'accessImports', + label: t('Access imports'), + nested: collect([ + new Permission( + key: 'viewImportConfigs', + label: t('View import configs'), + nested: collect([ + new Permission( + key: 'editImportConfigs', + label: t('Edit import configs'), + info: t('Users with this permission can potentially create import config for items or sites that they don’t have permissions to. Take care when granting this permission.'), + ), + new Permission(key: 'deleteImportConfigs', label: t('Delete import configs')), + ]) + ), + new Permission( + key: 'viewImportRuns', + label: t('View import runs'), + nested: collect([ + new Permission( + key: 'editImportRuns', + label: t('Edit import runs'), + info: t('Users with this permission can create import runs that imports items that user doesn’t have permissions to. Take care when granting this permission.'), + ), + new Permission(key: 'deleteImportRuns', label: t('Delete import runs')), + new Permission( + key: 'triggerImportRuns', + label: t('Trigger import runs'), + warning: t('Users with this permission can manipulate content they don’t have access to. Take care when granting this permission.'), + ), + ]) + ), + ]) + ), + ]) + )); + } + /** * Filters out any permissions that aren't assignable by the current user. * diff --git a/yii2-adapter/legacy/web/assets/cp/dist/cp.js b/yii2-adapter/legacy/web/assets/cp/dist/cp.js index 54de43fece9..5d683f674dc 100644 --- a/yii2-adapter/legacy/web/assets/cp/dist/cp.js +++ b/yii2-adapter/legacy/web/assets/cp/dist/cp.js @@ -1,2 +1,2 @@ -(function(){var __webpack_modules__={333:function(){Craft.CpModal=Garnish.Modal.extend({action:null,namespace:null,showingLoadSpinner:!1,$loadSpinner:null,$container:null,$body:null,$content:null,$sidebar:null,$footer:null,$cancelBtn:null,$saveBtn:null,showingSidebar:!1,cancelToken:null,ignoreFailedRequest:!1,fieldsWithErrors:null,init:function(t,e){this.action=t,this.setSettings(e,Craft.CpModal.defaults),this.fieldsWithErrors=[],this.$body=$("
",{class:"cpmodal-body"}),this.$content=$("
",{class:"cpmodal-content"}).appendTo(this.$body),this.$footer=$("
",{class:"cpmodal-footer hidden"}),$("
",{class:"flex-grow"}).appendTo(this.$footer);const i=$("
",{class:"flex flex-nowrap"}).appendTo(this.$footer);this.$loadSpinner=$("
",{class:"spinner",title:Craft.t("app","Loading"),"aria-label":Craft.t("app","Loading")}).prependTo(i),this.$cancelBtn=$("\n
\n
\n
\n
\n`).prependTo(this.$container),this.menu=this.$chip.find(".action-btn").disclosureMenu().data("disclosureMenu"),this.initChip()},createTextInput:function(t){this.reset(),this.$textInput=Craft.ui.createTextInput(this.settings.inputAttributes).attr("name",this.settings.inputName).val(t).prependTo(this.$container),this.initTextInput(),this.$textInput.trigger("input")},switchToTextInput:function(){const t=this.removeFirstPrefix(this.$hiddenInput.val());this.createTextInput(t)},initTextInput:function(){this.addListener(this.$textInput,"input",()=>{const t=this.normalize(this.$textInput.val());this.$hiddenInput.val(t),this.field.updateLabel(this.removePrefix(t))}),this.addListener(this.$textInput,"blur",()=>{this.maybeSwitchToChip()}),this.addListener(this.$textInput,"keydown",t=>{t.keyCode===Garnish.ESC_KEY&&this.maybeSwitchToChip()&&(t.stopPropagation(),this.$chip.find("a").focus())})},normalize:function(t){if(!(t=Craft.trim(t)))return"";const e=this.ensurePrefix(t);return this.validate(e)?e:t},validate:function(t){return!!(t=s.toASCII(t)).match(new RegExp(this.settings.pattern,"i"))},maybeSwitchToChip:function(){if(!this.$textInput?.length)return;const t=this.normalize(this.$textInput.val());return!(!t||!this.validate(t)||(this.createChip(t),0))},initChip:function(){const t=this.menu.addItem({label:Craft.t("app","View in a new tab"),icon:async()=>await Craft.ui.icon("share")}),e=this.menu.addItem({label:Craft.t("app","Edit"),icon:async()=>await Craft.ui.icon("pencil")}),i=this.menu.addItem({label:Craft.t("app","Copy URL"),icon:async()=>await Craft.ui.icon("link")});this.menu.addHr(),this.menu.addGroup();const s=this.menu.addItem({label:"Remove",icon:async()=>await Craft.ui.icon("xmark"),destructive:!0});this.addListener(i,"activate",()=>{Craft.ui.createCopyTextPrompt({label:"Full URL",value:this.$hiddenInput.val().replace(/ /g,"+")})}),this.addListener(t,"activate",()=>{window.open(this.$chip.find("a").attr("href"))}),this.addListener(e,"activate",()=>{this.switchToTextInput(),this.$textInput.focus()}),this.addListener(s,"activate",()=>{this.createTextInput(""),this.$textInput.focus()})},reset:function(){this.$textInput?.remove(),this.$chip?.remove(),this.menu?.destroy(),this.$textInput=this.$chip=this.menu=null}},{defaults:{prefixes:null,pattern:null,textInputAttributes:{}}})},1761:function(){Craft.ThumbsElementIndexView=Craft.BaseElementIndexView.extend({getElementContainer:function(){return this.$container.children("ul")}})},2034:function(){Craft.BaseElementSelectorModal=Garnish.Modal.extend({elementType:null,elementIndex:null,supportSidebarToggleView:!1,$body:null,$content:null,$footer:null,$selectBtn:null,$sidebar:null,$sources:null,$sourceToggles:null,$sidebarToggleBtn:null,$sidebarCloseBtn:null,$mainHeading:null,$main:null,$search:null,$elements:null,$tbody:null,$primaryButtons:null,$secondaryButtons:null,$cancelBtn:null,init:function(t,e){this.elementType=t,this.setSettings(e,Craft.BaseElementSelectorModal.defaults);const i="elementSelectorModalHeading-"+Math.floor(1e6*Math.random()),s=$("
",{class:"modal elementselectormodal","aria-labelledby":i}).appendTo(Garnish.$bod),n=$("
",{class:this.settings.showTitle?"header":"visually-hidden"}).appendTo(s);$("

",{id:i,text:this.settings.modalTitle}).appendTo(n);const a=$("
",{class:"body"}).append($("
",{class:"spinner big"})).appendTo(s);this.$footer=$("
",{class:"footer"}).appendTo(s),this.settings.fullscreen&&(s.addClass("fullscreen"),this.settings.minGutter=0),this.base(s,this.settings),this.$secondaryButtons=$('
').appendTo(this.$footer),this.$primaryButtons=$('
').appendTo(this.$footer),this.$cancelBtn=$("
").appendTo(this.$promptChoices).find("input");this.addListener(l,"click",function(){r.removeClass("disabled")})}this.addListener(r,"activate",function(t){var e=$(t.currentTarget).parents(".modal").find("input[name=promptAction]:checked").val(),i=this.$promptApplyToRemainingCheckbox.prop("checked");this._selectPromptChoice(e,i)}),this.addListener(a,"activate",function(){var t=this.$promptApplyToRemainingCheckbox.prop("checked");this._selectPromptChoice("cancel",t)}),s&&(this.$promptApplyToRemainingContainer.show(),this.$promptApplyToRemainingLabel.html(" "+Craft.t("app","Apply this to the {number} remaining conflicts?",{number:s}))),this.modal.show(),this.modal.removeListener(Garnish.Modal.$shade,"click"),this.addListener(Garnish.Modal.$shade,"click","_cancelPrompt")},_selectPromptChoice:function(t,e){this.$prompt.fadeOut("fast",()=>{this.modal.hide(),this._promptCallback(t,e)})},_cancelPrompt:function(){this._selectPromptChoice("cancel",!0)}})},3374:function(){Craft.BaseUploader=Garnish.Base.extend({allowedKinds:null,$element:null,$fileInput:null,settings:null,fsType:null,formData:{},events:{},_rejectedFiles:{},_extensionList:null,_inProgressCounter:0,init:function(t,e){this._rejectedFiles={size:[],type:[],limit:[]},this.$element=t,this.settings=$.extend({},Craft.BaseUploader.defaults,e),this.formData=this.settings.formData,this.$fileInput=this.settings.fileInput||t,this.events=this.settings.events,this.settings.url||(this.settings.url=this.settings.replace?Craft.getActionUrl(this.settings.replaceAction):Craft.getActionUrl(this.settings.createAction)),this.settings.allowedKinds&&this.settings.allowedKinds.length&&("string"==typeof this.settings.allowedKinds&&(this.settings.allowedKinds=[this.settings.allowedKinds]),this.allowedKinds=this.settings.allowedKinds,delete this.settings.allowedKinds)},setParams:function(t){void 0!==Craft.csrfTokenName&&void 0!==Craft.csrfTokenValue&&(t[Craft.csrfTokenName]=Craft.csrfTokenValue),this.formData=t},getInProgress:function(){return this._inProgressCounter},isLastUpload:function(){return this.getInProgress()<2},processErrorMessages:function(){var t;this._rejectedFiles.type.length&&(t=1===this._rejectedFiles.type.length?"The file {files} could not be uploaded. The allowed file kinds are: {kinds}.":"The files {files} could not be uploaded. The allowed file kinds are: {kinds}.",t=Craft.t("app",t,{files:this._rejectedFiles.type.join(", "),kinds:this.allowedKinds.join(", ")}),this._rejectedFiles.type=[],Craft.cp.displayError(t)),this._rejectedFiles.size.length&&(t=1===this._rejectedFiles.size.length?"The file {files} could not be uploaded, because it exceeds the maximum upload size of {size}.":"The files {files} could not be uploaded, because they exceeded the maximum upload size of {size}.",t=Craft.t("app",t,{files:this._rejectedFiles.size.join(", "),size:this.humanFileSize(this.settings.maxFileSize)}),this._rejectedFiles.size=[],Craft.cp.displayError(t)),this._rejectedFiles.limit.length&&(t=1===this._rejectedFiles.limit.length?"The file {files} could not be uploaded, because the field limit has been reached.":"The files {files} could not be uploaded, because the field limit has been reached.",t=Craft.t("app",t,{files:this._rejectedFiles.limit.join(", ")}),this._rejectedFiles.limit=[],Craft.cp.displayError(t))},humanFileSize:function(t){var e=1024;if(t=e);return t.toFixed(1)+" "+["kB","MB","GB","TB","PB","EB","ZB","YB"][i]},_createExtensionList:function(){this._extensionList=[];for(var t=0;t