diff --git a/app/models/organization.rb b/app/models/organization.rb index 7b97fb755e..28c894ae03 100644 --- a/app/models/organization.rb +++ b/app/models/organization.rb @@ -84,7 +84,7 @@ def communications_email validates :organization_type_other, length: { maximum: 255 } validates :tax_id, length: { maximum: 255 } validates :website_url, length: { maximum: 255 } - validates :mission_vision_values, length: { maximum: 255 } + validates :mission_vision_values, length: { maximum: 16_000 } # Admin bypass stays on Current.user until #2833 moves it to a controller-set flag (models never call policies). validate :affiliation_dates_locked, if: -> { affiliations.any? && !Current.user&.super_user? } validate :parent_is_not_self_or_descendant, if: :parent_id_changed? diff --git a/app/services/organization_services/import_orgs.rb b/app/services/organization_services/import_orgs.rb new file mode 100644 index 0000000000..be3902324c --- /dev/null +++ b/app/services/organization_services/import_orgs.rb @@ -0,0 +1,207 @@ +module OrganizationServices + class ImportOrgs + include FilemakerValues + + OUTCOMES = %i[created linked_project already_imported skipped failed].freeze + BATCH_SIZE = 500 + TYPE_PATTERNS = { + /501|non.?profit/i => "501c3/nonprofit", + /for.?profit|\bLLC\b/i => "For-profit", + /govt|government/i => "Government agency", + /\Aother\z/i => Organization::ORGANIZATION_TYPE_OTHER + }.freeze + SCHOOL_PATTERN = /school/i + + attr_reader :results, :multi_org, :condensed, :suffixed, :retired, + :unmatched_statuses, :unmatched_types, :unmatched_project_codes + + def unmatched_staff = @staff.unmatched + + def initialize(progress: nil) + @progress = progress + @results = OUTCOMES.index_with { [] } + @multi_org = [] + @condensed = [] + @suffixed = [] + @retired = [] + @unmatched_statuses = Hash.new(0) + @unmatched_types = Hash.new(0) + @placed_by_org = Hash.new { |hash, key| hash[key] = [] } + end + + def call + load_indexes + read = 0 + FmOrganization.order(:fm_id).pluck(:id).each_slice(BATCH_SIZE) do |ids| + ActiveRecord::Base.transaction(requires_new: true) do + FmOrganization.where(id: ids).order(:fm_id).each { |row| import_row(row) } + end + read += ids.size + @progress&.call(read) + end + @unmatched_project_codes = project_codes.reject { |code| @codes.include?(code) } + @condensed = @placed_by_org.select { |_id, codes| codes.size > 1 } + .map { |id, codes| "##{id} #{@org_by_id[id]&.name}: #{codes.join(', ')}" } + self + end + + private + + def load_indexes + @codes = Set.new + @holders = Hash.new { |hash, key| hash[key] = [] } + @org_by_id = {} + Organization.find_each do |organization| + @org_by_id[organization.id] = organization + organization.filemaker_codes.each do |code| + @codes << code + @holders[code] << organization + end + end + @projects_by_org = FmProject.all.group_by { |project| project.data["OrgID"].to_s } + @statuses = OrganizationStatus.all.index_by { |status| status.name.downcase } + @staff = StaffTranslation.new + @import_user_id = Setting.import_user&.id + end + + def import_row(fm_organization) + data = fm_organization.data + code = fm_organization.fm_id + name = clip(data["OrgName"].to_s.squish) + return record(:skipped, "#{code}: no name") if name.blank? + return record(:already_imported, "#{code} #{name}") if @codes.include?(code) + + note_retired(code, name) + projects = @projects_by_org[code] || [] + targets = projects.flat_map { |project| @holders[project.fm_id] }.uniq + return attach_to_targets(code, name, targets) if targets.any? + + create_org(code, name, data, projects) + end + + def attach_to_targets(code, name, targets) + failed = targets.reject do |organization| + organization.filemaker_code = Organization.join_filemaker_codes(organization.filemaker_code, code) + organization.save + end + unless failed.empty? + return record(:failed, "#{code} #{name}: could not add the code to #{failed.map { |organization| "##{organization.id}" }.join(', ')}") + end + + targets.each do |organization| + @codes << code + @holders[code] << organization unless @holders[code].include?(organization) + @placed_by_org[organization.id] << code + end + if targets.size > 1 + @multi_org << "#{code} #{name} → #{targets.map { |organization| "##{organization.id} #{organization.name}" }.join('; ')}" + end + record(:linked_project, "#{code} #{name} → #{targets.map { |organization| "##{organization.id}" }.join(', ')}") + end + + def create_org(code, name, data, projects) + organization = Organization.new(name: unique_name(name, code)) + fill(organization, attributes(data)) + apply_status(organization, data["Status"]) + stamp(organization, data) + organization.filemaker_code = Organization.join_filemaker_codes(code, *claimable_project_codes(projects)) + return failed(organization, "#{code} #{name}") unless organization.save + + index(organization) + record(:created, "#{code} #{name} → ##{organization.id}") + end + + def claimable_project_codes(projects) + projects.one? ? [ projects.first.fm_id ] : [] + end + + def attributes(data) + { + website_url: clip(data["Website"]), + description: data["Description"].to_s.strip.presence, + mission_vision_values: data["Mission"].to_s.strip.presence, + **organization_type(data["OrgType"]), + **archive_timestamps(data) + } + end + + def archive_timestamps(data) + created = parse_date(data["CreationDate"]) + { created_at: created, updated_at: parse_date(data["ModificationDateTime"]) || created } + end + + def organization_type(value) + value = value.to_s.squish + return {} if value.blank? + + school = value.match?(SCHOOL_PATTERN) ? { school: true } : {} + type = TYPE_PATTERNS.find { |pattern, _type| value.match?(pattern) }&.last + return school.merge(organization_type: type) if type + + @unmatched_types[value] += 1 + school.merge(organization_type: Organization::ORGANIZATION_TYPE_OTHER, organization_type_other: clip(value)) + end + + def apply_status(organization, value) + name = value.to_s.strip + return if name.blank? + + status = @statuses[name.downcase] + if status.nil? + @unmatched_statuses[name] += 1 + return + end + + organization.organization_status = status + end + + def fill(organization, attributes) + attributes.each do |attribute, value| + next if value.blank? + next if attribute == :organization_type_other && organization.organization_type != Organization::ORGANIZATION_TYPE_OTHER + + organization[attribute] = value + end + end + + def stamp(organization, data) + organization.created_by_id ||= @staff.user_id_for(data["CreatedBy"]) || @import_user_id + organization.updated_by_id ||= @staff.user_id_for(data["ModifiedBy"]) || @import_user_id + end + + def unique_name(name, code) + return name unless Organization.exists?(name:) + + final = clip("#{name} (#{code})") + @suffixed << "#{code} #{name} → #{final}" + final + end + + def index(organization) + @org_by_id[organization.id] = organization + organization.filemaker_codes.each do |code| + @codes << code + @holders[code] << organization unless @holders[code].include?(organization) + end + end + + def note_retired(code, name) + return unless retired_record?(code) || name.match?(RETIRED_MARKER) + + @retired << "#{code} #{name}" + end + + def project_codes + @project_codes ||= @projects_by_org.values.flatten.map(&:fm_id) + end + + def failed(organization, label) + record(:failed, "#{label}: #{organization.errors.full_messages.to_sentence}") + nil + end + + def record(outcome, message) + @results[outcome] << message + end + end +end diff --git a/lib/tasks/fm_import_orgs.rake b/lib/tasks/fm_import_orgs.rake new file mode 100644 index 0000000000..66aecb7004 --- /dev/null +++ b/lib/tasks/fm_import_orgs.rake @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +namespace :fm do + desc "Import every FileMaker organization into portal organizations (round one). Writes; reruns skip finished rows." + task import_orgs: :environment do + report = [] + say = ->(line = "") { puts line; report << line } + $stdout.sync = true + started = Time.current + total = FmOrganization.count + Current.source = "fm_import_orgs" + + say.("Reading #{total} FileMaker organizations…") + progress = ->(read) { puts " #{read} / #{total} rows read — #{(Time.current - started).round}s" } + + begin + import = OrganizationServices::ImportOrgs.new(progress:).call + + say.("\n== Result ==") + import.results.each { |outcome, messages| say.("#{outcome.to_s.humanize.ljust(18)} #{messages.size}") } + %i[skipped failed].each do |outcome| + messages = import.results[outcome] + next if messages.empty? + + say.("\n#{outcome.to_s.humanize} (#{messages.size}):") + messages.each { |message| say.(" #{message}") } + end + + sections = { + "Rows whose ORG code lands on more than one organization" => import.multi_org, + "Portal organizations receiving ORG codes from more than one FileMaker organization" => import.condensed, + "Project codes not in the portal (round 2)" => import.unmatched_project_codes, + "Duplicate names told apart by code" => import.suffixed, + "Retired in FileMaker" => import.retired + } + sections.each do |heading, lines| + next if lines.blank? + + say.("\n#{heading} (#{lines.size}):") + lines.each { |line| say.(" #{line}") } + end + + tallies = { + "Status values nothing maps (left blank)" => import.unmatched_statuses, + "Organization types read as Other, kept as its description" => import.unmatched_types, + "Staff names no alias answers to (credited to the import user)" => import.unmatched_staff + } + tallies.each do |heading, counts| + next if counts.blank? + + say.("\n#{heading}:") + counts.sort_by { |name, count| [ -count, name ] }.each { |name, count| say.(" #{count.to_s.rjust(5)} #{name}") } + end + + say.("\nTook #{(Time.current - started).round}s.") + say.("Done.") + rescue StandardError => e + say.("\nStopped by #{e.class}: #{e.message} — the batches before this one are saved; rerun to continue.") + raise + ensure + begin + blob = ActiveStorage::Blob.create_and_upload!( + io: StringIO.new(report.join("\n")), content_type: "text/plain", + filename: "fm-import-orgs-#{Time.current.strftime("%Y%m%d-%H%M%S")}.txt" + ) + puts "\nReport saved. Read it back with: bin/rails runner 'puts ActiveStorage::Blob.find(#{blob.id}).download'" + rescue StandardError => e + puts "\nCould not save the report (#{e.class}: #{e.message}) — copy the output above." + end + end + end +end