From 17eb613784576869c211f58aa72fb180b3a4c418 Mon Sep 17 00:00:00 2001 From: Justin Miller <16829344+jmilljr24@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:07:26 -0500 Subject: [PATCH 1/2] add import task and service --- app/models/organization.rb | 2 +- .../organization_services/import_orgs.rb | 244 ++++++++++++++++++ lib/tasks/fm_import_orgs.rake | 75 ++++++ 3 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 app/services/organization_services/import_orgs.rb create mode 100644 lib/tasks/fm_import_orgs.rake diff --git a/app/models/organization.rb b/app/models/organization.rb index 7b97fb755e..28c894ae03 100644 --- a/app/models/organization.rb +++ b/app/models/organization.rb @@ -84,7 +84,7 @@ def communications_email validates :organization_type_other, length: { maximum: 255 } validates :tax_id, length: { maximum: 255 } validates :website_url, length: { maximum: 255 } - validates :mission_vision_values, length: { maximum: 255 } + validates :mission_vision_values, length: { maximum: 16_000 } # Admin bypass stays on Current.user until #2833 moves it to a controller-set flag (models never call policies). validate :affiliation_dates_locked, if: -> { affiliations.any? && !Current.user&.super_user? } validate :parent_is_not_self_or_descendant, if: :parent_id_changed? diff --git a/app/services/organization_services/import_orgs.rb b/app/services/organization_services/import_orgs.rb new file mode 100644 index 0000000000..f2407c0b9a --- /dev/null +++ b/app/services/organization_services/import_orgs.rb @@ -0,0 +1,244 @@ +module OrganizationServices + class ImportOrgs + include FilemakerValues + + OUTCOMES = %i[created linked_name linked_project already_imported skipped failed].freeze + BATCH_SIZE = 500 + ORG_CODE = /\AORG\d+\z/ + TYPE_PATTERNS = { + /501|non.?profit/i => "501c3/nonprofit", + /for.?profit|\bLLC\b/i => "For-profit", + /govt|government/i => "Government agency", + /\Aother\z/i => Organization::ORGANIZATION_TYPE_OTHER + }.freeze + SCHOOL_PATTERN = /school/i + + attr_reader :results, :differences, :status_disagreements, :conflicts, + :multi_project_only, :withheld_codes, :suffixed, :retired, + :unmatched_statuses, :unmatched_types, :unmatched_project_codes + + def unmatched_staff = @staff.unmatched + + def initialize(progress: nil) + @progress = progress + @results = OUTCOMES.index_with { [] } + @differences = [] + @status_disagreements = [] + @conflicts = [] + @multi_project_only = [] + @withheld_codes = [] + @suffixed = [] + @retired = [] + @unmatched_statuses = Hash.new(0) + @unmatched_types = Hash.new(0) + end + + def call + load_indexes + read = 0 + FmOrganization.order(:fm_id).pluck(:id).each_slice(BATCH_SIZE) do |ids| + ActiveRecord::Base.transaction(requires_new: true) do + FmOrganization.where(id: ids).order(:fm_id).each { |row| import_row(row) } + end + read += ids.size + @progress&.call(read) + end + @unmatched_project_codes = project_codes.reject { |code| @org_by_code.key?(code) } + self + end + + private + + def load_indexes + @orgs_by_name_key = Hash.new { |hash, key| hash[key] = [] } + @org_by_code = {} + Organization.includes(:organization_status).find_each do |organization| + @orgs_by_name_key[FilemakerNames.same_name_key(organization.name)] << organization + organization.filemaker_codes.each { |code| @org_by_code[code] = organization } + end + @projects_by_org = FmProject.all.group_by { |project| project.data["OrgID"].to_s } + @statuses = OrganizationStatus.all.index_by { |status| status.name.downcase } + @staff = StaffTranslation.new + @import_user_id = Setting.import_user&.id + end + + def import_row(fm_organization) + data = fm_organization.data + code = fm_organization.fm_id + name = clip(data["OrgName"].to_s.squish) + return record(:skipped, "#{code}: no name") if name.blank? + + if (organization = @org_by_code[code]) + return record(:already_imported, "#{code} #{name} → ##{organization.id}") + end + + projects = @projects_by_org[code] || [] + organization, outcome = find_match(name, projects) + if organization.nil? + organization = Organization.new(name: unique_name(name, code)) + outcome = :created + end + + kept = fill(organization, attributes(data)) + apply_status(organization, data["Status"]) + stamp(organization, data) + organization.filemaker_code = Organization.join_filemaker_codes( + organization.filemaker_code, code, *claimable_project_codes(projects) + ) + note_retired(code, name) + return failed(organization, "#{code} #{name}") unless organization.save + + index(organization) + note_conflicts(organization, code, name, projects, outcome) + note_differences(organization, code, name, kept) if kept.any? + record(outcome, "#{code} #{name} → ##{organization.id}") + end + + def find_match(name, projects) + organization = @orgs_by_name_key[FilemakerNames.same_name_key(name)].find { |candidate| uncoded?(candidate) } + return [ organization, :linked_name ] if organization + + project = projects.first if projects.one? + organization = project && @org_by_code[project.fm_id] + return [ organization, :linked_project ] if organization && uncoded?(organization) + + [ nil, nil ] + end + + def uncoded?(organization) + organization.filemaker_codes.none? { |code| code.match?(ORG_CODE) } + end + + def claimable_project_codes(projects) + return [] unless projects.one? + + code = projects.first.fm_id + @org_by_code.key?(code) ? [] : [ code ] + end + + def attributes(data) + { + website_url: clip(data["Website"]), + description: data["Description"].to_s.strip.presence, + mission_vision_values: data["Mission"].to_s.strip.presence, + **organization_type(data["OrgType"]), + **archive_timestamps(data) + } + end + + def archive_timestamps(data) + created = parse_date(data["CreationDate"]) + { created_at: created, updated_at: parse_date(data["ModificationDateTime"]) || created } + end + + def organization_type(value) + value = value.to_s.squish + return {} if value.blank? + + school = value.match?(SCHOOL_PATTERN) ? { school: true } : {} + type = TYPE_PATTERNS.find { |pattern, _type| value.match?(pattern) }&.last + return school.merge(organization_type: type) if type + + @unmatched_types[value] += 1 + school.merge(organization_type: Organization::ORGANIZATION_TYPE_OTHER, organization_type_other: clip(value)) + end + + def apply_status(organization, value) + name = value.to_s.strip + return if name.blank? + + status = @statuses[name.downcase] + if status.nil? + @unmatched_statuses[name] += 1 + return + end + + current = organization.organization_status + if current.nil? + organization.organization_status = status + elsif current.id != status.id + @status_disagreements << "#{organization.name}: portal #{current.name}, FileMaker #{name}" + end + end + + def fill(organization, attributes) + timestamps, fields = attributes.partition { |attribute, _value| %i[created_at updated_at].include?(attribute) } + timestamps.each { |attribute, value| organization[attribute] = value if organization.new_record? && value } + fields.filter_map do |attribute, value| + next if value.blank? + next if attribute == :organization_type_other && organization.organization_type != Organization::ORGANIZATION_TYPE_OTHER + + if organization[attribute].blank? + organization[attribute] = value + next + end + next if organization[attribute].to_s.squish.casecmp?(value.to_s.squish) + + [ Organization.human_attribute_name(attribute), organization[attribute].to_s.truncate(80), value.to_s.truncate(80) ] + end + end + + def stamp(organization, data) + organization.created_by_id ||= @staff.user_id_for(data["CreatedBy"]) || @import_user_id + organization.updated_by_id ||= @staff.user_id_for(data["ModifiedBy"]) || @import_user_id + end + + def unique_name(name, code) + return name unless Organization.exists?(name:) + + final = clip("#{name} (#{code})") + @suffixed << "#{code} #{name} → #{final}" + final + end + + def index(organization) + key = FilemakerNames.same_name_key(organization.name) + list = @orgs_by_name_key[key] + list << organization unless list.include?(organization) + organization.filemaker_codes.each { |code| @org_by_code[code] = organization } + end + + def note_conflicts(organization, code, name, projects, outcome) + held = projects.filter_map do |project| + holder = @org_by_code[project.fm_id] + [ project.fm_id, holder ] if holder && holder != organization + end + return if held.empty? + + held.each do |project_code, holder| + @withheld_codes << "#{project_code} (row #{code} #{name}): already on ##{holder.id} #{holder.name}" + end + where = held.map { |project_code, holder| "#{project_code} ##{holder.id} #{holder.name}" }.join("; ") + + if outcome == :linked_name + @conflicts << "#{code} #{name} → ##{organization.id}; project codes sit on #{where}" + elsif projects.size > 1 + @multi_project_only << "#{code} #{name} → ##{organization.id}; project codes sit on #{where}" + end + end + + def note_retired(code, name) + return unless retired_record?(code) || name.match?(RETIRED_MARKER) + + @retired << "#{code} #{name}" + end + + def note_differences(organization, code, name, kept) + lines = kept.map { |field, portal, archive| "#{field}: kept “#{portal}”, FileMaker has “#{archive}”" } + @differences << "#{code} #{name} → ##{organization.id}: #{lines.join('; ')}" + end + + def project_codes + @project_codes ||= @projects_by_org.values.flatten.map(&:fm_id) + end + + def failed(organization, label) + record(:failed, "#{label}: #{"##{organization.id} " if organization.persisted?}#{organization.errors.full_messages.to_sentence}") + nil + end + + def record(outcome, message) + @results[outcome] << message + end + end +end diff --git a/lib/tasks/fm_import_orgs.rake b/lib/tasks/fm_import_orgs.rake new file mode 100644 index 0000000000..f2fe256ba5 --- /dev/null +++ b/lib/tasks/fm_import_orgs.rake @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +namespace :fm do + desc "Import every FileMaker organization into portal organizations (round one). Writes; reruns skip finished rows." + task import_orgs: :environment do + report = [] + say = ->(line = "") { puts line; report << line } + $stdout.sync = true + started = Time.current + total = FmOrganization.count + Current.source = "fm_import_orgs" + + say.("Reading #{total} FileMaker organizations…") + progress = ->(read) { puts " #{read} / #{total} rows read — #{(Time.current - started).round}s" } + + begin + import = OrganizationServices::ImportOrgs.new(progress:).call + + say.("\n== Result ==") + import.results.each { |outcome, messages| say.("#{outcome.to_s.humanize.ljust(18)} #{messages.size}") } + %i[skipped failed].each do |outcome| + messages = import.results[outcome] + next if messages.empty? + + say.("\n#{outcome.to_s.humanize} (#{messages.size}):") + messages.each { |message| say.(" #{message}") } + end + + sections = { + "Status disagreements (portal kept its value)" => import.status_disagreements, + "Name matches whose project codes sit on another organization" => import.conflicts, + "Multi-project organizations whose programs are already in the portal" => import.multi_project_only, + "Project codes withheld because another organization holds them" => import.withheld_codes, + "Project codes not in the portal (round 2)" => import.unmatched_project_codes, + "Duplicate names told apart by code" => import.suffixed, + "Retired in FileMaker, imported as-is" => import.retired, + "Field differences (portal kept its value)" => import.differences + } + sections.each do |heading, lines| + next if lines.blank? + + say.("\n#{heading} (#{lines.size}):") + lines.each { |line| say.(" #{line}") } + end + + tallies = { + "Status values nothing maps (left blank)" => import.unmatched_statuses, + "Organization types read as Other, kept as its description" => import.unmatched_types, + "Staff names no alias answers to (credited to the import user)" => import.unmatched_staff + } + tallies.each do |heading, counts| + next if counts.blank? + + say.("\n#{heading}:") + counts.sort_by { |name, count| [ -count, name ] }.each { |name, count| say.(" #{count.to_s.rjust(5)} #{name}") } + end + + say.("\nTook #{(Time.current - started).round}s.") + say.("Done.") + rescue StandardError => e + say.("\nStopped by #{e.class}: #{e.message} — the batches before this one are saved; rerun to continue.") + raise + ensure + begin + blob = ActiveStorage::Blob.create_and_upload!( + io: StringIO.new(report.join("\n")), content_type: "text/plain", + filename: "fm-import-orgs-#{Time.current.strftime("%Y%m%d-%H%M%S")}.txt" + ) + puts "\nReport saved. Read it back with: bin/rails runner 'puts ActiveStorage::Blob.find(#{blob.id}).download'" + rescue StandardError => e + puts "\nCould not save the report (#{e.class}: #{e.message}) — copy the output above." + end + end + end +end From ad979c9215dc5a3620416273cfc09389424621b5 Mon Sep 17 00:00:00 2001 From: Justin Miller <16829344+jmilljr24@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:30:58 -0500 Subject: [PATCH 2/2] don't match on name --- .../organization_services/import_orgs.rb | 149 +++++++----------- lib/tasks/fm_import_orgs.rake | 9 +- 2 files changed, 59 insertions(+), 99 deletions(-) diff --git a/app/services/organization_services/import_orgs.rb b/app/services/organization_services/import_orgs.rb index f2407c0b9a..be3902324c 100644 --- a/app/services/organization_services/import_orgs.rb +++ b/app/services/organization_services/import_orgs.rb @@ -2,9 +2,8 @@ module OrganizationServices class ImportOrgs include FilemakerValues - OUTCOMES = %i[created linked_name linked_project already_imported skipped failed].freeze + OUTCOMES = %i[created linked_project already_imported skipped failed].freeze BATCH_SIZE = 500 - ORG_CODE = /\AORG\d+\z/ TYPE_PATTERNS = { /501|non.?profit/i => "501c3/nonprofit", /for.?profit|\bLLC\b/i => "For-profit", @@ -13,8 +12,7 @@ class ImportOrgs }.freeze SCHOOL_PATTERN = /school/i - attr_reader :results, :differences, :status_disagreements, :conflicts, - :multi_project_only, :withheld_codes, :suffixed, :retired, + attr_reader :results, :multi_org, :condensed, :suffixed, :retired, :unmatched_statuses, :unmatched_types, :unmatched_project_codes def unmatched_staff = @staff.unmatched @@ -22,15 +20,13 @@ def unmatched_staff = @staff.unmatched def initialize(progress: nil) @progress = progress @results = OUTCOMES.index_with { [] } - @differences = [] - @status_disagreements = [] - @conflicts = [] - @multi_project_only = [] - @withheld_codes = [] + @multi_org = [] + @condensed = [] @suffixed = [] @retired = [] @unmatched_statuses = Hash.new(0) @unmatched_types = Hash.new(0) + @placed_by_org = Hash.new { |hash, key| hash[key] = [] } end def call @@ -43,18 +39,24 @@ def call read += ids.size @progress&.call(read) end - @unmatched_project_codes = project_codes.reject { |code| @org_by_code.key?(code) } + @unmatched_project_codes = project_codes.reject { |code| @codes.include?(code) } + @condensed = @placed_by_org.select { |_id, codes| codes.size > 1 } + .map { |id, codes| "##{id} #{@org_by_id[id]&.name}: #{codes.join(', ')}" } self end private def load_indexes - @orgs_by_name_key = Hash.new { |hash, key| hash[key] = [] } - @org_by_code = {} - Organization.includes(:organization_status).find_each do |organization| - @orgs_by_name_key[FilemakerNames.same_name_key(organization.name)] << organization - organization.filemaker_codes.each { |code| @org_by_code[code] = organization } + @codes = Set.new + @holders = Hash.new { |hash, key| hash[key] = [] } + @org_by_id = {} + Organization.find_each do |organization| + @org_by_id[organization.id] = organization + organization.filemaker_codes.each do |code| + @codes << code + @holders[code] << organization + end end @projects_by_org = FmProject.all.group_by { |project| project.data["OrgID"].to_s } @statuses = OrganizationStatus.all.index_by { |status| status.name.downcase } @@ -67,53 +69,50 @@ def import_row(fm_organization) code = fm_organization.fm_id name = clip(data["OrgName"].to_s.squish) return record(:skipped, "#{code}: no name") if name.blank? + return record(:already_imported, "#{code} #{name}") if @codes.include?(code) + + note_retired(code, name) + projects = @projects_by_org[code] || [] + targets = projects.flat_map { |project| @holders[project.fm_id] }.uniq + return attach_to_targets(code, name, targets) if targets.any? - if (organization = @org_by_code[code]) - return record(:already_imported, "#{code} #{name} → ##{organization.id}") + create_org(code, name, data, projects) + end + + def attach_to_targets(code, name, targets) + failed = targets.reject do |organization| + organization.filemaker_code = Organization.join_filemaker_codes(organization.filemaker_code, code) + organization.save + end + unless failed.empty? + return record(:failed, "#{code} #{name}: could not add the code to #{failed.map { |organization| "##{organization.id}" }.join(', ')}") end - projects = @projects_by_org[code] || [] - organization, outcome = find_match(name, projects) - if organization.nil? - organization = Organization.new(name: unique_name(name, code)) - outcome = :created + targets.each do |organization| + @codes << code + @holders[code] << organization unless @holders[code].include?(organization) + @placed_by_org[organization.id] << code end + if targets.size > 1 + @multi_org << "#{code} #{name} → #{targets.map { |organization| "##{organization.id} #{organization.name}" }.join('; ')}" + end + record(:linked_project, "#{code} #{name} → #{targets.map { |organization| "##{organization.id}" }.join(', ')}") + end - kept = fill(organization, attributes(data)) + def create_org(code, name, data, projects) + organization = Organization.new(name: unique_name(name, code)) + fill(organization, attributes(data)) apply_status(organization, data["Status"]) stamp(organization, data) - organization.filemaker_code = Organization.join_filemaker_codes( - organization.filemaker_code, code, *claimable_project_codes(projects) - ) - note_retired(code, name) + organization.filemaker_code = Organization.join_filemaker_codes(code, *claimable_project_codes(projects)) return failed(organization, "#{code} #{name}") unless organization.save index(organization) - note_conflicts(organization, code, name, projects, outcome) - note_differences(organization, code, name, kept) if kept.any? - record(outcome, "#{code} #{name} → ##{organization.id}") - end - - def find_match(name, projects) - organization = @orgs_by_name_key[FilemakerNames.same_name_key(name)].find { |candidate| uncoded?(candidate) } - return [ organization, :linked_name ] if organization - - project = projects.first if projects.one? - organization = project && @org_by_code[project.fm_id] - return [ organization, :linked_project ] if organization && uncoded?(organization) - - [ nil, nil ] - end - - def uncoded?(organization) - organization.filemaker_codes.none? { |code| code.match?(ORG_CODE) } + record(:created, "#{code} #{name} → ##{organization.id}") end def claimable_project_codes(projects) - return [] unless projects.one? - - code = projects.first.fm_id - @org_by_code.key?(code) ? [] : [ code ] + projects.one? ? [ projects.first.fm_id ] : [] end def attributes(data) @@ -153,28 +152,15 @@ def apply_status(organization, value) return end - current = organization.organization_status - if current.nil? - organization.organization_status = status - elsif current.id != status.id - @status_disagreements << "#{organization.name}: portal #{current.name}, FileMaker #{name}" - end + organization.organization_status = status end def fill(organization, attributes) - timestamps, fields = attributes.partition { |attribute, _value| %i[created_at updated_at].include?(attribute) } - timestamps.each { |attribute, value| organization[attribute] = value if organization.new_record? && value } - fields.filter_map do |attribute, value| + attributes.each do |attribute, value| next if value.blank? next if attribute == :organization_type_other && organization.organization_type != Organization::ORGANIZATION_TYPE_OTHER - if organization[attribute].blank? - organization[attribute] = value - next - end - next if organization[attribute].to_s.squish.casecmp?(value.to_s.squish) - - [ Organization.human_attribute_name(attribute), organization[attribute].to_s.truncate(80), value.to_s.truncate(80) ] + organization[attribute] = value end end @@ -192,28 +178,10 @@ def unique_name(name, code) end def index(organization) - key = FilemakerNames.same_name_key(organization.name) - list = @orgs_by_name_key[key] - list << organization unless list.include?(organization) - organization.filemaker_codes.each { |code| @org_by_code[code] = organization } - end - - def note_conflicts(organization, code, name, projects, outcome) - held = projects.filter_map do |project| - holder = @org_by_code[project.fm_id] - [ project.fm_id, holder ] if holder && holder != organization - end - return if held.empty? - - held.each do |project_code, holder| - @withheld_codes << "#{project_code} (row #{code} #{name}): already on ##{holder.id} #{holder.name}" - end - where = held.map { |project_code, holder| "#{project_code} ##{holder.id} #{holder.name}" }.join("; ") - - if outcome == :linked_name - @conflicts << "#{code} #{name} → ##{organization.id}; project codes sit on #{where}" - elsif projects.size > 1 - @multi_project_only << "#{code} #{name} → ##{organization.id}; project codes sit on #{where}" + @org_by_id[organization.id] = organization + organization.filemaker_codes.each do |code| + @codes << code + @holders[code] << organization unless @holders[code].include?(organization) end end @@ -223,17 +191,12 @@ def note_retired(code, name) @retired << "#{code} #{name}" end - def note_differences(organization, code, name, kept) - lines = kept.map { |field, portal, archive| "#{field}: kept “#{portal}”, FileMaker has “#{archive}”" } - @differences << "#{code} #{name} → ##{organization.id}: #{lines.join('; ')}" - end - def project_codes @project_codes ||= @projects_by_org.values.flatten.map(&:fm_id) end def failed(organization, label) - record(:failed, "#{label}: #{"##{organization.id} " if organization.persisted?}#{organization.errors.full_messages.to_sentence}") + record(:failed, "#{label}: #{organization.errors.full_messages.to_sentence}") nil end diff --git a/lib/tasks/fm_import_orgs.rake b/lib/tasks/fm_import_orgs.rake index f2fe256ba5..66aecb7004 100644 --- a/lib/tasks/fm_import_orgs.rake +++ b/lib/tasks/fm_import_orgs.rake @@ -27,14 +27,11 @@ namespace :fm do end sections = { - "Status disagreements (portal kept its value)" => import.status_disagreements, - "Name matches whose project codes sit on another organization" => import.conflicts, - "Multi-project organizations whose programs are already in the portal" => import.multi_project_only, - "Project codes withheld because another organization holds them" => import.withheld_codes, + "Rows whose ORG code lands on more than one organization" => import.multi_org, + "Portal organizations receiving ORG codes from more than one FileMaker organization" => import.condensed, "Project codes not in the portal (round 2)" => import.unmatched_project_codes, "Duplicate names told apart by code" => import.suffixed, - "Retired in FileMaker, imported as-is" => import.retired, - "Field differences (portal kept its value)" => import.differences + "Retired in FileMaker" => import.retired } sections.each do |heading, lines| next if lines.blank?