diff --git a/app/jobs/scsb_import_full_job.rb b/app/jobs/scsb_import_full_job.rb index 8630e5e5c..e9ad39847 100644 --- a/app/jobs/scsb_import_full_job.rb +++ b/app/jobs/scsb_import_full_job.rb @@ -12,12 +12,8 @@ def perform private - def dump_type - @dump_type ||= DumpType.find_or_create_by!(constant: DumpType::PARTNER_RECAP_FULL) - end - def created_dump - Dump.create!(dump_type:) + Dump.create!(dump_type_id: 5) end def delete_stale_files diff --git a/app/models/dump.rb b/app/models/dump.rb index d67cb3eae..c26cd0b35 100644 --- a/app/models/dump.rb +++ b/app/models/dump.rb @@ -22,10 +22,10 @@ class Dump < ActiveRecord::Base end end - scope :partner_recap_full, -> { where(dump_type: DumpType.where(constant: 'PARTNER_RECAP_FULL')) } - scope :partner_recap, -> { where(dump_type: DumpType.where(constant: 'PARTNER_RECAP')) } - scope :changed_records, -> { where(dump_type: DumpType.where(constant: 'CHANGED_RECORDS')) } - scope :full_dumps, -> { where(dump_type: DumpType.where(constant: 'ALL_RECORDS')) } + scope :partner_recap_full, -> { where(dump_type_id: 5) } + scope :partner_recap, -> { where(dump_type_id: 4) } + scope :changed_records, -> { where(dump_type_id: 2) } + scope :full_dumps, -> { where(dump_type_id: 1) } class << self ## @@ -34,7 +34,7 @@ def partner_update dump = nil timestamp = incremental_update_timestamp Event.record do |event| - dump = Dump.create(dump_type: DumpType.find_by(constant: "PARTNER_RECAP")) + dump = Dump.create(dump_type_id: 4) ScsbImportJob.perform_later(dump.id, timestamp) dump.event = event dump.save diff --git a/app/services/aws_sqs_poller.rb b/app/services/aws_sqs_poller.rb index 4dcc80ecc..c4c859473 100644 --- a/app/services/aws_sqs_poller.rb +++ b/app/services/aws_sqs_poller.rb @@ -41,18 +41,15 @@ def self.bib_dump(message) end def bib_dump - dump = Dump.create(dump_type:) + dump = Dump.create(dump_type_id:) dump.event = dump_event dump.generated_date = dump_event.start dump.save dump end - def dump_type - @dump_type ||= begin - dump_type_config = Rails.configuration.alma[:jobs][job_name]["dump_type"] - DumpType.find_by(constant: dump_type_config) - end + def dump_type_id + @dump_type_id ||= Rails.configuration.alma[:jobs][job_name]["dump_type_id"] end def job_name diff --git a/app/utils/data_seeder.rb b/app/utils/data_seeder.rb index 4d107029f..d0cea6d58 100644 --- a/app/utils/data_seeder.rb +++ b/app/utils/data_seeder.rb @@ -11,7 +11,7 @@ def initialize(logger = Logger.new(STDOUT)) def generate_dump_types dump_types.each do |dt| - DumpType.find_or_create_by(label: dt['label'], constant: dt['constant']) + DumpType.find_or_create_by(id: dt['id'], label: dt['label'], constant: dt['constant']) end logger.info "Created #{dump_types.count} dump types" end diff --git a/config/alma.yml b/config/alma.yml index 433769fc6..ef3ca75c7 100644 --- a/config/alma.yml +++ b/config/alma.yml @@ -8,12 +8,15 @@ default: &default sqs_queue_url: <%= ENV['SQS_QUEUE_URL'] %> jobs: "Publishing Platform Job General Publishing": + dump_type_id: 1 dump_type: "ALL_RECORDS" dump_file_type: "BIB_RECORDS" "Publishing Platform Job Incremental Publishing": + dump_type_id: 2 dump_type: "CHANGED_RECORDS" dump_file_type: "UPDATED_RECORDS" "Publishing Platform Job Incremental ReCAP Records": + dump_type_id: 3 dump_type: "PRINCETON_RECAP" dump_file_type: "RECAP_RECORDS" api_limit: <%= ENV["ALMA_API_LIMIT"] || 150000 %> diff --git a/config/marc_liberation.yml b/config/marc_liberation.yml index 0dd0c4ffe..8b775d3ce 100644 --- a/config/marc_liberation.yml +++ b/config/marc_liberation.yml @@ -9,19 +9,26 @@ figgy_base_url: "https://figgy.princeton.edu" location_files_dir: <%= ENV['LOCATION_FILES_DIR'] || 'config/locations' %> dump_types: -- label: 'All Records' # used for full dumps +- id: 1 + label: 'All Records' # used for full dumps constant: 'ALL_RECORDS' -- label: 'Changed Records' # used for incremental dumps +- id: 2 + label: 'Changed Records' # used for incremental dumps constant: 'CHANGED_RECORDS' -- label: 'Updated Princeton ReCAP Records' # used for Princeton records we send to SCSB +- id: 3 + label: 'Updated Princeton ReCAP Records' # used for Princeton records we send to SCSB constant: 'PRINCETON_RECAP' -- label: 'Updated Partner ReCAP Records' # used for records we get from SCSB partners (NYPL and Columbia) +- id: 4 + label: 'Updated Partner ReCAP Records' # used for records we get from SCSB partners (NYPL and Columbia) constant: 'PARTNER_RECAP' -- label: 'Full Partner ReCAP Records' # used for records we get from SCSB partners (NYPL and Columbia) +- id: 5 + label: 'Full Partner ReCAP Records' # used for records we get from SCSB partners (NYPL and Columbia) constant: 'PARTNER_RECAP_FULL' -- label: 'All Bib IDs' # no longer used +- id: 6 + label: 'All Bib IDs' # no longer used constant: 'BIB_IDS' -- label: 'All Bib IDs with Holdings' # no longer used +- id: 7 + label: 'All Bib IDs with Holdings' # no longer used constant: 'MERGED_IDS' dump_file_types: diff --git a/spec/factories/event.rb b/spec/factories/event.rb index 4e032e384..743781384 100644 --- a/spec/factories/event.rb +++ b/spec/factories/event.rb @@ -8,26 +8,6 @@ updated_at { finish } end - factory :full_dump_type, class: "DumpType" do - constant { "ALL_RECORDS" } - label { "All Records" } - end - - factory :incremental_dump_type, class: "DumpType" do - constant { "CHANGED_RECORDS" } - label { "Changed Records" } - end - - factory :partner_recap_daily_dump_type, class: "DumpType" do - constant { "PARTNER_RECAP" } - label { "Updated Partner ReCAP Records" } - end - - factory :partner_recap_full_dump_type, class: "DumpType" do - constant { "PARTNER_RECAP_FULL" } - label { "Full Partner ReCAP Records" } - end - factory :full_dump_file_type, class: "DumpFileType" do constant { "BIB_RECORDS" } label { "All Bib Records" } @@ -74,35 +54,35 @@ end factory :empty_dump, class: "Dump" do - association :dump_type, factory: :full_dump_type + dump_type_id { 1 } association :event end factory :empty_incremental_dump, class: "Dump" do - association :dump_type, factory: :incremental_dump_type + dump_type_id { 2 } association :event end factory :empty_partner_recap_incremental_dump, class: "Dump" do - association :dump_type, factory: :partner_recap_daily_dump_type + dump_type_id { 4 } association :event end factory :empty_partner_full_dump, class: "Dump" do - association :dump_type, factory: :partner_recap_full_dump_type + dump_type_id { 5 } association :event end factory :empty_partner_recap_dump, class: "Dump" do - dump_type { DumpType.find_by(constant: "PARTNER_RECAP") } + dump_type_id { 4 } association :event end factory :full_dump, class: "Dump" do - association :dump_type, factory: :full_dump_type generated_date { Time.new(2021, 7, 13, 11, 0, 0, "+00:00") } delete_ids { [] } update_ids { [] } + dump_type_id { 1 } dump_files do [ association(:dump_file, path: 'spec/fixtures/files/alma/full_dump/1.xml.tar.gz'), @@ -112,9 +92,9 @@ end factory :incremental_dump, class: "Dump" do - association :dump_type, factory: :incremental_dump_type delete_ids { [] } update_ids { [] } + dump_type_id { 2 } dump_files do [ association(:incremental_dump_file, path: 'spec/fixtures/files/alma/incremental_dump/1.tar.gz'), @@ -124,7 +104,7 @@ end factory :partner_recap_daily_dump, class: "Dump" do - association :dump_type, factory: :partner_recap_daily_dump_type + dump_type_id { 4 } delete_ids { [] } update_ids { [] } dump_files do @@ -136,7 +116,7 @@ end factory :partner_recap_full_dump, class: "Dump" do - association :dump_type, factory: :partner_recap_full_dump_type + dump_type_id { 5 } delete_ids { [] } update_ids { [] } dump_files do diff --git a/spec/models/dump_spec.rb b/spec/models/dump_spec.rb index 73b6c9b05..2e5a84cb2 100644 --- a/spec/models/dump_spec.rb +++ b/spec/models/dump_spec.rb @@ -4,9 +4,9 @@ let(:partner_recap) { 'PARTNER_RECAP' } let(:princeton_recap) { 'PRINCETON_RECAP' } let(:partner_recap_full) { 'PARTNER_RECAP_FULL' } - let(:princeton_recap_dump_type) { DumpType.find_by(constant: princeton_recap) } - let(:partner_recap_dump_type) { DumpType.find_by(constant: partner_recap) } - let(:partner_recap_full_dump_type) { DumpType.find_by(constant: partner_recap_full) } + let(:princeton_recap_dump_type) { DumpType.find(3) } + let(:partner_recap_dump_type) { DumpType.find(4) } + let(:partner_recap_full_dump_type) { DumpType.find(5) } let(:test_create_time) { '2017-04-29 20:10:29'.to_time } let(:event_success) { Event.create(start: '2020-10-20 19:00:15', finish: '2020-10-20 19:00:41', error: nil, success: true, created_at: "2020-10-20 19:00:41", updated_at: "2020-10-20 19:00:41") } let(:dump_princeton_recap_success) { described_class.create(event_id: event_success.id, dump_type_id: princeton_recap_dump_type.id, created_at: "2020-10-20 19:00:15", updated_at: "2020-10-20 19:00:41") } diff --git a/spec/services/aws_sqs_poller_spec.rb b/spec/services/aws_sqs_poller_spec.rb index 382574fca..833b6e70e 100644 --- a/spec/services/aws_sqs_poller_spec.rb +++ b/spec/services/aws_sqs_poller_spec.rb @@ -14,8 +14,6 @@ end before do - FactoryBot.create(:full_dump_type) - FactoryBot.create(:incremental_dump_type) Aws.config[:sqs] = { stub_responses: { receive_message: [