diff --git a/src-colladmin/actions/access_queue_action.rb b/src-colladmin/actions/access_queue_action.rb index dd26d97..4478334 100644 --- a/src-colladmin/actions/access_queue_action.rb +++ b/src-colladmin/actions/access_queue_action.rb @@ -7,11 +7,7 @@ class AccessQueueAction < ZookeeperListAction def perform_action jobs = [] - if ZookeeperListAction.migration_m3? - jobs = MerrittZK::Access.list_jobs_as_json(@zk) - else - jobs = MerrittZK::LegacyAccessJob.list_jobs_as_json(@zk) - end + jobs = MerrittZK::Access.list_jobs_as_json(@zk) jobs.each do |po| register_item(AccQueueEntry.new(po)) end diff --git a/src-colladmin/actions/forward_to_ingest_action.rb b/src-colladmin/actions/forward_to_ingest_action.rb index 002b80b..0ca9396 100644 --- a/src-colladmin/actions/forward_to_ingest_action.rb +++ b/src-colladmin/actions/forward_to_ingest_action.rb @@ -11,7 +11,6 @@ def initialize(config, action, path, myparams, endpoint) @endpoint = endpoint @zk = ZK.new(get_zookeeper_conn) - ZookeeperListAction.migration_level(@zk) end def get_zookeeper_conn diff --git a/src-colladmin/actions/ingest_collection_locks_action.rb b/src-colladmin/actions/ingest_collection_locks_action.rb index 5199dd7..51d3bb5 100644 --- a/src-colladmin/actions/ingest_collection_locks_action.rb +++ b/src-colladmin/actions/ingest_collection_locks_action.rb @@ -18,13 +18,9 @@ def initialize(config, action, path, myparams) @held_counts = {} ql = QueueList.new(@zk) ql.jobs.each do |qe| - if ZookeeperListAction.migration_m1? - next if qe.profile.empty? - # next unless MerrittZK::Locks.check_lock_collection(@zk, qe.profile) - next if qe.qstatus != 'Held' - elsif qe.qstatus != 'Held' - next - end + next if qe.profile.empty? + # next unless MerrittZK::Locks.check_lock_collection(@zk, qe.profile) + next if qe.qstatus != 'Held' @held_counts[qe.profile] = @held_counts.fetch(qe.profile, 0) + 1 end @@ -39,19 +35,11 @@ def specific_profile? end def table_headers - if ZookeeperListAction.migration_m1? - ['Profile', 'CollId', 'Name', 'Locked', 'Locks ZK', 'Held Items', 'Release'] - else - ['Profile', 'CollId', 'Name', 'Locked', 'Locks', 'Held Items', 'Release'] - end + ['Profile', 'CollId', 'Name', 'Locked', 'Locks ZK', 'Held Items', 'Release'] end def table_types - if ZookeeperListAction.migration_m1? - ['', 'colllist', '', '', 'colllockzk', 'dataint', 'collqitems-mrtzk'] - else - ['', 'colllist', '', '', 'colllock', 'dataint', 'collqitems-legacy'] - end + ['', 'colllist', '', '', 'colllockzk', 'dataint', 'collqitems-mrtzk'] end def table_rows(_body) @@ -91,13 +79,11 @@ def get_profile_names locked: false, name: p.get_value(:profileDescription) } - pstat[:locked] = MerrittZK::Locks.check_lock_collection(@zk, profile) if ZookeeperListAction.migration_m1? + pstat[:locked] = MerrittZK::Locks.check_lock_collection(@zk, profile) names[profile] = pstat end - unless ZookeeperListAction.migration_m1? - IngestStateAction.new(@config, {}, 'state', {}).get_locked_collections.each do |k| - names.fetch(k, {})[:locked] = true - end + IngestStateAction.new(@config, {}, 'state', {}).get_locked_collections.each do |k| + names.fetch(k, {})[:locked] = true end rescue StandardError => e log(e.message) diff --git a/src-colladmin/actions/inventory_queue_action.rb b/src-colladmin/actions/inventory_queue_action.rb deleted file mode 100644 index 5f3e7d7..0000000 --- a/src-colladmin/actions/inventory_queue_action.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -require_relative 'zookeeper_action' -require_relative '../lib/inv_queue' - -# Collection Admin Task class - see config/actions.yml for description -class InventoryQueueAction < ZookeeperListAction - def perform_action - jobs = ZookeeperListAction.migration_m1? ? [] : MerrittZK::LegacyInventoryJob.list_jobs_as_json(@zk) - jobs.each do |po| - register_item(InvQueueEntry.new(po)) - end - convert_json_to_table('') - end - - def get_title - 'List Inventory Queues' - end - - def table_headers - InvQueueEntry.table_headers - end - - def table_types - InvQueueEntry.table_types - end - - def has_table - true - end - - def init_status - :PASS - end - - def page_size - 100 - end - - def get_alternative_queries - [ - { - label: 'Requeue All', - url: '', - class: 'action requeue-all' - }, - { - label: 'Delete All', - url: '', - class: 'action deleteq-all' - }, - { - label: 'Cleanup Queue', - url: "#{LambdaBase.colladmin_url}?path=cleanup-queue&queue=queues-inv&reload_path=inv-queues", - class: 'action' - } - ] - end -end diff --git a/src-colladmin/actions/zookeeper_action.rb b/src-colladmin/actions/zookeeper_action.rb index c63695a..ee867a8 100644 --- a/src-colladmin/actions/zookeeper_action.rb +++ b/src-colladmin/actions/zookeeper_action.rb @@ -38,28 +38,9 @@ def initialize(config, action, path, myparams, _filters = {}) super(config, action, path, myparams) @filters = {} @zk = ZK.new(get_zookeeper_conn) - ZookeeperListAction.migration_level(@zk) @items = ZkList.new end - def self.migration_level(zk) - $migration = [] - $migration << :m1 if zk.exists?('/migration/m1') - $migration << :m3 if zk.exists?('/migration/m3') - end - - def self.migration_m1? - return false if $migration.nil? - - $migration.include?(:m1) - end - - def self.migration_m3? - return false if $migration.nil? - - $migration.include?(:m3) - end - attr_reader :items def register_item(item) @@ -84,7 +65,6 @@ class ZookeeperAction < AdminAction def initialize(config, action, path, myparams) super @zk = ZK.new(get_zookeeper_conn) - ZookeeperListAction.migration_level(@zk) @qpath = myparams.fetch('queue-path', '') end @@ -199,7 +179,7 @@ def report_node(n) show_data(n) if @mode == 'data' show_test(n) if @mode == 'test' else - @buf << " Legacy\n" + @buf << " Unsupported\n" end @buf << "\n" end @@ -232,7 +212,7 @@ def dump_node(n = '/') end ## Base class for new style action -class ZkM1Action < ZookeeperAction +class ZkAction < ZookeeperAction def get_id @qpath.split('/')[-1] end @@ -247,7 +227,7 @@ def perform_action end ## Queue manipulation action using new mrt-zk -class ZkRequeueM1Action < ZkM1Action +class ZkRequeueAction < ZkAction def perform_action if @qpath =~ /access/ acc = MerrittZK::Access.new(get_access_queue, get_id) @@ -279,7 +259,7 @@ def perform_action end ## Queue manipulation action using new mrt-zk -class ZkDeleteM1Action < ZkM1Action +class ZkDeleteAction < ZkAction def perform_action if @qpath =~ /access/ acc = MerrittZK::Access.new(get_access_queue, get_id) @@ -296,7 +276,7 @@ def perform_action end ## Queue manipulation action using new mrt-zk -class ZkDeleteBatchAction < ZkM1Action +class ZkDeleteBatchAction < ZkAction def perform_action b = MerrittZK::Batch.new(get_id) b.load(@zk) @@ -306,7 +286,7 @@ def perform_action end ## Queue manipulation action using new mrt-zk -class ZkUpdateReportingBatchAction < ZkM1Action +class ZkUpdateReportingBatchAction < ZkAction def perform_action b = MerrittZK::Batch.new(get_id) b.load(@zk) @@ -316,7 +296,7 @@ def perform_action end ## Queue manipulation action using new mrt-zk -class ZkHoldM1Action < ZkM1Action +class ZkHoldAction < ZkAction def perform_action job = MerrittZK::Job.new(get_id) job.load(@zk) @@ -326,7 +306,7 @@ def perform_action end ## Queue manipulation action using new mrt-zk -class ZkReleaseM1Action < ZkM1Action +class ZkReleaseAction < ZkAction def perform_action job = MerrittZK::Job.new(get_id) job.load(@zk) @@ -335,116 +315,11 @@ def perform_action end end -## Legacy Queue manipulation action using new mrt-zk -class LegacyZkAction < ZookeeperAction - def legacy_status_vals - MerrittZK::LegacyItem::STATUS_VALS - end - - def prefix - 'na' - end - - attr_reader :qpath - - def bytes(p) - data = @zk.get(p) - return if data.nil? - - data[0].bytes - end - - def orig_stat(p) - return if bytes(p).nil? - - bytes(p)[0] - end - - def orig_stat_name(p) - return if orig_stat(p).nil? - - legacy_status_vals[orig_stat(p)] - end - - def write_status(p, status) - pbytes = bytes(p) - pbytes[0] = status - @zk.set(p, pbytes.pack('CCCCCCCCCc*')) - end - - def set_legacy_status(p, status) - i = legacy_status_vals.find_index(status) - return if i.nil? - - orig_name = orig_stat_name(p) - return { message: 'Illegal status' }.to_json unless check_status(orig_name) - - write_status(p, i) - { message: "Status #{orig_name} -- > #{status}" }.to_json - end - - def check_status(_status) - true - end -end - -## -# Legacy Ingest queue action -class ZkRequeueLegacyAction < LegacyZkAction - def perform_action - set_legacy_status(qpath, 'Pending') - end - - def check_status(status) - %w[Consumed Failed].include?(status) - end -end - -## -# Legacy Ingest queue action -class ZkDeleteLegacyAction < LegacyZkAction - def perform_action - set_legacy_status(qpath, 'Deleted') - end - - def check_status(status) - %w[Failed Completed].include?(status) - end -end - -## -# Legacy Ingest queue action -class ZkHoldLegacyAction < LegacyZkAction - def perform_action - set_legacy_status(qpath, 'Held') - end - - def check_status(status) - %w[Pending].include?(status) - end -end - -## -# Legacy Ingest queue action -class ZkReleaseLegacyAction < LegacyZkAction - def perform_action - set_legacy_status(qpath, 'Pending') - end - - def check_status(status) - %w[Held].include?(status) - end -end - -## Class for reading the legacy Merritt Ingest Queue +## Class for reading the Merritt Ingest Queue class IngestQueueZookeeperAction < ZookeeperListAction def perform_action jobs = [] - if ZookeeperListAction.migration_m1? - jobs = MerrittZK::Job.list_jobs_as_json(@zk) - else - jobs = MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk) - end + jobs = MerrittZK::Job.list_jobs_as_json(@zk) jobs.each do |po| register_item(QueueEntry.new(po)) end @@ -464,7 +339,7 @@ def perform_action end ## Lock collection action -class CollLockZkAction < ZkM1Action +class CollLockZkAction < ZkAction def initialize(config, action, path, myparams) super @coll = myparams.fetch('coll', '') @@ -477,7 +352,7 @@ def perform_action end ## Unlock collection action -class CollUnlockZkAction < ZkM1Action +class CollUnlockZkAction < ZkAction def initialize(config, action, path, myparams) super @coll = myparams.fetch('coll', '') @@ -490,7 +365,7 @@ def perform_action end # Collection Admin Task class - see config/actions.yml for description -class CollIterateQueueM1Action < ZkM1Action +class CollIterateQueueAction < ZkAction def initialize(config, action, path, myparams) super @coll = myparams.fetch('coll', '') @@ -509,27 +384,6 @@ def perform_action end end -# Collection Admin Task class - see config/actions.yml for description -class CollIterateQueueLegacyAction < LegacyZkAction - def initialize(config, action, path, myparams) - super - @coll = myparams.fetch('coll', '') - end - - def perform_action - count = 0 - MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk).each do |j| - job = QueueEntry.new(j) - next unless job.qstatus == 'Held' - next unless job.profile == @coll - - set_legacy_status("/ingest/#{job.queue_id}", 'Pending') - count += 1 - end - { message: "queue release submitted for #{count}" }.to_json - end -end - # Collection Admin Task class - see config/actions.yml for description class IterateQueueAction < ZookeeperAction def initialize(config, action, path, myparams) @@ -538,17 +392,8 @@ def initialize(config, action, path, myparams) @reload_path = myparams.fetch('reload_path', 'na') end - def legacy_delete(job) - status = job.fetch(:status, '') - path = job.fetch(:path, '') - return unless %w[Completed Deleted].include?(status) - return if path.empty? - - @zk.delete(path) - end - def perform_action - if @queue == 'queues-acc' && ZookeeperListAction.migration_m3? + if @queue == 'queues-acc' MerrittZK::Access.list_jobs_as_json(@zk).each do |job| qn = job.fetch(:queueNode, MerrittZK::Access::SMALL).gsub(%r{^/access/}, '') j = MerrittZK::Access.new(qn, job.fetch(:id, '')) @@ -557,17 +402,7 @@ def perform_action j.delete(@zk) end - elsif @queue == 'queues-acc' - MerrittZK::LegacyAccessJob.list_jobs_as_json(@zk).each do |job| - legacy_delete(job) - end - elsif @queue == 'queues-inv' && ZookeeperListAction.migration_m1? - # no action - elsif @queue == 'queues-inv' - MerrittZK::LegacyInventoryJob.list_jobs_as_json(@zk).each do |job| - legacy_delete(job) - end - elsif ZookeeperListAction.migration_m1? + else ql = QueueList.new(@zk, { deletable: true }) ql.batches.each_key do |bid| puts "Eval Deleting Batch #{bid}" @@ -582,10 +417,6 @@ def perform_action puts "Deleting Batch #{bid}" batch.delete(@zk) end - else - MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk).each do |job| - legacy_delete(job) - end end { redirect_location: "/web/collIndex.html?path=#{@reload_path}" @@ -593,7 +424,7 @@ def perform_action end end -## Control Access Queue Hold/Release (legacy and mrt-zk) +## Control Access Queue Hold/Release class AccessLockAction < ZookeeperAction def initialize(config, action, path, myparams) super @@ -608,22 +439,12 @@ def largeq? def perform_action lockpath = 'tbd' - if ZookeeperListAction.migration_m3? - lockpath = largeq? ? MerrittZK::Locks::LOCKS_QUEUE_ACCESS_LARGE : MerrittZK::Locks::LOCKS_QUEUE_ACCESS_SMALL - case @op - when 'set' - largeq? ? MerrittZK::Locks.lock_large_access_queue(@zk) : MerrittZK::Locks.lock_small_access_queue(@zk) - when 'clear' - largeq? ? MerrittZK::Locks.unlock_large_access_queue(@zk) : MerrittZK::Locks.unlock_small_access_queue(@zk) - end - else - lockpath = "/mrt.lock/access/#{@flag}" - case @op - when 'set' - @zk.create(lockpath, data: nil) unless @zk.exists?(lockpath) - when 'clear' - @zk.delete(lockpath) if @zk.exists?(lockpath) - end + lockpath = largeq? ? MerrittZK::Locks::LOCKS_QUEUE_ACCESS_LARGE : MerrittZK::Locks::LOCKS_QUEUE_ACCESS_SMALL + case @op + when 'set' + largeq? ? MerrittZK::Locks.lock_large_access_queue(@zk) : MerrittZK::Locks.lock_small_access_queue(@zk) + when 'clear' + largeq? ? MerrittZK::Locks.unlock_large_access_queue(@zk) : MerrittZK::Locks.unlock_small_access_queue(@zk) end state = @zk.exists?(lockpath) ? 'Held' : 'Released' message_as_table("Lock #{@op} status result: #{lockpath}=#{state}").to_json @@ -661,7 +482,7 @@ def table_types end def table_rows(_body) - dir = ZookeeperListAction.migration_m1? ? MerrittZK::Locks::LOCKS_STORAGE : '/mrt.lock' + dir = MerrittZK::Locks::LOCKS_STORAGE rows = [] @zk.children(dir).each do |cp| next unless cp =~ /^ark/ diff --git a/src-colladmin/config/actions.yml b/src-colladmin/config/actions.yml index 05d8e42..454709c 100644 --- a/src-colladmin/config/actions.yml +++ b/src-colladmin/config/actions.yml @@ -49,7 +49,6 @@ profiles: documentation: | Ingest: GET /admin/profiles-full Ingest: GET /admin/profile/{profile} - migration: m4 collection-locks: link-title: List Merritt Collections and Collection Level Locks breadcrumb: bp_coll @@ -68,7 +67,6 @@ collection-locks: documentation: | Ingest: GET /admin/profiles-full Ingest: GET /admin/queues - migration: m2 adminprofiles: link-title: List Merritt Admin Profiles breadcrumb: bp_ingest @@ -85,7 +83,6 @@ adminprofiles: documentation: | Ingest: GET /admin/profiles/admin Ingest: GET /admin/profiles/admin/{type} - migration: m4 state: link-title: Ingest Service Status breadcrumb: bp_ingest @@ -97,7 +94,6 @@ state: Display the state of the ingest service. documentation: | Ingest: GET /state - migration: m0 queues: link-title: Ingest Queue Jobs breadcrumb: bp_ingest @@ -115,7 +111,6 @@ queues: documentation: | Ingest: GET /admin/queues Ingest: GET /admin/queue/{node} - migration: m1 batch-queues: link-title: Ingest Queue Jobs breadcrumb: bp_ingest @@ -144,7 +139,6 @@ ingest-queue-by-profile: documentation: | Ingest: GET /admin/queues Ingest: GET /admin/queue/{node} - migration: m1 ingest-queue-by-profile-batch: link-title: Ingest Queue Activity by Profile and Batch breadcrumb: bp_ingest @@ -157,21 +151,6 @@ ingest-queue-by-profile-batch: documentation: | Ingest: GET /admin/queues Ingest: GET /admin/queue/{node} - migration: m1 -inv-queues: - link-title: Inventory Queue Jobs - breadcrumb: bp_inventory - class: InventoryQueueAction - category: Queue Management - sensitivity: readonly - testing: contextually automated - description: | - Display the contents of the *inventory queue*. Items in this queue are slated to be processed by the Ingest service. - From this screen, items can be requeued. - documentation: | - Ingest: GET /admin/queues-inv - Ingest: GET /admin/queue-inv/{node} - migration: m1 acc-queues: link-title: Access Queue Jobs breadcrumb: bp_storage @@ -188,7 +167,6 @@ acc-queues: documentation: | Ingest: GET /admin/queues-acc Ingest: GET /admin/queue-acc/{node} - migration: m1 batchFolders: link-title: Ingest Batch Folders breadcrumb: bp_ingest @@ -209,7 +187,6 @@ batchFolders: Ingest: GET /admin/bids/{days} Ingest: GET /admin/queues Ingest: GET /admin/queue/{node} - migration: m0 batch: link-title: Ingest Batch breadcrumb: bp_ingest @@ -225,7 +202,6 @@ batch: documentation: | Ingest: GET /admin/queues Ingest: GET /admin/queue/{node} - migration: m1 job: link-title: Ingest Job breadcrumb: bp_ingest @@ -237,7 +213,6 @@ job: Display details about a specific *job* within an ingest batch. documentation: | Ingest: GET /admin/jid-erc/{batch}/{job} - migration: m0 manifest: link-title: Ingest Job Manifest Contents breadcrumb: bp_ingest @@ -251,7 +226,6 @@ manifest: - manifest documentation: | Ingest: GET /admin/jid-manifest/{batch}/{job} - migration: m0 files: link-title: Ingest Job Folder File List breadcrumb: bp_ingest @@ -266,7 +240,6 @@ files: - system files will be retained documentation: | Ingest: GET /admin/jid-file/{batch}/{job} - migration: m0 ldap/users: link-title: List Merritt LDAP Users breadcrumb: bp_users @@ -278,7 +251,6 @@ ldap/users: Display the list of Merritt users defined in the Merritt LDAP database. documentation: | Ldap: Query - migration: m0 ldap/user: link-title: List details for an Merritt LDAP User breadcrumb: bp_users @@ -296,7 +268,6 @@ ldap/user: Ldap: Query test_params: uid: merritt_test - migration: m0 ldap/roles: link-title: List Merritt LDAP Roles breadcrumb: bp_users @@ -308,7 +279,6 @@ ldap/roles: Display the set of permission groups (roles) defined in the Merritt LDAP database. documentation: | Ldap: Query - migration: m0 ldap/role: link-title: List details for an Merritt LDAP Role breadcrumb: bp_users @@ -321,7 +291,6 @@ ldap/role: *This report does not seem to be linked in the application**. documentation: | Ldap: Query - migration: m0 ldap/colls: link-title: List Merritt LDAP Collections breadcrumb: bp_users @@ -334,7 +303,6 @@ ldap/colls: Oddly, the Merritt UI uses the Collection name defined in LDAP as the definitive display name for the collection. documentation: | Ldap: Query - migration: m0 ldap/coll: link-title: List Details for an LDAP Collection breadcrumb: bp_users @@ -350,7 +318,6 @@ ldap/coll: - ldapcoll documentation: | Ldap: Query - migration: m0 ldap/collmap: link-title: Merritt LDAP / Inventory Database Mismatches breadcrumb: bp_users @@ -363,7 +330,6 @@ ldap/collmap: For any items that do not match, an error will be reported in the report. This indicates that maintenance needs to be performed on the LDAP entries for Merritt. documentation: | Ldap: Query - migration: m0 ldap/collark: link-title: List LDAP Users for a specific collection ARK breadcrumb: bp_users @@ -379,7 +345,6 @@ ldap/collark: Ldap: Query test_params: ark: ark:/13030/m5rn35s8 - migration: m0 instances: link-title: List and Describe Merritt EC2 Servers breadcrumb: bp_internals @@ -402,7 +367,6 @@ instances: - instances documentation: | Aws: Tag Query - migration: m0 uc3-instances: link-title: List and Describe UC3 EC2 Servers breadcrumb: bp_internals @@ -416,7 +380,6 @@ uc3-instances: - instances documentation: | Aws: Tag Query - migration: m0 ssm-describe: link-title: List and Describe Merritt SSM Variables breadcrumb: bp_internals @@ -429,7 +392,6 @@ ssm-describe: These variables are compared against a [registry of expected SSM variables](https://github.com/CDLUC3/mrt-admin-lambda/blob/main/src-colladmin/config/ssm.registry.yml). documentation: | Aws: SSM Query - migration: m0 opensearch-describe: link-title: List and Describe Merritt OpenSearch Fields breadcrumb: bp_internals @@ -442,7 +404,6 @@ opensearch-describe: These variables are compared against a [TBD registry of expected OpenSearch fields](https://github.com/CDLUC3/mrt-admin-lambda/blob/main/src-colladmin/import/filebeat/opensearch.registry.yml). documentation: | Aws: OpenSearch Query - migration: m0 storage-get-manifest: link-title: Get Storage Manifest for an Object from Cloud Storage class: StorageAction @@ -457,7 +418,6 @@ storage-get-manifest: Download the XML *storage manifest* representation of a object on a specific storage node. This request will be forwarded to the storage service. documentation: | Storage: GET /manifest/{nodenum}/{ark} - migration: m0 storage-get-manifest-yaml: link-title: Get Storage Manifest for an Object from Cloud Storage as a Yaml file class: StorageAction @@ -473,7 +433,6 @@ storage-get-manifest-yaml: Convert the downloaded xml file to a user-friendly yaml format. documentation: | Storage: GET /manifest/{nodenum}/{ark} - migration: m0 storage-get-ingest-checkm: link-title: Generate Ingest Checkm Manifest Using Content from Cloud Storage class: StorageAction @@ -489,7 +448,6 @@ storage-get-ingest-checkm: This file could be utilized to re-ingest an object under a new ark. documentation: | Storage: GET /ingestlink/{nodenum}/{ark}/{ver}?presign=true - migration: m0 cognito-users: link-title: List Merritt Admin Tools Users breadcrumb: bp_users @@ -505,7 +463,6 @@ cognito-users: Note: While the tool supports non-SAML users, in practice, we have not authorized any non-SAML users. documentation: | Aws: Lambda Invoke (Lambda interface to Cognito enpoints) - migration: m0 replication-state: link-title: Display the Status of the Replciation Microservice breadcrumb: bp_replic @@ -519,7 +476,6 @@ replication-state: This function determines whether or not to enable the *Stop Storage Scan* and *Enable Storage Scan* buttons. documentation: | Replication: GET /state - migration: m0 storage-review-csv: link-title: Generate and download a CSV of storage scan exceptions class: ReplicationAction @@ -533,7 +489,6 @@ storage-review-csv: SQL: select inv_storage_maints Aws: S3 PUT Aws: S3 presign - migration: m0 ingest-locks: link-title: Display ingest storage locks on specific object arks breadcrumb: bp_internals @@ -545,7 +500,6 @@ ingest-locks: Display the ingest storage locks existing for object arks. documentation: | Ingest: GET /admin/lock/mrt.lock - migration: m1 # Operations that change system state - Reversible @@ -562,7 +516,6 @@ submissions/pause: This is typically done before a patching cycle or a software release that impacts ingest or storage. documentation: | Ingest: POST /admin/submissions/freeze - migration: m2 submissions/unpause: link-title: Unpause all ingest processing breadcrumb: bp_ingest @@ -575,7 +528,6 @@ submissions/unpause: Unpause ingest processing. Return the effective state of the ingest queues. documentation: | Ingest: POST /admin/submissions/thaw - migration: m2 ing-lock: link-title: Pause all ingest processing breadcrumb: bp_ingest @@ -588,7 +540,6 @@ ing-lock: This is typically done before a patching cycle or a software release that impacts ingest or storage. documentation: | ZK - migration: m2 ing-unlock: link-title: Unpause all ingest processing breadcrumb: bp_ingest @@ -600,7 +551,6 @@ ing-unlock: Unpause ingest processing. Return the effective state of the ingest queues. documentation: | ZK - migration: m2 unlock-coll: link-title: Unlock ingest for a collection class: CollQueueAction @@ -614,7 +564,6 @@ unlock-coll: - colllock documentation: | Ingest: POST /admin/submission/thaw/{coll} - migration: m2 lock-coll: link-title: Lock ingest for a collection class: CollQueueAction @@ -628,7 +577,6 @@ lock-coll: - colllock documentation: | Ingest: POST /admin/submission/freeze/{coll} - migration: m2 unlock-collzk: link-title: Unlock ingest for a collection class: CollUnlockZkAction @@ -641,7 +589,6 @@ unlock-collzk: - colllock documentation: | ZK - migration: m2 lock-collzk: link-title: Lock ingest for a collection class: CollLockZkAction @@ -654,23 +601,9 @@ lock-collzk: - colllock documentation: | ZK - migration: m2 -release-coll-queue-items-m1: +release-coll-queue-items: link-title: Release any held items for a collection - class: CollIterateQueueM1Action - category: Queue Management - sensitivity: reversible change - testing: automatable - description: | - Release any held items for a collection. - report-datatypes: - - collqitems - documentation: | - Ingest: POST /admin/release-all/queue/{coll} - migration: m1 -release-coll-queue-items-legacy: - link-title: Release any held items for a collection (legacy) - class: CollIterateQueueLegacyAction + class: CollIterateQueueAction category: Queue Management sensitivity: reversible change testing: automatable @@ -680,7 +613,6 @@ release-coll-queue-items-legacy: - collqitems documentation: | Ingest: POST /admin/release-all/queue/{coll} - migration: m1 storage-force-audit-for-object: link-title: Trigger audit processing for all files in an object on a specific node class: StorageAction @@ -691,7 +623,6 @@ storage-force-audit-for-object: Trigger audit processing for all files in an object on a specific node. documentation: | SQL: update inv_audits set verified - migration: m0 storage-rerun-audit-for-object: link-title: Trigger audit processing for all unverified files in an object on a specific node class: StorageAction @@ -703,7 +634,6 @@ storage-rerun-audit-for-object: This action is occasionally needed when the audit server is halted or experienced a networking issue on a previous audit. documentation: | SQL: update inv_audits set verified - migration: m0 storage-force-replic-for-object: link-title: Re-trigger replication for an existing object class: StorageAction @@ -715,7 +645,6 @@ storage-force-replic-for-object: This action is occasionally needed when the replic server is halted or experienced a networking issue on a previous replication. documentation: | SQL: update inv_nodes_inv_objects inio set replicated - migration: m0 storage-clear-audit-batch: link-title: Release old audit batches class: StorageAction @@ -728,7 +657,6 @@ storage-clear-audit-batch: This function clears all audit batches older than a specified amount of time. documentation: | SQL: UPDATE inv_audits SET status='unknown', verified=null - migration: m0 storage-retry-audit-status: link-title: Trigger an audit retry for all audit records with a specific status class: StorageAction @@ -739,7 +667,6 @@ storage-retry-audit-status: Trigger an audit retry for all audit records with a specific status. documentation: | SQL: UPDATE inv_audits SET status='unknown', verified=null - migration: m0 storage-cancel-all-scans: link-title: Halt all storage scan processing by the replciation service class: ReplicationAction @@ -750,7 +677,6 @@ storage-cancel-all-scans: Halt all storage scan processing by the replciation service. This will give priority to content replication. documentation: | Replication: POST /scan/allow/false - migration: m0 storage-allow-all-scans: link-title: Re-enable storage scan processing by the replciation service class: ReplicationAction @@ -761,7 +687,6 @@ storage-allow-all-scans: Re-enable storage scan processing by the replciation service. This can be run after halting/canceling all scans. documentation: | Replication: POST /scan/allow/true - migration: m0 storage-cancel-scan-node: link-title: Cancel storage scan processing on a specific storage node by the replication service class: ReplicationAction @@ -772,7 +697,6 @@ storage-cancel-scan-node: Cancel storage scan processing *on a specific storage node* by the replication service. documentation: | Replication: POST /scan/cancel/{scanid} - migration: m0 storage-resume-scan-node: link-title: Resume storage scan processing on a specific storage node by the replication service class: ReplicationAction @@ -783,7 +707,6 @@ storage-resume-scan-node: Resume storage scan processing *on a specific storage node* by the replication service. documentation: | Replication: POST /scan/restart/{scanid} - migration: m0 storage-clear-scan-entries: link-title: Clear storage scan entries for a specific ark class: StorageAction @@ -795,7 +718,6 @@ storage-clear-scan-entries: This function is intended to be run after *rebuilding an inventory record* via the admin tool. documentation: | SQL: DELETE FROM inv_storage_maints WHERE s3key like - migration: m0 storage-scan-node: link-title: Initiate storage scan processing on a specific storage node by the replication service class: ReplicationAction @@ -810,7 +732,6 @@ storage-scan-node: Instead, an inventory dump must be manually requested from SDSC and uploaded to S3 for comparison processing. documentation: | Replication: POST /scan/start/{nodeid} - migration: m0 storage-delete-node-key: link-title: Mark a single scan result with a DELETE status class: ReplicationAction @@ -821,7 +742,6 @@ storage-delete-node-key: Mark a single scan result with a *DELETE* status. documentation: | SQL: UPDATE inv_storage_maints SET maint_status - migration: m0 storage-delete-node-page: link-title: Mark a page of scan results with a DELETE status class: ReplicationAction @@ -832,7 +752,6 @@ storage-delete-node-page: Mark a page of scan results with a *DELETE* status. documentation: | SQL: UPDATE inv_storage_maints SET maint_status - migration: m0 storage-hold-node-key: link-title: Mark a single scan result with a HOLD status class: ReplicationAction @@ -843,7 +762,6 @@ storage-hold-node-key: Mark a single scan result with a *HOLD* status. documentation: | SQL: UPDATE inv_storage_maints SET maint_status - migration: m0 storage-hold-node-page: link-title: Mark a page of scan results with a HOLD status class: ReplicationAction @@ -854,7 +772,6 @@ storage-hold-node-page: Mark a page of scan results with a *HOLD* status. documentation: | SQL: UPDATE inv_storage_maints SET maint_status - migration: m0 storage-review-node-key: link-title: Mark a single scan result with a REVIEW status class: ReplicationAction @@ -865,7 +782,6 @@ storage-review-node-key: Mark a single scan result with a *REVIEW* status. documentation: | SQL: UPDATE inv_storage_maints SET maint_status - migration: m0 storage-review-node-page: link-title: Mark a page of scan results with a REVIEW status class: ReplicationAction @@ -876,7 +792,6 @@ storage-review-node-page: Mark a page of scan results with a *REVIEW* status. documentation: | SQL: UPDATE inv_storage_maints SET maint_status - migration: m0 # Operations that change system state - Not reversible @@ -892,7 +807,6 @@ submit-profile: This will trigger the creation of an object in the Merritt inventory database. documentation: | Ingest: POST /poster/update - migration: m0 set_mnemonic: link-title: Set the mnemonic field for a collection to match the value defined in the collection admin profile class: AdminProfileAction @@ -905,7 +819,6 @@ set_mnemonic: This option must be manually run due to a legacy limitation of the collection creation process for Merritt. documentation: | SQL: update inv_collections set mnemonic - migration: m0 set_coll_name: link-title: Set the description field for a collection to match the value defined in the collection admin profile class: AdminProfileAction @@ -917,7 +830,6 @@ set_coll_name: To apply future changes to this field, you must update the collection admin profile. documentation: | SQL: update inv_collections set name - migration: m0 set_sla_name: link-title: Set the description field for an SLA to match the value defined in the SLA admin profile class: AdminProfileAction @@ -929,7 +841,6 @@ set_sla_name: To apply future changes to this field, you must update the SLA admin profile. documentation: | SQL: update inv_collections set name - migration: m0 set_own_name: link-title: Set the description field for an Owner to match the value defined in the Owner admin profile class: AdminProfileAction @@ -941,7 +852,6 @@ set_own_name: To apply future changes to this field, you must update the Owner admin profile. documentation: | SQL: update inv_owners set name= - migration: m0 create_owner_record: link-title: Generate an owner record in the Merritt inventory database based on the submission of a dummy object using the Owner Admin profile class: AdminProfileAction @@ -952,7 +862,6 @@ create_owner_record: Generate an owner record in the Merritt inventory database based on the submission of a dummy object using the Owner Admin profile. documentation: | SQL: insert into inv_owners(inv_object_id, ark) - migration: m0 create_coll_record: link-title: Generate an SLA record in the Merritt inventory database based on the submission of a dummy object using the SLA Admin profile class: AdminProfileAction @@ -963,7 +872,6 @@ create_coll_record: Generate an SLA record in the Merritt inventory database based on the submission of a dummy object using the SLA Admin profile. documentation: | SQL: insert into inv_collections(inv_object_id, ark) - migration: m0 createProfile/profile: link-title: Send a request to the ingest service to generate a new Ingest Profile class: PostToIngestMultipartAction @@ -976,7 +884,6 @@ createProfile/profile: A new ARK will be minted as needed. documentation: | Ingest: POST /admin/profile/profile - migration: m5 createProfile/collection: link-title: Send a request to the ingest service to generate a new Collection Admin Profile class: PostToIngestMultipartAction @@ -989,7 +896,6 @@ createProfile/collection: A new ARK will be minted as needed. documentation: | Ingest: POST /admin/profile/collection - migration: m5 createProfile/owner: link-title: Send a request to the ingest service to generate a new Owner Admin Profile class: PostToIngestMultipartAction @@ -1002,7 +908,6 @@ createProfile/owner: A new ARK will be minted as needed. documentation: | Ingest: POST /admin/profile/owner - migration: m5 createProfile/sla: link-title: Send a request to the ingest service to generate a new SLA Admin Profile class: PostToIngestMultipartAction @@ -1015,10 +920,9 @@ createProfile/sla: A new ARK will be minted as needed. documentation: | Ingest: POST /admin/profile/sla - migration: m5 queue-delete-mrtzk: link-title: Remove an item from a Zookeeper queue - class: ZkDeleteM1Action + class: ZkDeleteAction category: Queue Management sensitivity: irreversible change testing: manual @@ -1028,7 +932,6 @@ queue-delete-mrtzk: - qdelete documentation: | Zk: tbd - migration: m1 queue-delete-batch-mrtzk: link-title: Remove an item from a Zookeeper queue class: ZkDeleteBatchAction @@ -1041,7 +944,6 @@ queue-delete-batch-mrtzk: - qdelete documentation: | Zk: tbd - migration: m1 queue-update-batch-mrtzk: link-title: Update a batch from a Zookeeper queue class: ZkUpdateReportingBatchAction @@ -1054,23 +956,9 @@ queue-update-batch-mrtzk: - qdelete documentation: | Zk: tbd - migration: m1 -queue-delete-legacy: - link-title: Remove an item from a Zookeeper queue - class: ZkDeleteLegacyAction - category: Queue Management - sensitivity: irreversible change - testing: manual - description: | - Remove an item from a Zookeeper queue. - report-datatypes: - - qdelete - documentation: | - Zk: tbd - migration: m1 requeue-mrtzk: link-title: Re-queue an item from a Zookeeper queue - class: ZkRequeueM1Action + class: ZkRequeueAction category: Queue Management sensitivity: irreversible change testing: manual @@ -1080,23 +968,9 @@ requeue-mrtzk: - requeue documentation: | ZK tbd - migration: m1 -requeue-legacy: - link-title: Re-queue an item from a Zookeeper queue - class: ZkRequeueLegacyAction - category: Queue Management - sensitivity: irreversible change - testing: manual - description: | - Re-queue an item from a Zookeeper queue. - report-datatypes: - - requeue - documentation: | - ZK tbd - migration: m1 hold-queue-item-mrtzk: link-title: Move a pending item from in Zookeeper queue to a Held status - class: ZkHoldM1Action + class: ZkHoldAction category: Queue Management sensitivity: irreversible change testing: manual @@ -1106,36 +980,9 @@ hold-queue-item-mrtzk: - hold documentation: | Zk: tbd - migration: m1 -hold-queue-item-legacy: - link-title: Move a pending item from in Zookeeper queue to a Held status - class: ZkHoldLegacyAction - category: Queue Management - sensitivity: irreversible change - testing: manual - description: | - Move a pending item from in Zookeeper queue to a Held status. - report-datatypes: - - hold - documentation: | - Zk: tbd - migration: m1 release-queue-item-mrtzk: link-title: Release a held item from in Zookeeper queue to a Pending status - class: ZkReleaseM1Action - category: Queue Management - sensitivity: irreversible change - testing: manual - description: | - Release a held item from in Zookeeper queue to a Pending status. - report-datatypes: - - release - documentation: | - Zk: tbd - migration: m1 -release-queue-item-legacy: - link-title: Release a held item from in Zookeeper queue to a Pending status - class: ZkReleaseLegacyAction + class: ZkReleaseAction category: Queue Management sensitivity: irreversible change testing: manual @@ -1145,7 +992,6 @@ release-queue-item-legacy: - release documentation: | Zk: tbd - migration: m1 cleanup-queue: link-title: Cleanup Deleted and Completed items from a Zookeeper Queue class: IterateQueueAction @@ -1158,7 +1004,6 @@ cleanup-queue: - release documentation: | Ingest: POST /admin/cleanupq/{queue} - migration: m1 apply-review-changes: link-title: Apply bulk changes to storage scan results by uploading a CSV containing a set of desired changes class: ReplicationAction @@ -1169,7 +1014,6 @@ apply-review-changes: Apply bulk changes to storage scan results by uploading a CSV containing a set of desired changes. documentation: | SQL: update inv_storage_maints set maint_status = ?, note = ? - migration: m0 # Sensitive Operations @@ -1185,7 +1029,6 @@ cognito-remove-user-from-group: - cognito documentation: | Aws: invoke lambda - migration: m0 cognito-add-user-to-group: link-title: Add a Cognito User to a specific Cognito User Group class: CognitoAction @@ -1198,7 +1041,6 @@ cognito-add-user-to-group: - cognito documentation: | Aws: invoke lambda - migration: m0 storage-rebuild-inventory: link-title: Rebuild inventory record from storage manifest class: StorageAction @@ -1212,7 +1054,6 @@ storage-rebuild-inventory: Inventory: DELETE /object/{ark} Storage: GET /manifest/{nodenum}/{ark} Inventory: POST /add - migration: m0 storage-add-node-for-collection: link-title: Add a secondary storage node to the configuration for a collection class: StorageAction @@ -1224,7 +1065,6 @@ storage-add-node-for-collection: documentation: | SQL: INSERT INTO inv_collections_inv_nodes(inv_collection_id, inv_node_id) SQL: update inv_nodes_inv_objects inio set replicated = null - migration: m0 storage-del-node-for-collection: link-title: Remove a storage node from a collection configuration class: StorageAction @@ -1236,7 +1076,6 @@ storage-del-node-for-collection: All objects in the collection will need to be cleared from the storage node. documentation: | SQL: DELETE FROM inv_collections_inv_nodes WHERE inv_collection_id = ? - migration: m0 access-lock-control: link-title: Storage set zookeeper flag class: AccessLockAction @@ -1247,7 +1086,6 @@ access-lock-control: Change the state of a zookeeper flag (SET or CLEAR). documentation: | ZK: set boolean flag - migration: m3 replic-delete-coll-batch-from-node: link-title: Delete a batch of objects from the collection from storage node class: ReplicationAction @@ -1259,7 +1097,6 @@ replic-delete-coll-batch-from-node: documentation: | SQL: SELECT o.ark from inv_objects o Replication: DELETE /delete/{nodenum}/{ark} - migration: m0 storage-del-object-from-node: link-title: Delete object from storage node implemented: false @@ -1271,7 +1108,6 @@ storage-del-object-from-node: [Feature Design](https://cdluc3.github.io/mrt-doc/diagrams/store-admin-del-node-obj) documentation: | Not yet implemented - migration: m0 storage-change-primary-for-collection: link-title: Change primary storage node for collection implemented: false @@ -1283,7 +1119,6 @@ storage-change-primary-for-collection: [Feature Design](https://cdluc3.github.io/mrt-doc/diagrams/store-admin-change-primary-node) documentation: | Not yet implemented - migration: m0 storage-perform-delete-node-key: link-title: Delete copy of a file from a storage node class: ReplicationAction @@ -1294,7 +1129,6 @@ storage-perform-delete-node-key: Process the delete of a single storage scan record. documentation: | Replication: DELETE /scandelete/{maintid} - migration: m0 storage-perform-delete-node-batch: link-title: Process all scan results marked with a DELETE status class: ReplicationAction @@ -1305,7 +1139,6 @@ storage-perform-delete-node-batch: Process all scan results marked with a *DELETE* status. documentation: | Replication: DELETE /scandelete-list/{nodenum} - migration: m0 storage-delete-obj: link-title: Trigger the deletion of a Merritt Object implemented: false @@ -1318,7 +1151,6 @@ storage-delete-obj: [Feature Design](https://cdluc3.github.io/mrt-doc/diagrams/store-admin-del-obj) documentation: | Not yet implemented - migration: m0 consistency-report-cleanup: link-title: Cleanup Old Consistency Reports class: ConsistencyReportCleanupAction @@ -1331,7 +1163,6 @@ consistency-report-cleanup: A maximum of 5000 reports will be deleted in one invocation. documentation: | S3: DELETE consistency-reports/... - migration: m0 coll_lambda_tag: link-title: Lambda Tag Check class: LambdaTagAction diff --git a/src-colladmin/lambda_function.rb b/src-colladmin/lambda_function.rb index cc4d1c0..dc1d35f 100644 --- a/src-colladmin/lambda_function.rb +++ b/src-colladmin/lambda_function.rb @@ -6,7 +6,6 @@ require_relative 'actions/forward_to_ingest_action' require_relative 'actions/ingest_queue_action' require_relative 'actions/ingest_queue_profile_action' -require_relative 'actions/inventory_queue_action' require_relative 'actions/access_queue_action' require_relative 'actions/ingest_state_action' require_relative 'actions/ingest_profile_action' diff --git a/src-colladmin/lib/acc_queue.rb b/src-colladmin/lib/acc_queue.rb index 06eb160..595b66b 100644 --- a/src-colladmin/lib/acc_queue.rb +++ b/src-colladmin/lib/acc_queue.rb @@ -93,13 +93,8 @@ def self.table_types type = 'name' if sym == :token type = 'bytes' if sym == :bytes type = 'datetime' if sym == :date - if ZookeeperListAction.migration_m3? - type = 'qdelete-mrtzk' if sym == :qdelete - type = 'requeue-mrtzk' if sym == :requeue - else - type = 'qdelete-legacy' if sym == :qdelete - type = 'requeue-legacy' if sym == :requeue - end + type = 'qdelete-mrtzk' if sym == :qdelete + type = 'requeue-mrtzk' if sym == :requeue type = 'zkacc' if sym == :queueId arr.append(type) end diff --git a/src-colladmin/lib/inv_queue.rb b/src-colladmin/lib/inv_queue.rb index 52ea974..3a9609e 100644 --- a/src-colladmin/lib/inv_queue.rb +++ b/src-colladmin/lib/inv_queue.rb @@ -80,14 +80,8 @@ def self.table_types type = '' type = 'status' if sym == :status type = 'datetime' if sym == :date - if ZookeeperListAction.migration_m1? - # under m1 migration, there is no INV queue - type = 'qdelete-mrtzk' if sym == :qdelete - type = 'requeue-mrtzk' if sym == :requeue - else - type = 'qdelete-legacy' if sym == :qdelete - type = 'requeue-legacy' if sym == :requeue - end + type = 'qdelete-mrtzk' if sym == :qdelete + type = 'requeue-mrtzk' if sym == :requeue arr.append(type) end arr diff --git a/src-colladmin/lib/queue.rb b/src-colladmin/lib/queue.rb index 0ea2db2..23552d4 100644 --- a/src-colladmin/lib/queue.rb +++ b/src-colladmin/lib/queue.rb @@ -129,17 +129,10 @@ def self.table_types type = 'qjob' if sym == :job type = 'status' if sym == :status type = 'datetime' if sym == :date - if ZookeeperListAction.migration_m1? - type = 'qdelete-mrtzk' if sym == :qdelete - type = 'requeue-mrtzk' if sym == :requeue - type = 'hold-mrtzk' if sym == :hold - type = 'release-mrtzk' if sym == :release - else - type = 'qdelete-legacy' if sym == :qdelete - type = 'requeue-legacy' if sym == :requeue - type = 'hold-legacy' if sym == :hold - type = 'release-legacy' if sym == :release - end + type = 'qdelete-mrtzk' if sym == :qdelete + type = 'requeue-mrtzk' if sym == :requeue + type = 'hold-mrtzk' if sym == :hold + type = 'release-mrtzk' if sym == :release type = 'container' if sym == :queue type = 'zkjob' if sym == :queueId arr.append(type) @@ -198,7 +191,7 @@ def date end def get_queue_node - ZookeeperListAction.migration_m1? ? '/jobs' : '/ingest' + '/jobs' end end @@ -375,11 +368,7 @@ def initialize(zk, filter = {}) @filter = filter jobs = [] - if ZookeeperListAction.migration_m1? - jobs = MerrittZK::Job.list_jobs_as_json(zk) - else - jobs = MerrittZK::LegacyIngestJob.list_jobs_as_json(zk) - end + jobs = MerrittZK::Job.list_jobs_as_json(zk) jobs.each do |j| job = QueueEntry.new(j) next if @filter.key?(:batch) && job.bid != @filter[:batch] diff --git a/src-colladmin/lib/queue_json.rb b/src-colladmin/lib/queue_json.rb index 91ae0d8..fb23070 100644 --- a/src-colladmin/lib/queue_json.rb +++ b/src-colladmin/lib/queue_json.rb @@ -14,11 +14,7 @@ def path def get_del_queue_path_m1 st = get_value(:qstatus, '') - if ZookeeperListAction.migration_m1? - return '' unless %w[Failed Held].include?(st) - else - return '' unless %w[Failed Completed Held].include?(st) - end + return '' unless %w[Failed Held].include?(st) path end diff --git a/src-colladmin/web/describeActions.html b/src-colladmin/web/describeActions.html index c855c13..c54455a 100644 --- a/src-colladmin/web/describeActions.html +++ b/src-colladmin/web/describeActions.html @@ -46,17 +46,6 @@

Action Index

Migration Levels

- -
diff --git a/src-colladmin/web/storeCollNode.html b/src-colladmin/web/storeCollNode.html index a7c7cc6..356c109 100644 --- a/src-colladmin/web/storeCollNode.html +++ b/src-colladmin/web/storeCollNode.html @@ -41,10 +41,6 @@

Collection Ingest Processing Status

{{profname}}: {{ingest_locks}} {{#ingest_unpaused}} - -