From 90d86b58c21773b36d46ef6ef76e55625a75fe81 Mon Sep 17 00:00:00 2001 From: Pete Matsyburka Date: Tue, 22 Oct 2024 23:40:30 +0300 Subject: [PATCH] adjust migration --- app/jobs/process_submitter_completion_job.rb | 25 +++++++----- app/models/completed_document.rb | 2 + app/models/completed_submitter.rb | 8 +++- ...eate_completed_submitters_and_documents.rb | 2 +- ...late_completed_submitters_and_documents.rb | 40 +++++++++++-------- db/schema.rb | 1 + 6 files changed, 48 insertions(+), 30 deletions(-) diff --git a/app/jobs/process_submitter_completion_job.rb b/app/jobs/process_submitter_completion_job.rb index 8ca357e5..56ed4c5d 100644 --- a/app/jobs/process_submitter_completion_job.rb +++ b/app/jobs/process_submitter_completion_job.rb @@ -6,8 +6,6 @@ class ProcessSubmitterCompletionJob def perform(params = {}) submitter = Submitter.find(params['submitter_id']) - create_completed_submitter!(submitter) - is_all_completed = !submitter.submission.submitters.exists?(completed_at: nil) if !is_all_completed && submitter.submission.submitters_order_preserved? @@ -26,31 +24,36 @@ class ProcessSubmitterCompletionJob enqueue_completed_emails(submitter) end - create_completed_documents!(submitter) + create_completed_submitter!(submitter) enqueue_completed_webhooks(submitter, is_all_completed:) end def create_completed_submitter!(submitter) + completed_submitter = CompletedSubmitter.find_or_initialize_by(submitter_id: submitter.id) + + return completed_submitter if completed_submitter.persisted? + submission = submitter.submission - sms_count = submitter.submission_events.where(event_type: %w[send_sms send_2fa_sms]).count - completed_submitter = CompletedSubmitter.where(submitter_id: submitter.id).first_or_initialize + completed_submitter.assign_attributes( submission_id: submitter.submission_id, account_id: submission.account_id, template_id: submission.template_id, source: submission.source, - sms_count:, + sms_count: submitter.submission_events.where(event_type: %w[send_sms send_2fa_sms]).count, completed_at: submitter.completed_at ) - completed_submitter.save! - end + submitter.documents.each do |attachment| + next if attachment.metadata['sha256'].blank? - def create_completed_documents!(submitter) - submitter.documents.map { |s| s.metadata['sha256'] }.compact_blank.each do |sha256| - CompletedDocument.where(submitter_id: submitter.id, sha256:).first_or_create! + completed_submitter.completed_documents << CompletedDocument.new(sha256: attachment.metadata['sha256']) end + + completed_submitter.save! + + completed_submitter end def enqueue_completed_webhooks(submitter, is_all_completed: false) diff --git a/app/models/completed_document.rb b/app/models/completed_document.rb index fb4bac39..2c842e29 100644 --- a/app/models/completed_document.rb +++ b/app/models/completed_document.rb @@ -17,4 +17,6 @@ # class CompletedDocument < ApplicationRecord belongs_to :submitter + + has_one :completed_submitter, primary_key: :submitter_id, inverse_of: :completed_documents, dependent: :destroy end diff --git a/app/models/completed_submitter.rb b/app/models/completed_submitter.rb index 1018baf0..181f2241 100644 --- a/app/models/completed_submitter.rb +++ b/app/models/completed_submitter.rb @@ -17,11 +17,17 @@ # # Indexes # -# index_completed_submitters_on_account_id (account_id) +# index_completed_submitters_on_account_id (account_id) +# index_completed_submitters_on_submitter_id (submitter_id) # class CompletedSubmitter < ApplicationRecord belongs_to :submitter belongs_to :submission belongs_to :account belongs_to :template + + has_many :completed_documents, dependent: :destroy, + primary_key: :submitter_id, + foreign_key: :submitter_id, + inverse_of: :submitter end diff --git a/db/migrate/20241018115034_create_completed_submitters_and_documents.rb b/db/migrate/20241018115034_create_completed_submitters_and_documents.rb index e804b436..eb917606 100644 --- a/db/migrate/20241018115034_create_completed_submitters_and_documents.rb +++ b/db/migrate/20241018115034_create_completed_submitters_and_documents.rb @@ -3,7 +3,7 @@ class CreateCompletedSubmittersAndDocuments < ActiveRecord::Migration[7.2] def change create_table :completed_submitters do |t| - t.bigint :submitter_id, null: false + t.bigint :submitter_id, null: false, index: true t.bigint :submission_id, null: false t.bigint :account_id, null: false, index: true t.bigint :template_id, null: false diff --git a/db/migrate/20241022125135_populate_completed_submitters_and_documents.rb b/db/migrate/20241022125135_populate_completed_submitters_and_documents.rb index 01acd4aa..c2c3dd57 100644 --- a/db/migrate/20241022125135_populate_completed_submitters_and_documents.rb +++ b/db/migrate/20241022125135_populate_completed_submitters_and_documents.rb @@ -7,7 +7,8 @@ class PopulateCompletedSubmittersAndDocuments < ActiveRecord::Migration[7.2] self.table_name = 'submitters' belongs_to :submission, class_name: 'MigrationSubmission' - has_many :submission_events, class_name: 'MigrationSubmissionEvent', foreign_key: :submitter_id + has_many :submission_sms_events, -> { where(event_type: %w[send_sms send_2fa_sms]) }, + class_name: 'MigrationSubmissionEvent', foreign_key: :submitter_id end class MigrationSubmission < ApplicationRecord @@ -27,18 +28,26 @@ class PopulateCompletedSubmittersAndDocuments < ActiveRecord::Migration[7.2] end def up - completed_submitters = MigrationSubmitter.where.not(completed_at: nil) + submitters = MigrationSubmitter.where.not(completed_at: nil) + .preload(:submission, :submission_sms_events) + + count = submitters.count + + puts "Updating the database - it might take ~#{(count / 1000 * 3) + 1} seconds" if count > 2000 + + submitters.find_each do |submitter| + completed_submitter = MigrationCompletedSubmitter.find_or_initialize_by(submitter_id: submitter.id) + + next if completed_submitter.persisted? - completed_submitters.order(created_at: :asc).preload(:submission).find_each do |submitter| submission = submitter.submission - sms_count = submitter.submission_events.where(event_type: %w[send_sms send_2fa_sms]).count - completed_submitter = MigrationCompletedSubmitter.where(submitter_id: submitter.id).first_or_initialize + completed_submitter.assign_attributes( submission_id: submitter.submission_id, account_id: submission.account_id, template_id: submission.template_id, source: submission.source, - sms_count:, + sms_count: submitter.submission_sms_events.size, completed_at: submitter.completed_at, created_at: submitter.completed_at, updated_at: submitter.completed_at @@ -47,21 +56,18 @@ class PopulateCompletedSubmittersAndDocuments < ActiveRecord::Migration[7.2] completed_submitter.save! end - ActiveStorage::Attachment.where(record_id: completed_submitters.select(:id), - record_type: 'Submitter', - name: 'documents') - .order(created_at: :asc) - .find_each do |attachment| + attachments = ActiveStorage::Attachment.where(record_type: 'Submitter', name: 'documents').preload(:blob) + + attachments.find_each do |attachment| sha256 = attachment.metadata['sha256'] - submitter_id = attachment.record_id next if sha256.blank? - completed_document = MigrationCompletedDocument.where(submitter_id:, sha256:).first_or_initialize - completed_document.assign_attributes( - created_at: attachment.created_at, - updated_at: attachment.created_at - ) + completed_document = MigrationCompletedDocument.find_or_initialize_by(submitter_id: attachment.record_id, sha256:) + + next if completed_document.persisted? + + completed_document.assign_attributes(created_at: attachment.created_at, updated_at: attachment.created_at) completed_document.save! end diff --git a/db/schema.rb b/db/schema.rb index 79b893b8..55651476 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -109,6 +109,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_22_125135) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_completed_submitters_on_account_id" + t.index ["submitter_id"], name: "index_completed_submitters_on_submitter_id" end create_table "document_generation_events", force: :cascade do |t|