use submission completed_at

app/controllers/api/submission_documents_controller.rb

@@ -9,7 +9,7 @@ module Api
                  (@submission.schema_documents || @submission.template.schema_documents).size > 1
 
       documents =
-        if @submission.submitters.all?(&:completed_at?)
+        if @submission.completed_at?
           build_completed_documents(@submission, merge: is_merge)
         else
           build_preview_documents(@submission, merge: is_merge)

app/controllers/api/submission_events_controller.rb

@@ -5,7 +5,7 @@ module Api
     load_and_authorize_resource :submission, parent: false
 
     def index
-      submissions = build_completed_query(@submissions)
+      submissions = @submissions.active.where.not(completed_at: nil)
 
       params[:after] = Time.zone.at(params[:after].to_i) if params[:after].present?
       params[:before] = Time.zone.at(params[:before].to_i) if params[:before].present?
@@ -36,20 +36,5 @@ module Api
         }
       }
     end
-
-    private
-
-    def build_completed_query(submissions)
-      submissions = submissions.where(
-        Submitter.where(completed_at: nil).where(
-          Submitter.arel_table[:submission_id].eq(Submission.arel_table[:id])
-        ).select(1).arel.exists.not
-      )
-
-      submissions.joins(:submitters)
-                 .group(:id)
-                 .select(Submission.arel_table[Arel.star],
-                         Submitter.arel_table[:completed_at].maximum.as('completed_at'))
-    end
   end
 end

app/controllers/api/submissions_controller.rb

@@ -2,7 +2,7 @@
 
 module Api
   class SubmissionsController < ApiBaseController
-    SUBMISSION_COLUMNS = %i[id name slug source submitters_order expire_at created_at updated_at
+    SUBMISSION_COLUMNS = %i[id name slug source submitters_order expire_at completed_at created_at updated_at
                             archived_at variables template_id template_submitters created_by_user_id].freeze
     TEMPLATE_COLUMNS = %i[id name external_id created_at updated_at folder_id submitters].freeze
 
@@ -13,6 +13,8 @@ module Api
       authorize!(:create, Submission)
     end
 
+    before_action :maybe_return_template_error, only: :create
+
     def index
       submissions = Submissions.search(current_user, @submissions, params[:q])
       submissions = filter_submissions(submissions, params)
@@ -58,7 +60,7 @@ module Api
         end
       end
 
-      if @submission.audit_trail_attachment.blank? && submitters.all?(&:completed_at?)
+      if @submission.audit_trail_attachment.blank? && @submission.completed_at?
         @submission.audit_trail_attachment = Submissions::EnsureAuditGenerated.call(@submission)
       end
 
@@ -68,20 +70,6 @@ module Api
     def create
       Params::SubmissionCreateValidator.call(params)
 
-      return render json: { error: 'Template not found' }, status: :unprocessable_content if @template.nil?
-
-      if @template.archived_at?
-        Rollbar.warning("Archived template submission: #{@template.id}") if defined?(Rollbar)
-
-        return render json: { error: 'Template has been archived' }, status: :unprocessable_content
-      end
-
-      if @template.fields.blank?
-        Rollbar.warning("Template does not contain fields: #{@template.id}") if defined?(Rollbar)
-
-        return render json: { error: 'Template does not contain fields' }, status: :unprocessable_content
-      end
-
       params[:send_email] = true unless params.key?(:send_email)
       params[:send_sms] = false unless params.key?(:send_sms)
 
@@ -129,6 +117,22 @@ module Api
 
     private
 
+    def maybe_return_template_error
+      return render json: { error: 'Template not found' }, status: :unprocessable_content if @template.nil?
+
+      if @template.archived_at?
+        Rollbar.warning("Archived template submission: #{@template.id}") if defined?(Rollbar)
+
+        return render json: { error: 'Template has been archived' }, status: :unprocessable_content
+      end
+
+      return if @template.fields.present?
+
+      Rollbar.warning("Template does not contain fields: #{@template.id}") if defined?(Rollbar)
+
+      render json: { error: 'Template does not contain fields' }, status: :unprocessable_content
+    end
+
     def filter_submissions(submissions, params)
       submissions = submissions.where(template_id: params[:template_id]) if params[:template_id].present?
       submissions = submissions.where(slug: params[:slug]) if params[:slug].present?

app/controllers/api/submitters_controller.rb

@@ -4,6 +4,8 @@ module Api
   class SubmittersController < ApiBaseController
     load_and_authorize_resource :submitter
 
+    before_action :maybe_return_submitter_error, only: :update
+
     def index
       submitters = Submitters.search(current_user, @submitters, params[:q])
 
@@ -36,14 +38,6 @@ module Api
 
     # rubocop:disable Metrics/MethodLength
     def update
-      if @submitter.completed_at?
-        return render json: { error: 'Submitter has already completed the submission.' }, status: :unprocessable_content
-      end
-
-      if @submitter.declined_at?
-        return render json: { error: 'Submitter has already declined the submission.' }, status: :unprocessable_content
-      end
-
       submission = @submitter.submission
       role = submission.template_submitters.find { |e| e['uuid'] == @submitter.uuid }['name']
 
@@ -106,6 +100,16 @@ module Api
 
     private
 
+    def maybe_return_submitter_error
+      if @submitter.completed_at?
+        return render json: { error: 'Submitter has already completed the submission.' }, status: :unprocessable_content
+      end
+
+      return unless @submitter.declined_at?
+
+      render json: { error: 'Submitter has already declined the submission.' }, status: :unprocessable_content
+    end
+
     def maybe_filter_by_completed_at(submitters, params)
       if params[:completed_after].present?
         submitters = submitters.where(completed_at: Time.zone.parse(params[:completed_after])..)

app/controllers/submissions_archived_controller.rb

@@ -12,8 +12,9 @@ class SubmissionsArchivedController < ApplicationController
     @submissions = Submissions.search(current_user, @submissions, params[:q], search_template: true)
     @submissions = Submissions::Filter.call(@submissions, current_user, params)
 
-    @submissions = if params[:completed_at_from].present? || params[:completed_at_to].present?
+    @submissions =
-                     @submissions.order(Submitter.arel_table[:completed_at].maximum.desc)
+      if params[:status] == 'completed' || params[:completed_at_from].present? || params[:completed_at_to].present?
+        @submissions.order(completed_at: :desc)
       else
         @submissions.order(id: :desc)
       end

app/controllers/submissions_controller.rb

@@ -21,7 +21,7 @@ class SubmissionsController < ApplicationController
   def show
     @submission = Submissions.preload_with_pages(@submission)
 
-    unless @submission.submitters.all?(&:completed_at?)
+    unless @submission.completed_at?
       ActiveRecord::Associations::Preloader.new(
         records: [@submission],
         associations: [{ submitters: :start_form_submission_events }]

app/controllers/submissions_dashboard_controller.rb

@@ -13,8 +13,9 @@ class SubmissionsDashboardController < ApplicationController
     @submissions = Submissions.search(current_user, @submissions, params[:q], search_template: true)
     @submissions = Submissions::Filter.call(@submissions, current_user, params)
 
-    @submissions = if params[:completed_at_from].present? || params[:completed_at_to].present?
+    @submissions =
-                     @submissions.order(Submitter.arel_table[:completed_at].maximum.desc)
+      if params[:status] == 'completed' || params[:completed_at_from].present? || params[:completed_at_to].present?
+        @submissions.order(completed_at: :desc)
       else
         @submissions.order(id: :desc)
       end

app/controllers/submissions_preview_controller.rb

@@ -25,7 +25,7 @@ class SubmissionsPreviewController < ApplicationController
 
     raise ActionController::RoutingError, I18n.t('not_found') if @submission.account.archived_at?
 
-    if !@submission.submitters.all?(&:completed_at?) && !signature_valid &&
+    if !@submission.completed_at? && !signature_valid &&
        (!current_user || !current_ability.can?(:read, @submission))
       raise ActionController::RoutingError, I18n.t('not_found')
     end

app/controllers/templates_archived_submissions_controller.rb

@@ -10,7 +10,7 @@ class TemplatesArchivedSubmissionsController < ApplicationController
     @submissions = Submissions::Filter.call(@submissions, current_user, params)
 
     @submissions = if params[:completed_at_from].present? || params[:completed_at_to].present?
-                     @submissions.order(Submitter.arel_table[:completed_at].maximum.desc)
+                     @submissions.order(completed_at: :desc)
                    else
                      @submissions.order(id: :desc)
                    end

app/controllers/templates_controller.rb

@@ -14,7 +14,7 @@ class TemplatesController < ApplicationController
     submissions = Submissions::Filter.filter_by_status(submissions, params)
 
     submissions = if params[:completed_at_from].present? || params[:completed_at_to].present?
-                    submissions.order(Submitter.arel_table[:completed_at].maximum.desc)
+                    submissions.order(completed_at: :desc)
                   else
                     submissions.order(id: :desc)
                   end

app/jobs/process_submission_expired_job.rb

@@ -11,7 +11,7 @@ class ProcessSubmissionExpiredJob
     return if submission.archived_at?
     return if submission.template&.archived_at?
     return if submission.submitters.where.not(declined_at: nil).exists?
-    return unless submission.submitters.exists?(completed_at: nil)
+    return if submission.completed_at?
 
     WebhookUrls.enqueue_events(submission, 'submission.expired')
   end

app/models/submission.rb

@@ -26,7 +26,9 @@
 #
 # Indexes
 #
+#  index_submissions_on_account_id_and_completed_at                 (account_id,completed_at) WHERE ((completed_at IS NOT NULL) AND (archived_at IS NULL))
 #  index_submissions_on_account_id_and_id                           (account_id,id)
+#  index_submissions_on_account_id_and_id_pending                   (account_id,id) WHERE ((completed_at IS NULL) AND (archived_at IS NULL))
 #  index_submissions_on_account_id_and_template_id_and_id           (account_id,template_id,id) WHERE (archived_at IS NULL)
 #  index_submissions_on_account_id_and_template_id_and_id_archived  (account_id,template_id,id) WHERE (archived_at IS NOT NULL)
 #  index_submissions_on_created_by_user_id                          (created_by_user_id)
@@ -90,26 +92,17 @@ class Submission < ApplicationRecord
   scope :active, -> { where(archived_at: nil) }
   scope :archived, -> { where.not(archived_at: nil) }
   scope :pending, lambda {
-    where(expire_at: nil).or(where(expire_at: Time.current..))
+    where(expire_at: nil).or(where(expire_at: Time.current..)).where(completed_at: nil)
-                         .where(Submitter.where(Submitter.arel_table[:submission_id].eq(Submission.arel_table[:id])
-                                         .and(Submitter.arel_table[:completed_at].eq(nil))).select(1).arel.exists)
-  }
-  scope :completed, lambda {
-    where.not(Submitter.where(Submitter.arel_table[:submission_id].eq(Submission.arel_table[:id])
-     .and(Submitter.arel_table[:completed_at].eq(nil))).select(1).arel.exists)
   }
+  scope :completed, -> { where.not(completed_at: nil) }
   scope :declined, lambda {
     where(Submitter.where(Submitter.arel_table[:submission_id].eq(Submission.arel_table[:id])
      .and(Submitter.arel_table[:declined_at].not_eq(nil))).select(1).arel.exists)
   }
-  scope :expired, lambda {
+  scope :expired, -> { where(expire_at: ..Time.current).where(completed_at: nil) }
-    where(expire_at: ..Time.current)
-      .where(Submitter.where(Submitter.arel_table[:submission_id].eq(Submission.arel_table[:id])
-                      .and(Submitter.arel_table[:completed_at].eq(nil))).select(1).arel.exists)
-  }
 
   scope :select_for_list, lambda {
-    select(:id, :name, :created_by_user_id, :account_id,
+    select(:id, :name, :created_by_user_id, :account_id, :completed_at,
            :created_at, :archived_at, :expire_at, :template_id, :template_submitters)
   }
 

app/views/submissions/show.html.erb

@@ -15,7 +15,7 @@
       <h1 class="text-xl md:text-3xl font-semibold focus:text-clip" style="overflow: hidden; display: -webkit-box; -webkit-box-orient: vertical; -webkit-line-clamp: 2;"><% (@submission.name || @submission.template&.name).to_s.split(/(_)/).each do |item| %><%= item %><wbr><% end %></h1>
     </a>
     <div class="space-x-3 flex items-center">
-      <% is_all_completed = @submission.submitters.to_a.all?(&:completed_at?) %>
+      <% is_all_completed = @submission.completed_at? %>
       <% if signed_in? && can?(:destroy, @submission) && @submission.archived_at? && !is_all_completed %>
         <%= button_to button_title(title: t('unarchive'), disabled_with: t('unarchive')[0..-2], icon: svg_icon('rotate', class: 'w-6 h-6')), submission_unarchive_index_path(@submission), class: 'btn btn-primary btn-ghost text-base hidden md:flex' %>
       <% end %>

app/views/templates/_submission.html.erb

@@ -28,7 +28,7 @@
   <% end %>
   <div class="w-full flex flex-col md:flex-row space-y-4 md:space-y-0 md:justify-between px-5 md:px-6 pb-5 md:items-center pt-5 relative cursor-pointer">
     <% submitters = (submission.template_submitters || submission.template.submitters).filter_map { |item| submission.submitters.find { |e| e.uuid == item['uuid'] } } %>
-    <% is_submission_completed = submitters.all?(&:completed_at?) && submitters.size.positive? %>
+    <% is_submission_completed = submission.completed_at? %>
     <% if submitters.size == 1 %>
       <div>
         <% submitter = submitters.first %>

app/views/templates/show.html.erb

@@ -37,7 +37,7 @@
         </div>
         <% unless can?(:manage, :countless) %>
           <div class="badge badge-neutral badge-outline font-medium">
-            <%= params[:status].blank? && filter_params.blank? ? @pagy.count : @base_submissions.unscope(:group, :order).select(:id).distinct.count %>
+            <%= params[:status].blank? && filter_params.blank? ? @pagy.count : @base_submissions.count %>
           </div>
         <% end %>
       </a>
@@ -48,7 +48,7 @@
         </div>
         <% unless can?(:manage, :countless) %>
           <div class="badge badge-neutral badge-outline font-medium">
-            <%= params[:status] == 'pending' && filter_params.blank? ? @pagy.count : @base_submissions.pending.unscope(:group, :order).select(:id).distinct.count %>
+            <%= params[:status] == 'pending' && filter_params.blank? ? @pagy.count : @base_submissions.pending.count %>
           </div>
         <% end %>
       </a>
@@ -59,7 +59,7 @@
         </div>
         <% unless can?(:manage, :countless) %>
           <div class="badge badge-neutral badge-outline font-medium">
-            <%= params[:status] == 'completed' && filter_params.blank? ? @pagy.count : @base_submissions.completed.unscope(:group, :order).select(:id).distinct.count %>
+            <%= params[:status] == 'completed' && filter_params.blank? ? @pagy.count : @base_submissions.completed.count %>
           </div>
         <% end %>
       </a>

lib/submissions.rb

@@ -37,19 +37,22 @@ module Submissions
 
     arel_table = Submitter.arel_table
 
-    arel = arel_table[:email].lower.matches(term)
+    submitter_arel = arel_table[:email].lower.matches(term)
                                        .or(arel_table[:phone].matches(term))
                                        .or(arel_table[:name].lower.matches(term))
 
-    arel = arel.or(Arel::Table.new(:submitters)[:values].matches(term)) if search_values
+    submitter_arel = submitter_arel.or(arel_table[:values].matches(term)) if search_values
+
+    arel = Submitter.where(arel_table[:submission_id].eq(Submission.arel_table[:id]))
+                    .where(submitter_arel).select(1).arel.exists
 
     if search_template
       submissions = submissions.left_joins(:template)
 
-      arel = arel.or(Template.arel_table[:name].lower.matches("%#{sanitized}%"))
+      arel = arel.or(Template.arel_table[:name].lower.matches(term))
     end
 
-    submissions.joins(:submitters).where(arel).group(:id)
+    submissions.where(arel)
   end
 
   def fulltext_search(current_user, submissions, keyword, search_template: false)

lib/submissions/ensure_audit_generated.rb

@@ -15,7 +15,7 @@ module Submissions
     def call(submission)
       return nil unless submission
 
-      raise NotCompletedYet unless submission.submitters.all?(&:completed_at?)
+      raise NotCompletedYet unless submission.completed_at?
 
       total_wait_time ||= 0
       key = [KEY_PREFIX, submission.id].join(':')

lib/submissions/filter.rb

@@ -104,16 +104,15 @@ module Submissions
     def filter_by_completed_at(submissions, filters)
       return submissions unless filters[:completed_at_from].present? || filters[:completed_at_to].present?
 
-      completed_arel = Submitter.arel_table[:completed_at].maximum
+      submissions = submissions.completed
-      submissions = submissions.completed.joins(:submitters).group(:id)
 
       if filters[:completed_at_from].present?
-        submissions = submissions.having(completed_arel.gteq(filters[:completed_at_from]))
+        submissions = submissions.where(completed_at: filters[:completed_at_from]..)
       end
 
       return submissions if filters[:completed_at_to].blank?
 
-      submissions.having(completed_arel.lteq(filters[:completed_at_to].end_of_day))
+      submissions.where(completed_at: ..filters[:completed_at_to].end_of_day)
     end
 
     def normalize_filter_params(params, current_user)

lib/submissions/serialize_for_api.rb

@@ -37,8 +37,8 @@ module Submissions
         json['fields'] = submission.template_fields || submission.template&.fields
       end
 
-      if submitters.all?(&:completed_at?)
+      if submission.completed_at?
-        last_submitter = submitters.max_by(&:completed_at)
+        last_submitter = submitters.select(&:completed_at?).max_by(&:completed_at)
 
         if with_documents
           json['documents'] = serialized_submitters.find { |e| e['id'] == last_submitter.id }['documents']
@@ -49,7 +49,7 @@ module Submissions
         json['combined_document_url'] ||= maybe_build_combined_url(submitters, submission, params, expires_at:)
 
         json['status'] = 'completed'
-        json['completed_at'] = last_submitter.completed_at.as_json
+        json['completed_at'] = submission.completed_at.as_json
       else
         json['documents'] = [] if with_documents
         json['audit_log_url'] = nil
@@ -73,12 +73,12 @@ module Submissions
     end
 
     def maybe_build_combined_url(submitters, submission, params, expires_at: nil)
-      return unless submitters.all?(&:completed_at?)
+      return unless submission.completed_at?
 
       attachment = submission.combined_document_attachment
 
       if !attachment && params[:include].to_s.include?('combined_document_url')
-        submitter = submitters.max_by(&:completed_at)
+        submitter = submitters.select(&:completed_at?).max_by(&:completed_at)
 
         attachment = Submissions::EnsureCombinedGenerated.call(submitter)
       end

lib/submitters.rb

@@ -108,7 +108,7 @@ module Submitters
     if AccountConfig.exists?(account_id: submitter.submission.account_id,
                              key: AccountConfig::COMBINE_PDF_RESULT_KEY,
                              value: true) &&
-       submitter.submission.submitters.all?(&:completed_at?) &&
+       submitter.submission.completed_at? &&
        submitter.submission.template_fields.none? { |f| f['type'] == 'verification' }
       return [submitter.submission.combined_document_attachment || Submissions::EnsureCombinedGenerated.call(submitter)]
     end
@@ -207,7 +207,7 @@ module Submitters
 
     filename = filename.gsub('{document.name}', blob.filename.base)
     filename = filename.gsub(' - {submission.status}') do
-      if submitter.submission.submitters.all?(&:completed_at?)
+      if submitter.submission.completed_at?
         status =
           if submitter.submission.template_fields.any? { |f| f['type'] == 'signature' }
             I18n.t(:signed)
@@ -264,7 +264,7 @@ module Submitters
   end
 
   def build_combined_url(submitter, ttl: FILES_TTL)
-    return if submitter.submission.submitters.exists?(completed_at: nil)
+    return unless submitter.submission.completed_at?
     return if submitter.submission.submitters.order(:completed_at).last != submitter
 
     attachment = submitter.submission.combined_document_attachment

lib/submitters/serialize_for_webhook.rb

@@ -93,11 +93,9 @@ module Submitters
     end
 
     def build_submission_status(submission)
-      submitters = submission.submitters
+      if submission.completed_at?
-
-      if submitters.all?(&:completed_at?)
         'completed'
-      elsif submitters.any?(&:declined_at?)
+      elsif submission.submitters.any?(&:declined_at?)
         'declined'
       else
         submission.expired? ? 'expired' : 'pending'

spec/jobs/process_submitter_completion_job_spec.rb

@@ -10,6 +10,8 @@ RSpec.describe ProcessSubmitterCompletionJob do
   before do
     create(:encrypted_config, key: EncryptedConfig::ESIGN_CERTS_KEY,
                               value: GenerateCertificate.call.transform_values(&:to_pem))
+
+    Submissions.maybe_update_completed_at(submitter.submission)
   end
 
   describe '#perform' do

spec/system/submission_preview_spec.rb

@@ -26,6 +26,7 @@ RSpec.describe 'Submission Preview' do
           create(:encrypted_config, account:, key: EncryptedConfig::EMAIL_SMTP_KEY, value: '{}')
 
           submission.submitters.each { |s| s.update(completed_at: 1.day.ago) }
+          Submissions.maybe_update_completed_at(submission)
 
           visit submissions_preview_path(slug: submission.slug)
         end
@@ -47,6 +48,7 @@ RSpec.describe 'Submission Preview' do
 
       it "doesn't display the email form if SMTP is not configured" do
         submission.submitters.each { |s| s.update(completed_at: 1.day.ago) }
+        Submissions.maybe_update_completed_at(submission)
 
         visit submissions_preview_path(slug: submission.slug)
 

spec/system/template_spec.rb

@@ -207,6 +207,8 @@ RSpec.describe 'Template' do
         submitter.update!(completed_at: rand(2..5).days.ago)
       end
 
+      (last_week_submissions + this_week_submissions).each { |s| Submissions.maybe_update_completed_at(s) }
+
       visit template_path(template)
 
       (last_week_submissions + this_week_submissions).map(&:submitters).flatten.last(10).uniq.each do |submitter|