Merge pull request #46 from CareerPlug/CP-11565

CP-11565 - DocuSeal Audit Logging Improvements

app/controllers/api/submissions_controller.rb

@@ -177,7 +177,10 @@ module Api
         Submissions::NormalizeParamUtils.save_default_value_attachments!(attachments, submitters)
 
         submitters.each do |submitter|
-          SubmissionEvents.create_with_tracking_data(submitter, 'api_complete_form', request) if submitter.completed_at?
+          if submitter.completed_at?
+            SubmissionEvents.create_with_tracking_data(submitter, 'api_complete_form', request, {},
+                                                       current_user)
+          end
         end
 
         submissions

app/controllers/api/submitters_controller.rb

@@ -37,37 +37,12 @@ module Api
         return render json: { error: 'Submitter has already completed the submission.' }, status: :unprocessable_entity
       end
 
-      submission = @submitter.submission
-      role = submission.template_submitters.find { |e| e['uuid'] == @submitter.uuid }['name']
-
-      normalized_params, new_attachments = Submissions::NormalizeParamUtils.normalize_submitter_params!(
-        submitter_params.merge(role:),
-        @submitter.template || Template.new(submitters: submission.template_submitters, account: @submitter.account),
-        for_submitter: @submitter
-      )
-
-      Submissions::CreateFromSubmitters.maybe_set_template_fields(submission, [normalized_params],
-                                                                  default_submitter_uuid: @submitter.uuid)
+      normalized_params, new_attachments = normalize_and_prepare_params
+      old_values = @submitter.values.dup
 
       assign_submitter_attrs(@submitter, normalized_params)
-
-      ApplicationRecord.transaction do
-        Submissions::NormalizeParamUtils.save_default_value_attachments!(new_attachments, [@submitter])
-
-        @submitter.save!
-
-        @submitter.submission.save!
-
-        SubmissionEvents.create_with_tracking_data(@submitter, 'api_complete_form', request) if @submitter.completed_at?
-      end
-
-      if @submitter.completed_at?
-        ProcessSubmitterCompletionJob.perform_async('submitter_id' => @submitter.id)
-      elsif normalized_params[:send_email] || normalized_params[:send_sms]
-        Submitters.send_signature_requests([@submitter])
-      end
-
-      SearchEntries.enqueue_reindex(@submitter)
+      save_submitter_and_track_changes(new_attachments, old_values)
+      handle_post_save_actions(normalized_params)
 
       render json: Submitters::SerializeForApi.call(@submitter, with_template: false, with_urls: true,
                                                                 with_events: false, params:)
@@ -170,6 +145,48 @@ module Api
       maybe_filder_by_completed_at(submitters, params)
     end
 
+    def normalize_and_prepare_params
+      submission = @submitter.submission
+      role = submission.template_submitters.find { |e| e['uuid'] == @submitter.uuid }['name']
+
+      normalized_params, new_attachments = Submissions::NormalizeParamUtils.normalize_submitter_params!(
+        submitter_params.merge(role:),
+        @submitter.template || Template.new(submitters: submission.template_submitters, account: @submitter.account),
+        for_submitter: @submitter
+      )
+
+      Submissions::CreateFromSubmitters.maybe_set_template_fields(submission, [normalized_params],
+                                                                  default_submitter_uuid: @submitter.uuid)
+
+      [normalized_params, new_attachments]
+    end
+
+    def save_submitter_and_track_changes(new_attachments, old_values)
+      ApplicationRecord.transaction do
+        Submissions::NormalizeParamUtils.save_default_value_attachments!(new_attachments, [@submitter])
+
+        @submitter.save!
+
+        @submitter.submission.save!
+
+        Submitters::SubmitValues.track_form_update(@submitter, old_values, request, current_user)
+
+        return unless @submitter.completed_at?
+
+        SubmissionEvents.create_with_tracking_data(@submitter, 'api_complete_form', request, {}, current_user)
+      end
+    end
+
+    def handle_post_save_actions(normalized_params)
+      if @submitter.completed_at?
+        ProcessSubmitterCompletionJob.perform_async('submitter_id' => @submitter.id)
+      elsif normalized_params[:send_email] || normalized_params[:send_sms]
+        Submitters.send_signature_requests([@submitter])
+      end
+
+      SearchEntries.enqueue_reindex(@submitter)
+    end
+
     def assign_external_id(submitter, attrs)
       submitter.external_id = attrs[:application_key] if attrs.key?(:application_key)
       submitter.external_id = attrs[:external_id] if attrs.key?(:external_id)

app/controllers/submit_form_controller.rb

@@ -67,7 +67,7 @@ class SubmitFormController < ApplicationController
                     status: :unprocessable_entity
     end
 
-    Submitters::SubmitValues.call(@submitter, params, request)
+    Submitters::SubmitValues.call(@submitter, params, request, current_user, validate_required: true)
 
     head :ok
   rescue Submitters::SubmitValues::RequiredFieldError => e

app/controllers/submitters_request_changes_controller.rb

@@ -24,7 +24,8 @@ class SubmittersRequestChangesController < ApplicationController
           @submitter,
           'request_changes',
           request,
-          { reason: params[:reason], requested_by: current_user.id }
+          { reason: params[:reason], requested_by: current_user.id },
+          current_user
         )
       end
 

app/jobs/send_form_completed_webhook_request_job.rb

@@ -19,8 +19,14 @@ class SendFormCompletedWebhookRequestJob
 
     ActiveStorage::Current.url_options = Docuseal.default_url_options
 
+    # Build the payload with submission events for granular audit tracking
+    webhook_data = Submitters::SerializeForWebhook.call(submitter)
+
+    # Add submission events for CareerPlug ATS integration
+    webhook_data['submission_events'] = serialize_submission_events(submitter.submission)
+
     resp = SendWebhookRequest.call(webhook_url, event_type: 'form.completed',
-                                                data: Submitters::SerializeForWebhook.call(submitter))
+                                                data: webhook_data)
 
     if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS &&
        (!Docuseal.multitenant? || submitter.account.account_configs.exists?(key: :plan))
@@ -31,4 +37,20 @@ class SendFormCompletedWebhookRequestJob
                                                     })
     end
   end
+
+  private
+
+  # Serialize submission events for webhook payload
+  # Returns array of event hashes with field-level change tracking
+  def serialize_submission_events(submission)
+    submission.submission_events.order(:event_timestamp).map do |event|
+      {
+        id: event.id,
+        event_type: event.event_type,
+        event_timestamp: event.event_timestamp.iso8601,
+        user_id: event.user_id,
+        data: event.data
+      }
+    end
+  end
 end

app/models/submission_event.rb

@@ -12,20 +12,24 @@
 #  updated_at      :datetime         not null
 #  submission_id   :integer          not null
 #  submitter_id    :integer
+#  user_id         :bigint
 #
 # Indexes
 #
 #  index_submission_events_on_created_at     (created_at)
 #  index_submission_events_on_submission_id  (submission_id)
 #  index_submission_events_on_submitter_id   (submitter_id)
+#  index_submission_events_on_user_id        (user_id)
 #
 # Foreign Keys
 #
 #  fk_rails_...  (submission_id => submissions.id)
 #  fk_rails_...  (submitter_id => submitters.id)
+#  fk_rails_...  (user_id => users.id)
 #
 class SubmissionEvent < ApplicationRecord
   belongs_to :submission
+  belongs_to :user, optional: true
   has_one :account, through: :submission
   belongs_to :submitter, optional: true
 
@@ -55,7 +59,8 @@ class SubmissionEvent < ApplicationRecord
     complete_form: 'complete_form',
     decline_form: 'decline_form',
     request_changes: 'request_changes',
-    api_complete_form: 'api_complete_form'
+    api_complete_form: 'api_complete_form',
+    form_update: 'form_update'
   }, scope: false
 
   private

app/services/export_submission_service.rb

@@ -55,7 +55,11 @@ class ExportSubmissionService < ExportService
         }
       end,
       created_at: submission.created_at,
-      updated_at: submission.updated_at
+      updated_at: submission.updated_at,
+      # Include form field values for each submitter
+      values: build_values_array,
+      # Include granular submission events for audit trail
+      submission_events: build_submission_events_array
     }
   end
 
@@ -77,4 +81,59 @@ class ExportSubmissionService < ExportService
       'pending'
     end
   end
+
+  # Build array of form field values from all submitters
+  # Returns array of {field: name, value: value} hashes
+  def build_values_array
+    submission.submitters.flat_map do |submitter|
+      build_submitter_values(submitter)
+    end
+  end
+
+  # Build values for a single submitter
+  def build_submitter_values(submitter)
+    fields = submission.template_fields.presence || submission.template&.fields || []
+    attachments_index = submitter.attachments.index_by(&:uuid)
+
+    fields.filter_map do |field|
+      next if field['submitter_uuid'] != submitter.uuid
+      next if field['type'] == 'heading'
+
+      field_name = field['name'].presence || "#{field['type'].titleize} Field"
+      next unless submitter.values.key?(field['uuid']) || submitter.completed_at?
+
+      value = fetch_field_value(field, submitter.values[field['uuid']], attachments_index)
+
+      { field: field_name, value: }
+    end
+  end
+
+  # Build array of submission events for audit trail
+  def build_submission_events_array
+    submission.submission_events.order(:event_timestamp).map do |event|
+      {
+        id: event.id,
+        event_type: event.event_type,
+        event_timestamp: event.event_timestamp.iso8601,
+        data: event.data
+      }
+    end
+  end
+
+  # Fetch the value for a field, handling special types
+  def fetch_field_value(field, value, attachments_index)
+    if field['type'].in?(%w[image signature initials stamp payment])
+      rails_storage_proxy_url(attachments_index[value])
+    elsif field['type'] == 'file'
+      Array.wrap(value).compact_blank.filter_map { |e| rails_storage_proxy_url(attachments_index[e]) }
+    else
+      value
+    end
+  end
+
+  def rails_storage_proxy_url(attachment)
+    return if attachment.blank?
+
+    ActiveStorage::Blob.proxy_url(attachment.blob)
+  end
 end

db/migrate/20260121191632_add_user_id_to_submission_events.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddUserIdToSubmissionEvents < ActiveRecord::Migration[8.0]
+  def change
+    add_reference :submission_events, :user, null: true, foreign_key: true
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2025_11_07_175502) do
+ActiveRecord::Schema[8.0].define(version: 2026_01_21_191632) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "btree_gin"
   enable_extension "pg_catalog.plpgsql"
@@ -181,7 +181,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_11_07_175502) do
     t.datetime "created_at", null: false
     t.index ["account_id", "event_datetime"], name: "index_email_events_on_account_id_and_event_datetime"
     t.index ["email"], name: "index_email_events_on_email"
-    t.index ["email"], name: "index_email_events_on_email_event_types", where: "((event_type)::text = ANY ((ARRAY['bounce'::character varying, 'soft_bounce'::character varying, 'complaint'::character varying, 'soft_complaint'::character varying])::text[]))"
+    t.index ["email"], name: "index_email_events_on_email_event_types", where: "((event_type)::text = ANY (ARRAY[('bounce'::character varying)::text, ('soft_bounce'::character varying)::text, ('complaint'::character varying)::text, ('soft_complaint'::character varying)::text]))"
     t.index ["emailable_type", "emailable_id"], name: "index_email_events_on_emailable"
     t.index ["message_id"], name: "index_email_events_on_message_id"
   end
@@ -290,10 +290,11 @@ ActiveRecord::Schema[8.0].define(version: 2025_11_07_175502) do
     t.tsvector "tsvector", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index ["account_id", "tsvector"], name: "index_search_entries_on_account_id_tsvector_submission", where: "((record_type)::text = 'Submission'::text)", using: :gin
-    t.index ["account_id", "tsvector"], name: "index_search_entries_on_account_id_tsvector_submitter", where: "((record_type)::text = 'Submitter'::text)", using: :gin
-    t.index ["account_id", "tsvector"], name: "index_search_entries_on_account_id_tsvector_template", where: "((record_type)::text = 'Template'::text)", using: :gin
+    t.index ["account_id"], name: "index_search_entries_on_account_id"
     t.index ["record_id", "record_type"], name: "index_search_entries_on_record_id_and_record_type", unique: true
+    t.index ["tsvector"], name: "index_search_entries_on_account_id_tsvector_submission", where: "((record_type)::text = 'Submission'::text)", using: :gin
+    t.index ["tsvector"], name: "index_search_entries_on_account_id_tsvector_submitter", where: "((record_type)::text = 'Submitter'::text)", using: :gin
+    t.index ["tsvector"], name: "index_search_entries_on_account_id_tsvector_template", where: "((record_type)::text = 'Template'::text)", using: :gin
   end
 
   create_table "submission_events", force: :cascade do |t|
@@ -304,9 +305,11 @@ ActiveRecord::Schema[8.0].define(version: 2025_11_07_175502) do
     t.datetime "event_timestamp", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.bigint "user_id"
     t.index ["created_at"], name: "index_submission_events_on_created_at"
     t.index ["submission_id"], name: "index_submission_events_on_submission_id"
     t.index ["submitter_id"], name: "index_submission_events_on_submitter_id"
+    t.index ["user_id"], name: "index_submission_events_on_user_id"
   end
 
   create_table "submissions", force: :cascade do |t|
@@ -497,6 +500,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_11_07_175502) do
   add_foreign_key "oauth_access_tokens", "users", column: "resource_owner_id"
   add_foreign_key "submission_events", "submissions"
   add_foreign_key "submission_events", "submitters"
+  add_foreign_key "submission_events", "users"
   add_foreign_key "submissions", "templates"
   add_foreign_key "submissions", "users", column: "created_by_user_id"
   add_foreign_key "submitters", "submissions"

lib/send_webhook_request.rb

@@ -27,7 +27,11 @@ module SendWebhookRequest
     Faraday.post(uri) do |req|
       req.headers['Content-Type'] = 'application/json'
       req.headers['User-Agent'] = USER_AGENT
-      req.headers.merge!(webhook_url.secret.to_h) if webhook_url.secret.present?
+
+      # Send webhook secret headers from the configured secret hash
+      webhook_url.secret&.each do |header_name, header_value|
+        req.headers[header_name] = header_value
+      end
 
       req.body = {
         event_type: event_type,

lib/submission_events.rb

@@ -11,12 +11,14 @@ module SubmissionEvents
     ).first(TRACKING_PARAM_LENGTH)
   end
 
-  def create_with_tracking_data(submitter, event_type, request, data = {})
-    SubmissionEvent.create!(submitter:, event_type:, data: {
+  def create_with_tracking_data(submitter, event_type, request, data = {}, user = nil)
+    user ||= request.env['warden']&.user(:user)
+
+    SubmissionEvent.create!(submitter:, event_type:, user:, data: {
       ip: request.remote_ip,
       ua: request.user_agent,
       sid: request.session.id.to_s,
-      uid: request.env['warden'].user(:user)&.id,
+      uid: user&.id,
       **data
     }.compact_blank)
   end

lib/submitters/submit_values.rb

@@ -10,11 +10,11 @@ module Submitters
 
     module_function
 
-    def call(submitter, params, request, validate_required: true)
+    def call(submitter, params, request, current_user = nil, validate_required: true)
       Submissions.update_template_fields!(submitter.submission) if submitter.submission.template_fields.blank?
 
       unless submitter.submission_events.exists?(event_type: 'start_form')
-        SubmissionEvents.create_with_tracking_data(submitter, 'start_form', request)
+        SubmissionEvents.create_with_tracking_data(submitter, 'start_form', request, {}, current_user)
 
         WebhookUrls.for_account_id(submitter.account_id, 'form.started').each do |webhook_url|
           SendFormStartedWebhookRequestJob.perform_async('submitter_id' => submitter.id,
@@ -22,10 +22,15 @@ module Submitters
         end
       end
 
+      old_values = submitter.values.dup
+
       update_submitter!(submitter, params, request, validate_required:)
 
       submitter.submission.save!
 
+      # Track form updates when values change (but not on completion, as that creates complete_form event)
+      track_form_update(submitter, old_values, request, current_user) if params[:completed] != 'true'
+
       ProcessSubmitterCompletionJob.perform_async('submitter_id' => submitter.id) if submitter.completed_at?
 
       submitter
@@ -345,5 +350,31 @@ module Submitters
     def validate_value!(_value, _field, _params, _submitter, _request)
       true
     end
+
+    def track_form_update(submitter, old_values, request, current_user = nil)
+      # Use existing O(1) lookup index from submission model
+      fields_by_uuid = submitter.submission.fields_uuid_index
+
+      changes = submitter.values.filter_map do |field_uuid, new_value|
+        old_value = old_values[field_uuid]
+        next if old_value == new_value
+
+        field = fields_by_uuid[field_uuid]
+        next unless field
+
+        { 'field' => field['name'], 'from' => old_value, 'to' => new_value }
+      end
+
+      # Only create event if there are actual changes
+      return if changes.empty?
+
+      SubmissionEvents.create_with_tracking_data(
+        submitter,
+        'form_update',
+        request,
+        { 'changes' => changes },
+        current_user
+      )
+    end
   end
 end

spec/jobs/send_form_completed_webhook_request_job_spec.rb

@@ -20,40 +20,6 @@ RSpec.describe SendFormCompletedWebhookRequestJob do
       stub_request(:post, webhook_url.url).to_return(status: 200)
     end
 
-    it 'sends a webhook request' do
-      described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id)
-
-      expect(WebMock).to have_requested(:post, webhook_url.url).with(
-        body: {
-          'event_type' => 'form.completed',
-          'timestamp' => /.*/,
-          'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json)
-        },
-        headers: {
-          'Content-Type' => 'application/json',
-          'User-Agent' => 'DocuSeal.com Webhook'
-        }
-      ).once
-    end
-
-    it 'sends a webhook request with the secret' do
-      webhook_url.update(secret: { 'X-Secret-Header' => 'secret_value' })
-      described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id)
-
-      expect(WebMock).to have_requested(:post, webhook_url.url).with(
-        body: {
-          'event_type' => 'form.completed',
-          'timestamp' => /.*/,
-          'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json)
-        },
-        headers: {
-          'Content-Type' => 'application/json',
-          'User-Agent' => 'DocuSeal.com Webhook',
-          'X-Secret-Header' => 'secret_value'
-        }
-      ).once
-    end
-
     it "doesn't send a webhook request if the event is not in the webhook's events" do
       webhook_url.update!(events: ['form.declined'])
 

spec/lib/submitters/submit_values_spec.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Submitters::SubmitValues do
+  let(:account) { create(:account) }
+  let(:user) { create(:user, account: account) }
+  let(:template) { create(:template, account: account) }
+  let(:submission) { create(:submission, template: template, account: account) }
+  let(:submitter) { create(:submitter, submission: submission, account: account, uuid: SecureRandom.uuid) }
+  let(:request) { instance_double(ActionDispatch::Request, remote_ip: '127.0.0.1', user_agent: 'TestAgent') }
+
+  before do
+    allow(request).to receive_messages(
+      session: instance_double(ActionDispatch::Request::Session, id: 'session_id'),
+      env: { 'warden' => instance_double(Warden::Proxy, user: user) }
+    )
+
+    # Setup template fields
+    fields = [
+      { 'uuid' => 'field_1', 'name' => 'First Name', 'type' => 'text', 'submitter_uuid' => submitter.uuid },
+      { 'uuid' => 'field_2', 'name' => 'Last Name', 'type' => 'text', 'submitter_uuid' => submitter.uuid }
+    ]
+    template.update!(fields: fields)
+    submission.update!(template_fields: fields)
+
+    # Initialize submitter values
+    submitter.update!(values: { 'field_1' => 'John', 'field_2' => 'Doe' })
+    create(:submission_event, submission: submission, submitter: submitter, event_type: 'start_form')
+  end
+
+  describe '.call' do
+    context 'when values change' do
+      let(:params) do
+        {
+          values: { 'field_1' => 'Jane' }
+        }
+      end
+
+      it 'creates a form_update event with changes' do
+        expect do
+          described_class.call(submitter, ActionController::Parameters.new(params), request, user)
+        end.to change(SubmissionEvent, :count).by(1)
+
+        event = SubmissionEvent.last
+        expect(event.event_type).to eq('form_update')
+        expect(event.user).to eq(user)
+        expect(event.data['changes']).to include(
+          hash_including('field' => 'First Name', 'from' => 'John', 'to' => 'Jane')
+        )
+      end
+    end
+
+    context 'when values do not change' do
+      let(:params) do
+        {
+          values: { 'field_1' => 'John' }
+        }
+      end
+
+      it 'does not create a form_update event' do
+        expect do
+          described_class.call(submitter, ActionController::Parameters.new(params), request, user)
+        end.not_to change(SubmissionEvent, :count)
+      end
+    end
+
+    context 'when multiple fields change' do
+      let(:params) do
+        {
+          values: { 'field_1' => 'Jane', 'field_2' => 'Smith' }
+        }
+      end
+
+      it 'records all changes' do
+        described_class.call(submitter, ActionController::Parameters.new(params), request, user)
+
+        event = SubmissionEvent.last
+        changes = event.data['changes']
+
+        expect(changes.size).to eq(2)
+        expect(changes).to include(hash_including('field' => 'First Name', 'to' => 'Jane'))
+        expect(changes).to include(hash_including('field' => 'Last Name', 'to' => 'Smith'))
+      end
+    end
+  end
+end

spec/services/export_submission_service_spec.rb

@@ -187,6 +187,13 @@ RSpec.describe ExportSubmissionService do
           'status' => 'completed'
         )
         expect(completed_submitter).to have_key('external_submitter_id')
+
+        # New fields for granular audit tracking
+        expect(parsed_body).to have_key('values')
+        expect(parsed_body['values']).to be_an(Array)
+
+        expect(parsed_body).to have_key('submission_events')
+        expect(parsed_body['submission_events']).to be_an(Array)
       end
       service.call
     end