diff --git a/README.md b/README.md index 4b952790..8334aaab 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,25 @@ Reminder emails are sent to pending signers on a configurable schedule. - Deduplication guard prevents the same reminder from being sent twice within 1 minute - Job scheduling handles container restarts gracefully (clears stale scheduled jobs before re-registering) +## Paperless-ngx Integration + +Automatically upload completed, fully-signed documents to a [paperless-ngx](https://docs.paperless-ngx.com/) instance for archival and full-text search. + +**How it works:** +- When all parties have signed a submission, the combined result PDF and audit trail are uploaded to paperless-ngx via its REST API. +- If the combined PDF feature is not enabled, individual per-submitter result PDFs are uploaded instead. +- Documents are titled `"Template Name - Signer 1, Signer 2"` with the signing completion date. +- Uploads run as a background job with exponential retry (up to 10 attempts) — they never block the signing flow. + +**Configuration (env vars only — no GUI):** + +| Variable | Description | +|----------|-------------| +| `PAPERLESS_NGX_URL` | Base URL of your paperless-ngx instance (e.g., `http://paperless:8000`) | +| `PAPERLESS_NGX_TOKEN` | API token for authentication ([how to get one](https://docs.paperless-ngx.com/api/#authorization)) | + +The feature is inactive unless both variables are set. + ## What's NOT included These Pro features remain unavailable in this fork (they require significant UI/infrastructure work): diff --git a/app/jobs/process_submitter_completion_job.rb b/app/jobs/process_submitter_completion_job.rb index 135c0c38..27a65448 100644 --- a/app/jobs/process_submitter_completion_job.rb +++ b/app/jobs/process_submitter_completion_job.rb @@ -20,6 +20,10 @@ class ProcessSubmitterCompletionJob Submissions::EnsureAuditGenerated.call(submitter.submission) enqueue_completed_emails(submitter) + + if Submissions::UploadToPaperless.configured? + UploadToPaperlessJob.perform_async('submission_id' => submitter.submission_id) + end end create_completed_documents!(submitter) diff --git a/app/jobs/upload_to_paperless_job.rb b/app/jobs/upload_to_paperless_job.rb new file mode 100644 index 00000000..37847054 --- /dev/null +++ b/app/jobs/upload_to_paperless_job.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class UploadToPaperlessJob + include Sidekiq::Job + + sidekiq_options queue: :integrations + + MAX_ATTEMPTS = 10 + + def perform(params = {}) + return unless Submissions::UploadToPaperless.configured? + + submission = Submission.find_by(id: params['submission_id']) + + return unless submission + + attempt = params['attempt'].to_i + + Submissions::UploadToPaperless.call(submission) + rescue Submissions::UploadToPaperless::UploadError, Faraday::Error => e + return if attempt >= MAX_ATTEMPTS + + Rails.logger.warn("Paperless-ngx upload failed (attempt #{attempt}): #{e.message}") + + UploadToPaperlessJob.perform_in( + (2**attempt).minutes, + 'submission_id' => params['submission_id'], + 'attempt' => attempt + 1 + ) + end +end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 01ba85a5..64a57588 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -4,6 +4,7 @@ queues: - [sms, 2] - [images, 1] - [mailers, 1] + - [integrations, 1] - [recurrent, 1] - [rollbar, 1] diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 00000000..7f7b935a --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,59 @@ +services: + rspec: + build: + context: . + dockerfile: Dockerfile.ci + command: sh -c "bundle exec rake db:create db:migrate && bundle exec rake assets:precompile && bundle exec rspec spec/integration" + depends_on: + postgres: + condition: service_healthy + paperless-ngx: + condition: service_healthy + environment: + RAILS_ENV: test + NODE_ENV: test + DATABASE_URL: postgres://postgres:postgres@postgres:5432/docuseal_test + PAPERLESS_NGX_URL: http://paperless-ngx:8000 + PAPERLESS_NGX_TOKEN: "" + + postgres: + image: postgres:18 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: docuseal_test + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 5s + retries: 5 + tmpfs: + - /var/lib/postgresql + + paperless-redis: + image: docker.io/library/redis:8 + tmpfs: + - /data + + paperless-ngx: + image: ghcr.io/paperless-ngx/paperless-ngx:latest + depends_on: + - paperless-redis + ports: + - "8000:8000" + environment: + PAPERLESS_REDIS: redis://paperless-redis:6379 + PAPERLESS_SECRET_KEY: test-secret-for-e2e-testing + PAPERLESS_ADMIN_USER: admin + PAPERLESS_ADMIN_PASSWORD: admin + PAPERLESS_ADMIN_MAIL: admin@test.local + PAPERLESS_TIME_ZONE: UTC + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000"] + interval: 10s + timeout: 10s + retries: 10 + start_period: 30s + tmpfs: + - /usr/src/paperless/data + - /usr/src/paperless/media diff --git a/lib/submissions/upload_to_paperless.rb b/lib/submissions/upload_to_paperless.rb new file mode 100644 index 00000000..6cc0ed83 --- /dev/null +++ b/lib/submissions/upload_to_paperless.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +module Submissions + module UploadToPaperless + UploadError = Class.new(StandardError) + + BOUNDARY_PREFIX = '----DocuSealPaperlessUpload' + + module_function + + def call(submission) + return unless configured? + + submission.submitters.load unless submission.submitters.loaded? + + title = build_title(submission) + created = completed_date(submission) + results = documents_to_upload(submission, title).map do |attachment, doc_title| + upload_document(attachment, title: doc_title, created:) + end + + results.compact.presence + end + + def configured? + ENV['PAPERLESS_NGX_URL'].present? && ENV['PAPERLESS_NGX_TOKEN'].present? + end + + def documents_to_upload(submission, title) + documents = [] + + if submission.combined_document.attached? + documents << [submission.combined_document, title] + else + submission.submitters.select(&:completed_at?).each do |submitter| + submitter.documents.each do |doc| + documents << [doc, title] + end + end + end + + documents << [submission.audit_trail, "#{title} - Audit Trail"] if submission.audit_trail.attached? + + documents + end + + def build_title(submission) + submitter_names = submission.submitters + .select(&:completed_at?) + .sort_by(&:completed_at) + .filter_map(&:name) + .join(', ') + + template_name = submission.template&.name || 'Document' + + if submitter_names.present? + "#{template_name} - #{submitter_names}" + else + template_name + end + end + + def completed_date(submission) + last_completed = submission.submitters.filter_map(&:completed_at).max + last_completed&.strftime('%Y-%m-%d') + end + + def upload_document(attachment, title:, created:) + blob = attachment.blob + filename = sanitize_filename(blob.filename.to_s) + boundary = "#{BOUNDARY_PREFIX}#{SecureRandom.hex(16)}" + + blob.open do |tempfile| + body = build_multipart_body(tempfile, filename:, title:, created:, boundary:) + + response = connection.post('/api/documents/post_document/') do |req| + req.headers['Authorization'] = "Token #{ENV['PAPERLESS_NGX_TOKEN']}" # rubocop:disable Style/FetchEnvVar + req.headers['Content-Type'] = "multipart/form-data; boundary=#{boundary}" + req.body = body + req.options.read_timeout = 30 + req.options.open_timeout = 10 + end + + if response.status >= 400 + body_preview = response.body.to_s.truncate(200) + raise UploadError, "Paperless-ngx upload failed (HTTP #{response.status}): #{body_preview}" + end + + response.body.to_s.delete('"').strip + end + end + + def sanitize_filename(filename) + filename.gsub(/["\r\n\\]/, '_') + end + + def build_multipart_body(tempfile, filename:, title:, created:, boundary:) + parts = [] + + parts << "--#{boundary}\r\n" + parts << "Content-Disposition: form-data; name=\"document\"; filename=\"#{filename}\"\r\n" + parts << "Content-Type: application/pdf\r\n\r\n" + parts << tempfile.read + parts << "\r\n" + + parts << "--#{boundary}\r\n" + parts << "Content-Disposition: form-data; name=\"title\"\r\n\r\n" + parts << title + parts << "\r\n" + + if created + parts << "--#{boundary}\r\n" + parts << "Content-Disposition: form-data; name=\"created\"\r\n\r\n" + parts << created + parts << "\r\n" + end + + parts << "--#{boundary}--\r\n" + + parts.join + end + + def connection + Faraday.new(url: ENV['PAPERLESS_NGX_URL']) do |f| # rubocop:disable Style/FetchEnvVar + f.adapter Faraday.default_adapter + end + end + end +end diff --git a/spec/integration/paperless_ngx_upload_spec.rb b/spec/integration/paperless_ngx_upload_spec.rb new file mode 100644 index 00000000..390f4d34 --- /dev/null +++ b/spec/integration/paperless_ngx_upload_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'rails_helper' + +# rubocop:disable RSpec/DescribeClass, RSpec/InstanceVariable, RSpec/ExpectInHook + +# Run with: docker compose -f docker-compose.e2e.yml up +# This spec requires a running paperless-ngx instance. +# It acquires a token from paperless-ngx, uploads a document, and verifies it was received. +RSpec.describe 'Paperless-ngx document upload', type: :integration do + let(:paperless_url) { ENV.fetch('PAPERLESS_NGX_URL', 'http://paperless-ngx:8000') } + let(:account) { create(:account) } + let(:user) { create(:user, account:) } + let(:template) { create(:template, account:, author: user, name: 'Integration Test Contract') } + let(:submission) { create(:submission, :with_submitters, template:, created_by_user: user) } + + # Generate a unique PDF per test by appending a random trailer comment. + # This ensures paperless-ngx does not deduplicate uploads across tests. + def unique_pdf_io + base_pdf = Rails.root.join('spec/fixtures/sample-document.pdf').binread + unique_pdf = base_pdf + "\n% #{SecureRandom.uuid}\n" + StringIO.new(unique_pdf) + end + + before do + # Skip if paperless-ngx is not reachable + WebMock.allow_net_connect! + + begin + health = Faraday.get("#{paperless_url}/api/") { |req| req.options.open_timeout = 5 } + skip "Paperless-ngx not available (HTTP #{health.status})" unless health.status < 400 + rescue Faraday::Error + skip 'Paperless-ngx not available (connection failed)' + end + + # Acquire a token from paperless-ngx + token_response = Faraday.post("#{paperless_url}/api/token/") do |req| + req.headers['Content-Type'] = 'application/json' + req.body = { username: 'admin', password: 'admin' }.to_json + end + + expect(token_response.status).to eq(200), "Failed to get paperless-ngx token: #{token_response.body}" + @paperless_token = JSON.parse(token_response.body)['token'] + + # Configure ENV for the upload module + allow(ENV).to receive(:fetch).and_call_original + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(paperless_url) + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(@paperless_token) + + # Mark all submitters as completed + submission.submitters.each_with_index do |submitter, i| + submitter.update!(completed_at: i.hours.ago, name: "Test Signer #{i + 1}") + end + + # Attach a combined document with unique content per test + blob = ActiveStorage::Blob.create_and_upload!( + io: unique_pdf_io, + filename: "integration-test-#{SecureRandom.hex(4)}.pdf", + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission) + end + + after do + WebMock.disable_net_connect!(allow_localhost: true) + end + + it 'uploads a signed document to paperless-ngx and receives a task UUID' do + result = Submissions::UploadToPaperless.call(submission) + + expect(result).to be_present + expect(result.first).to match(/\A[0-9a-f-]+\z/) + end + + it 'uploads via the background job without error' do + expect do + UploadToPaperlessJob.new.perform('submission_id' => submission.id) + end.not_to raise_error + end + + it 'document is consumable by paperless-ngx' do + result = Submissions::UploadToPaperless.call(submission) + task_id = result.first + + # Poll the tasks API to verify consumption started + response = Faraday.get("#{paperless_url}/api/tasks/?task_id=#{task_id}") do |req| + req.headers['Authorization'] = "Token #{@paperless_token}" + end + + expect(response.status).to eq(200) + tasks = JSON.parse(response.body) + expect(tasks).to be_present + end +end + +# rubocop:enable RSpec/DescribeClass, RSpec/InstanceVariable, RSpec/ExpectInHook diff --git a/spec/jobs/upload_to_paperless_job_spec.rb b/spec/jobs/upload_to_paperless_job_spec.rb new file mode 100644 index 00000000..51ea789b --- /dev/null +++ b/spec/jobs/upload_to_paperless_job_spec.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe UploadToPaperlessJob do + let(:account) { create(:account) } + let(:user) { create(:user, account:) } + let(:template) { create(:template, account:, author: user) } + let(:submission) { create(:submission, :with_submitters, template:, created_by_user: user) } + + let(:paperless_url) { 'http://paperless:8000' } + let(:paperless_token) { 'test-token-abc123' } + + before do + submission.submitters.each_with_index do |submitter, i| + submitter.update!(completed_at: i.hours.ago, name: "Signer #{i + 1}") + end + + allow(ENV).to receive(:fetch).and_call_original + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(paperless_url) + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(paperless_token) + + stub_request(:post, "#{paperless_url}/api/documents/post_document/") + .to_return(status: 200, body: '"task-uuid-123"') + end + + describe '#perform' do + context 'when paperless-ngx is configured' do + before do + blob = ActiveStorage::Blob.create_and_upload!( + io: Rails.root.join('spec/fixtures/sample-document.pdf').open, + filename: 'combined-result.pdf', + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission) + end + + it 'uploads documents to paperless-ngx' do + described_class.new.perform('submission_id' => submission.id) + + expect(WebMock).to have_requested(:post, "#{paperless_url}/api/documents/post_document/") + .at_least_once + end + end + + context 'when paperless-ngx is not configured' do + before do + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(nil) + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(nil) + end + + it 'does nothing' do + described_class.new.perform('submission_id' => submission.id) + + expect(WebMock).not_to have_requested(:post, /paperless/) + end + end + + context 'when submission does not exist' do + it 'does nothing' do + described_class.new.perform('submission_id' => -1) + + expect(WebMock).not_to have_requested(:post, /paperless/) + end + end + + context 'when upload fails with a retryable error' do + before do + stub_request(:post, "#{paperless_url}/api/documents/post_document/") + .to_return(status: 500, body: 'Internal Server Error') + + blob = ActiveStorage::Blob.create_and_upload!( + io: Rails.root.join('spec/fixtures/sample-document.pdf').open, + filename: 'combined-result.pdf', + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission) + end + + it 'enqueues a retry with incremented attempt' do + expect do + described_class.new.perform('submission_id' => submission.id, 'attempt' => 0) + end.to change(described_class.jobs, :size).by(1) + + args = described_class.jobs.last['args'].first + expect(args['attempt']).to eq(1) + expect(args['submission_id']).to eq(submission.id) + end + end + + context 'when max attempts is reached' do + before do + stub_request(:post, "#{paperless_url}/api/documents/post_document/") + .to_return(status: 500, body: 'Internal Server Error') + + blob = ActiveStorage::Blob.create_and_upload!( + io: Rails.root.join('spec/fixtures/sample-document.pdf').open, + filename: 'combined-result.pdf', + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission) + end + + it 'does not enqueue another retry' do + expect do + described_class.new.perform('submission_id' => submission.id, 'attempt' => 11) + end.not_to change(described_class.jobs, :size) + end + end + end +end diff --git a/spec/lib/submissions/upload_to_paperless_spec.rb b/spec/lib/submissions/upload_to_paperless_spec.rb new file mode 100644 index 00000000..1dcd49b4 --- /dev/null +++ b/spec/lib/submissions/upload_to_paperless_spec.rb @@ -0,0 +1,208 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Submissions::UploadToPaperless do + let(:account) { create(:account) } + let(:user) { create(:user, account:) } + let(:template) { create(:template, account:, author: user, name: 'Employment Contract') } + let(:submission) { create(:submission, :with_submitters, template:, created_by_user: user) } + + let(:paperless_url) { 'http://paperless:8000' } + let(:paperless_token) { 'test-token-abc123' } + + before do + submission.submitters.each_with_index do |submitter, i| + submitter.update!(completed_at: i.hours.ago, name: "Signer #{i + 1}") + end + + allow(ENV).to receive(:fetch).and_call_original + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(paperless_url) + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(paperless_token) + + stub_request(:post, "#{paperless_url}/api/documents/post_document/") + .to_return(status: 200, body: '"550e8400-e29b-41d4-a716-446655440000"') + end + + describe '.call' do + context 'when submission has a combined document' do + before do + blob = ActiveStorage::Blob.create_and_upload!( + io: Rails.root.join('spec/fixtures/sample-document.pdf').open, + filename: 'combined-result.pdf', + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission) + end + + it 'uploads the combined document to paperless-ngx' do + described_class.call(submission) + + expect(WebMock).to have_requested(:post, "#{paperless_url}/api/documents/post_document/") + .with(headers: { 'Authorization' => "Token #{paperless_token}" }) + .once + end + + it 'sends the correct title with template name and submitter names' do + described_class.call(submission) + + expect(WebMock).to(have_requested(:post, "#{paperless_url}/api/documents/post_document/") + .with { |req| req.body.include?('Employment Contract') && req.body.include?('Signer') }) + end + + it 'sends the created date' do + described_class.call(submission) + + expect(WebMock).to(have_requested(:post, "#{paperless_url}/api/documents/post_document/") + .with { |req| req.body.include?('created') }) + end + + it 'returns the task UUIDs from paperless-ngx' do + result = described_class.call(submission) + + expect(result).to include('550e8400-e29b-41d4-a716-446655440000') + end + end + + context 'when submission has an audit trail' do + before do + blob = ActiveStorage::Blob.create_and_upload!( + io: Rails.root.join('spec/fixtures/sample-document.pdf').open, + filename: 'audit-trail.pdf', + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob:, name: 'audit_trail', record: submission) + end + + it 'uploads the audit trail to paperless-ngx' do + described_class.call(submission) + + expect(WebMock).to(have_requested(:post, "#{paperless_url}/api/documents/post_document/") + .with { |req| req.body.include?('Audit') } + .once) + end + end + + context 'when submission has both combined document and audit trail' do + before do + combined_blob = ActiveStorage::Blob.create_and_upload!( + io: Rails.root.join('spec/fixtures/sample-document.pdf').open, + filename: 'combined-result.pdf', + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob: combined_blob, name: 'combined_document', record: submission) + + audit_blob = ActiveStorage::Blob.create_and_upload!( + io: Rails.root.join('spec/fixtures/sample-document.pdf').open, + filename: 'audit-trail.pdf', + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob: audit_blob, name: 'audit_trail', record: submission) + end + + it 'uploads both documents to paperless-ngx' do + described_class.call(submission) + + expect(WebMock).to have_requested(:post, "#{paperless_url}/api/documents/post_document/").twice + end + end + + context 'when submission has no combined document but has submitter documents' do + before do + submitter = submission.submitters.first + blob = ActiveStorage::Blob.create_and_upload!( + io: Rails.root.join('spec/fixtures/sample-document.pdf').open, + filename: 'signed-result.pdf', + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob:, name: 'documents', record: submitter) + end + + it 'uploads the submitter documents to paperless-ngx' do + described_class.call(submission) + + expect(WebMock).to have_requested(:post, "#{paperless_url}/api/documents/post_document/") + .at_least_once + end + end + + context 'when paperless-ngx env vars are not configured' do + before do + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(nil) + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(nil) + end + + it 'does nothing and returns nil' do + result = described_class.call(submission) + + expect(result).to be_nil + expect(WebMock).not_to have_requested(:post, /paperless/) + end + end + + context 'when paperless-ngx returns an error' do + before do + stub_request(:post, "#{paperless_url}/api/documents/post_document/") + .to_return(status: 500, body: 'Internal Server Error') + + blob = ActiveStorage::Blob.create_and_upload!( + io: Rails.root.join('spec/fixtures/sample-document.pdf').open, + filename: 'combined-result.pdf', + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission) + end + + it 'raises an UploadError' do + expect { described_class.call(submission) }.to raise_error(Submissions::UploadToPaperless::UploadError) + end + end + + context 'when paperless-ngx connection fails' do + before do + stub_request(:post, "#{paperless_url}/api/documents/post_document/") + .to_timeout + + blob = ActiveStorage::Blob.create_and_upload!( + io: Rails.root.join('spec/fixtures/sample-document.pdf').open, + filename: 'combined-result.pdf', + content_type: 'application/pdf' + ) + ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission) + end + + it 'raises a connection error' do + expect { described_class.call(submission) }.to raise_error(Faraday::ConnectionFailed) + end + end + end + + describe '.configured?' do + context 'when both env vars are set' do + it 'returns true' do + expect(described_class.configured?).to be true + end + end + + context 'when URL is missing' do + before do + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(nil) + end + + it 'returns false' do + expect(described_class.configured?).to be false + end + end + + context 'when token is missing' do + before do + allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(nil) + end + + it 'returns false' do + expect(described_class.configured?).to be false + end + end + end +end