feat: add paperless-ngx integration for completed documents (#13)

Upload signed documents to paperless-ngx when all parties complete signing.
Triggered automatically after audit trail generation in the submission
completion flow.

Features:
- Uploads combined PDF (or individual results) + audit trail
- Title format: 'Template Name - Signer 1, Signer 2'
- ENV-based config (PAPERLESS_NGX_URL, PAPERLESS_NGX_TOKEN)
- Feature inactive unless both vars are set
- Background job with exponential retry (max 10 attempts)
- Dedicated 'integrations' Sidekiq queue

Files:
- lib/submissions/upload_to_paperless.rb — Faraday multipart upload
- app/jobs/upload_to_paperless_job.rb — Sidekiq job with retry
- app/jobs/process_submitter_completion_job.rb — hook (2 lines)
- config/sidekiq.yml — new queue
- docker-compose.e2e.yml — paperless-ngx for integration tests
- spec/ — unit tests (18 examples) + E2E tests (3 examples)

Co-authored-by: Sebastian Noe <sebastian.schneider@boxine.de>
pull/681/head
Sebastian Noe 1 month ago committed by GitHub
parent dada3013eb
commit df1ced2165
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -60,6 +60,25 @@ Reminder emails are sent to pending signers on a configurable schedule.
- Deduplication guard prevents the same reminder from being sent twice within 1 minute - Deduplication guard prevents the same reminder from being sent twice within 1 minute
- Job scheduling handles container restarts gracefully (clears stale scheduled jobs before re-registering) - Job scheduling handles container restarts gracefully (clears stale scheduled jobs before re-registering)
## Paperless-ngx Integration
Automatically upload completed, fully-signed documents to a [paperless-ngx](https://docs.paperless-ngx.com/) instance for archival and full-text search.
**How it works:**
- When all parties have signed a submission, the combined result PDF and audit trail are uploaded to paperless-ngx via its REST API.
- If the combined PDF feature is not enabled, individual per-submitter result PDFs are uploaded instead.
- Documents are titled `"Template Name - Signer 1, Signer 2"` with the signing completion date.
- Uploads run as a background job with exponential retry (up to 10 attempts) — they never block the signing flow.
**Configuration (env vars only — no GUI):**
| Variable | Description |
|----------|-------------|
| `PAPERLESS_NGX_URL` | Base URL of your paperless-ngx instance (e.g., `http://paperless:8000`) |
| `PAPERLESS_NGX_TOKEN` | API token for authentication ([how to get one](https://docs.paperless-ngx.com/api/#authorization)) |
The feature is inactive unless both variables are set.
## What's NOT included ## What's NOT included
These Pro features remain unavailable in this fork (they require significant UI/infrastructure work): These Pro features remain unavailable in this fork (they require significant UI/infrastructure work):

@ -20,6 +20,10 @@ class ProcessSubmitterCompletionJob
Submissions::EnsureAuditGenerated.call(submitter.submission) Submissions::EnsureAuditGenerated.call(submitter.submission)
enqueue_completed_emails(submitter) enqueue_completed_emails(submitter)
if Submissions::UploadToPaperless.configured?
UploadToPaperlessJob.perform_async('submission_id' => submitter.submission_id)
end
end end
create_completed_documents!(submitter) create_completed_documents!(submitter)

@ -0,0 +1,31 @@
# frozen_string_literal: true
class UploadToPaperlessJob
include Sidekiq::Job
sidekiq_options queue: :integrations
MAX_ATTEMPTS = 10
def perform(params = {})
return unless Submissions::UploadToPaperless.configured?
submission = Submission.find_by(id: params['submission_id'])
return unless submission
attempt = params['attempt'].to_i
Submissions::UploadToPaperless.call(submission)
rescue Submissions::UploadToPaperless::UploadError, Faraday::Error => e
return if attempt >= MAX_ATTEMPTS
Rails.logger.warn("Paperless-ngx upload failed (attempt #{attempt}): #{e.message}")
UploadToPaperlessJob.perform_in(
(2**attempt).minutes,
'submission_id' => params['submission_id'],
'attempt' => attempt + 1
)
end
end

@ -4,6 +4,7 @@ queues:
- [sms, 2] - [sms, 2]
- [images, 1] - [images, 1]
- [mailers, 1] - [mailers, 1]
- [integrations, 1]
- [recurrent, 1] - [recurrent, 1]
- [rollbar, 1] - [rollbar, 1]

@ -0,0 +1,59 @@
services:
rspec:
build:
context: .
dockerfile: Dockerfile.ci
command: sh -c "bundle exec rake db:create db:migrate && bundle exec rake assets:precompile && bundle exec rspec spec/integration"
depends_on:
postgres:
condition: service_healthy
paperless-ngx:
condition: service_healthy
environment:
RAILS_ENV: test
NODE_ENV: test
DATABASE_URL: postgres://postgres:postgres@postgres:5432/docuseal_test
PAPERLESS_NGX_URL: http://paperless-ngx:8000
PAPERLESS_NGX_TOKEN: ""
postgres:
image: postgres:18
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: docuseal_test
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s
timeout: 5s
retries: 5
tmpfs:
- /var/lib/postgresql
paperless-redis:
image: docker.io/library/redis:8
tmpfs:
- /data
paperless-ngx:
image: ghcr.io/paperless-ngx/paperless-ngx:latest
depends_on:
- paperless-redis
ports:
- "8000:8000"
environment:
PAPERLESS_REDIS: redis://paperless-redis:6379
PAPERLESS_SECRET_KEY: test-secret-for-e2e-testing
PAPERLESS_ADMIN_USER: admin
PAPERLESS_ADMIN_PASSWORD: admin
PAPERLESS_ADMIN_MAIL: admin@test.local
PAPERLESS_TIME_ZONE: UTC
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000"]
interval: 10s
timeout: 10s
retries: 10
start_period: 30s
tmpfs:
- /usr/src/paperless/data
- /usr/src/paperless/media

@ -0,0 +1,129 @@
# frozen_string_literal: true
module Submissions
module UploadToPaperless
UploadError = Class.new(StandardError)
BOUNDARY_PREFIX = '----DocuSealPaperlessUpload'
module_function
def call(submission)
return unless configured?
submission.submitters.load unless submission.submitters.loaded?
title = build_title(submission)
created = completed_date(submission)
results = documents_to_upload(submission, title).map do |attachment, doc_title|
upload_document(attachment, title: doc_title, created:)
end
results.compact.presence
end
def configured?
ENV['PAPERLESS_NGX_URL'].present? && ENV['PAPERLESS_NGX_TOKEN'].present?
end
def documents_to_upload(submission, title)
documents = []
if submission.combined_document.attached?
documents << [submission.combined_document, title]
else
submission.submitters.select(&:completed_at?).each do |submitter|
submitter.documents.each do |doc|
documents << [doc, title]
end
end
end
documents << [submission.audit_trail, "#{title} - Audit Trail"] if submission.audit_trail.attached?
documents
end
def build_title(submission)
submitter_names = submission.submitters
.select(&:completed_at?)
.sort_by(&:completed_at)
.filter_map(&:name)
.join(', ')
template_name = submission.template&.name || 'Document'
if submitter_names.present?
"#{template_name} - #{submitter_names}"
else
template_name
end
end
def completed_date(submission)
last_completed = submission.submitters.filter_map(&:completed_at).max
last_completed&.strftime('%Y-%m-%d')
end
def upload_document(attachment, title:, created:)
blob = attachment.blob
filename = sanitize_filename(blob.filename.to_s)
boundary = "#{BOUNDARY_PREFIX}#{SecureRandom.hex(16)}"
blob.open do |tempfile|
body = build_multipart_body(tempfile, filename:, title:, created:, boundary:)
response = connection.post('/api/documents/post_document/') do |req|
req.headers['Authorization'] = "Token #{ENV['PAPERLESS_NGX_TOKEN']}" # rubocop:disable Style/FetchEnvVar
req.headers['Content-Type'] = "multipart/form-data; boundary=#{boundary}"
req.body = body
req.options.read_timeout = 30
req.options.open_timeout = 10
end
if response.status >= 400
body_preview = response.body.to_s.truncate(200)
raise UploadError, "Paperless-ngx upload failed (HTTP #{response.status}): #{body_preview}"
end
response.body.to_s.delete('"').strip
end
end
def sanitize_filename(filename)
filename.gsub(/["\r\n\\]/, '_')
end
def build_multipart_body(tempfile, filename:, title:, created:, boundary:)
parts = []
parts << "--#{boundary}\r\n"
parts << "Content-Disposition: form-data; name=\"document\"; filename=\"#{filename}\"\r\n"
parts << "Content-Type: application/pdf\r\n\r\n"
parts << tempfile.read
parts << "\r\n"
parts << "--#{boundary}\r\n"
parts << "Content-Disposition: form-data; name=\"title\"\r\n\r\n"
parts << title
parts << "\r\n"
if created
parts << "--#{boundary}\r\n"
parts << "Content-Disposition: form-data; name=\"created\"\r\n\r\n"
parts << created
parts << "\r\n"
end
parts << "--#{boundary}--\r\n"
parts.join
end
def connection
Faraday.new(url: ENV['PAPERLESS_NGX_URL']) do |f| # rubocop:disable Style/FetchEnvVar
f.adapter Faraday.default_adapter
end
end
end
end

@ -0,0 +1,97 @@
# frozen_string_literal: true
require 'rails_helper'
# rubocop:disable RSpec/DescribeClass, RSpec/InstanceVariable, RSpec/ExpectInHook
# Run with: docker compose -f docker-compose.e2e.yml up
# This spec requires a running paperless-ngx instance.
# It acquires a token from paperless-ngx, uploads a document, and verifies it was received.
RSpec.describe 'Paperless-ngx document upload', type: :integration do
let(:paperless_url) { ENV.fetch('PAPERLESS_NGX_URL', 'http://paperless-ngx:8000') }
let(:account) { create(:account) }
let(:user) { create(:user, account:) }
let(:template) { create(:template, account:, author: user, name: 'Integration Test Contract') }
let(:submission) { create(:submission, :with_submitters, template:, created_by_user: user) }
# Generate a unique PDF per test by appending a random trailer comment.
# This ensures paperless-ngx does not deduplicate uploads across tests.
def unique_pdf_io
base_pdf = Rails.root.join('spec/fixtures/sample-document.pdf').binread
unique_pdf = base_pdf + "\n% #{SecureRandom.uuid}\n"
StringIO.new(unique_pdf)
end
before do
# Skip if paperless-ngx is not reachable
WebMock.allow_net_connect!
begin
health = Faraday.get("#{paperless_url}/api/") { |req| req.options.open_timeout = 5 }
skip "Paperless-ngx not available (HTTP #{health.status})" unless health.status < 400
rescue Faraday::Error
skip 'Paperless-ngx not available (connection failed)'
end
# Acquire a token from paperless-ngx
token_response = Faraday.post("#{paperless_url}/api/token/") do |req|
req.headers['Content-Type'] = 'application/json'
req.body = { username: 'admin', password: 'admin' }.to_json
end
expect(token_response.status).to eq(200), "Failed to get paperless-ngx token: #{token_response.body}"
@paperless_token = JSON.parse(token_response.body)['token']
# Configure ENV for the upload module
allow(ENV).to receive(:fetch).and_call_original
allow(ENV).to receive(:[]).and_call_original
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(paperless_url)
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(@paperless_token)
# Mark all submitters as completed
submission.submitters.each_with_index do |submitter, i|
submitter.update!(completed_at: i.hours.ago, name: "Test Signer #{i + 1}")
end
# Attach a combined document with unique content per test
blob = ActiveStorage::Blob.create_and_upload!(
io: unique_pdf_io,
filename: "integration-test-#{SecureRandom.hex(4)}.pdf",
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission)
end
after do
WebMock.disable_net_connect!(allow_localhost: true)
end
it 'uploads a signed document to paperless-ngx and receives a task UUID' do
result = Submissions::UploadToPaperless.call(submission)
expect(result).to be_present
expect(result.first).to match(/\A[0-9a-f-]+\z/)
end
it 'uploads via the background job without error' do
expect do
UploadToPaperlessJob.new.perform('submission_id' => submission.id)
end.not_to raise_error
end
it 'document is consumable by paperless-ngx' do
result = Submissions::UploadToPaperless.call(submission)
task_id = result.first
# Poll the tasks API to verify consumption started
response = Faraday.get("#{paperless_url}/api/tasks/?task_id=#{task_id}") do |req|
req.headers['Authorization'] = "Token #{@paperless_token}"
end
expect(response.status).to eq(200)
tasks = JSON.parse(response.body)
expect(tasks).to be_present
end
end
# rubocop:enable RSpec/DescribeClass, RSpec/InstanceVariable, RSpec/ExpectInHook

@ -0,0 +1,112 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe UploadToPaperlessJob do
let(:account) { create(:account) }
let(:user) { create(:user, account:) }
let(:template) { create(:template, account:, author: user) }
let(:submission) { create(:submission, :with_submitters, template:, created_by_user: user) }
let(:paperless_url) { 'http://paperless:8000' }
let(:paperless_token) { 'test-token-abc123' }
before do
submission.submitters.each_with_index do |submitter, i|
submitter.update!(completed_at: i.hours.ago, name: "Signer #{i + 1}")
end
allow(ENV).to receive(:fetch).and_call_original
allow(ENV).to receive(:[]).and_call_original
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(paperless_url)
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(paperless_token)
stub_request(:post, "#{paperless_url}/api/documents/post_document/")
.to_return(status: 200, body: '"task-uuid-123"')
end
describe '#perform' do
context 'when paperless-ngx is configured' do
before do
blob = ActiveStorage::Blob.create_and_upload!(
io: Rails.root.join('spec/fixtures/sample-document.pdf').open,
filename: 'combined-result.pdf',
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission)
end
it 'uploads documents to paperless-ngx' do
described_class.new.perform('submission_id' => submission.id)
expect(WebMock).to have_requested(:post, "#{paperless_url}/api/documents/post_document/")
.at_least_once
end
end
context 'when paperless-ngx is not configured' do
before do
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(nil)
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(nil)
end
it 'does nothing' do
described_class.new.perform('submission_id' => submission.id)
expect(WebMock).not_to have_requested(:post, /paperless/)
end
end
context 'when submission does not exist' do
it 'does nothing' do
described_class.new.perform('submission_id' => -1)
expect(WebMock).not_to have_requested(:post, /paperless/)
end
end
context 'when upload fails with a retryable error' do
before do
stub_request(:post, "#{paperless_url}/api/documents/post_document/")
.to_return(status: 500, body: 'Internal Server Error')
blob = ActiveStorage::Blob.create_and_upload!(
io: Rails.root.join('spec/fixtures/sample-document.pdf').open,
filename: 'combined-result.pdf',
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission)
end
it 'enqueues a retry with incremented attempt' do
expect do
described_class.new.perform('submission_id' => submission.id, 'attempt' => 0)
end.to change(described_class.jobs, :size).by(1)
args = described_class.jobs.last['args'].first
expect(args['attempt']).to eq(1)
expect(args['submission_id']).to eq(submission.id)
end
end
context 'when max attempts is reached' do
before do
stub_request(:post, "#{paperless_url}/api/documents/post_document/")
.to_return(status: 500, body: 'Internal Server Error')
blob = ActiveStorage::Blob.create_and_upload!(
io: Rails.root.join('spec/fixtures/sample-document.pdf').open,
filename: 'combined-result.pdf',
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission)
end
it 'does not enqueue another retry' do
expect do
described_class.new.perform('submission_id' => submission.id, 'attempt' => 11)
end.not_to change(described_class.jobs, :size)
end
end
end
end

@ -0,0 +1,208 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Submissions::UploadToPaperless do
let(:account) { create(:account) }
let(:user) { create(:user, account:) }
let(:template) { create(:template, account:, author: user, name: 'Employment Contract') }
let(:submission) { create(:submission, :with_submitters, template:, created_by_user: user) }
let(:paperless_url) { 'http://paperless:8000' }
let(:paperless_token) { 'test-token-abc123' }
before do
submission.submitters.each_with_index do |submitter, i|
submitter.update!(completed_at: i.hours.ago, name: "Signer #{i + 1}")
end
allow(ENV).to receive(:fetch).and_call_original
allow(ENV).to receive(:[]).and_call_original
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(paperless_url)
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(paperless_token)
stub_request(:post, "#{paperless_url}/api/documents/post_document/")
.to_return(status: 200, body: '"550e8400-e29b-41d4-a716-446655440000"')
end
describe '.call' do
context 'when submission has a combined document' do
before do
blob = ActiveStorage::Blob.create_and_upload!(
io: Rails.root.join('spec/fixtures/sample-document.pdf').open,
filename: 'combined-result.pdf',
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission)
end
it 'uploads the combined document to paperless-ngx' do
described_class.call(submission)
expect(WebMock).to have_requested(:post, "#{paperless_url}/api/documents/post_document/")
.with(headers: { 'Authorization' => "Token #{paperless_token}" })
.once
end
it 'sends the correct title with template name and submitter names' do
described_class.call(submission)
expect(WebMock).to(have_requested(:post, "#{paperless_url}/api/documents/post_document/")
.with { |req| req.body.include?('Employment Contract') && req.body.include?('Signer') })
end
it 'sends the created date' do
described_class.call(submission)
expect(WebMock).to(have_requested(:post, "#{paperless_url}/api/documents/post_document/")
.with { |req| req.body.include?('created') })
end
it 'returns the task UUIDs from paperless-ngx' do
result = described_class.call(submission)
expect(result).to include('550e8400-e29b-41d4-a716-446655440000')
end
end
context 'when submission has an audit trail' do
before do
blob = ActiveStorage::Blob.create_and_upload!(
io: Rails.root.join('spec/fixtures/sample-document.pdf').open,
filename: 'audit-trail.pdf',
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob:, name: 'audit_trail', record: submission)
end
it 'uploads the audit trail to paperless-ngx' do
described_class.call(submission)
expect(WebMock).to(have_requested(:post, "#{paperless_url}/api/documents/post_document/")
.with { |req| req.body.include?('Audit') }
.once)
end
end
context 'when submission has both combined document and audit trail' do
before do
combined_blob = ActiveStorage::Blob.create_and_upload!(
io: Rails.root.join('spec/fixtures/sample-document.pdf').open,
filename: 'combined-result.pdf',
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob: combined_blob, name: 'combined_document', record: submission)
audit_blob = ActiveStorage::Blob.create_and_upload!(
io: Rails.root.join('spec/fixtures/sample-document.pdf').open,
filename: 'audit-trail.pdf',
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob: audit_blob, name: 'audit_trail', record: submission)
end
it 'uploads both documents to paperless-ngx' do
described_class.call(submission)
expect(WebMock).to have_requested(:post, "#{paperless_url}/api/documents/post_document/").twice
end
end
context 'when submission has no combined document but has submitter documents' do
before do
submitter = submission.submitters.first
blob = ActiveStorage::Blob.create_and_upload!(
io: Rails.root.join('spec/fixtures/sample-document.pdf').open,
filename: 'signed-result.pdf',
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob:, name: 'documents', record: submitter)
end
it 'uploads the submitter documents to paperless-ngx' do
described_class.call(submission)
expect(WebMock).to have_requested(:post, "#{paperless_url}/api/documents/post_document/")
.at_least_once
end
end
context 'when paperless-ngx env vars are not configured' do
before do
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(nil)
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(nil)
end
it 'does nothing and returns nil' do
result = described_class.call(submission)
expect(result).to be_nil
expect(WebMock).not_to have_requested(:post, /paperless/)
end
end
context 'when paperless-ngx returns an error' do
before do
stub_request(:post, "#{paperless_url}/api/documents/post_document/")
.to_return(status: 500, body: 'Internal Server Error')
blob = ActiveStorage::Blob.create_and_upload!(
io: Rails.root.join('spec/fixtures/sample-document.pdf').open,
filename: 'combined-result.pdf',
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission)
end
it 'raises an UploadError' do
expect { described_class.call(submission) }.to raise_error(Submissions::UploadToPaperless::UploadError)
end
end
context 'when paperless-ngx connection fails' do
before do
stub_request(:post, "#{paperless_url}/api/documents/post_document/")
.to_timeout
blob = ActiveStorage::Blob.create_and_upload!(
io: Rails.root.join('spec/fixtures/sample-document.pdf').open,
filename: 'combined-result.pdf',
content_type: 'application/pdf'
)
ActiveStorage::Attachment.create!(blob:, name: 'combined_document', record: submission)
end
it 'raises a connection error' do
expect { described_class.call(submission) }.to raise_error(Faraday::ConnectionFailed)
end
end
end
describe '.configured?' do
context 'when both env vars are set' do
it 'returns true' do
expect(described_class.configured?).to be true
end
end
context 'when URL is missing' do
before do
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_URL').and_return(nil)
end
it 'returns false' do
expect(described_class.configured?).to be false
end
end
context 'when token is missing' do
before do
allow(ENV).to receive(:[]).with('PAPERLESS_NGX_TOKEN').and_return(nil)
end
it 'returns false' do
expect(described_class.configured?).to be false
end
end
end
end
Loading…
Cancel
Save