docuseal/lib/templates/process_document.rb

# frozen_string_literal: true

module Templates
  module ProcessDocument
    DPI = 200
    FORMAT = '.png'
    ATTACHMENT_NAME = 'preview_images'

    PDF_CONTENT_TYPE = 'application/pdf'
    CONCURRENCY = 2
    Q = 95
    JPEG_Q = ENV.fetch('PAGE_QUALITY', '35').to_i
    MAX_WIDTH = 1400
    MAX_NUMBER_OF_PAGES_PROCESSED = 15
    MAX_FLATTEN_FILE_SIZE = 20.megabytes
    GENERATE_PREVIEW_SIZE_LIMIT = 50.megabytes

    module_function

    def call(attachment, data, extract_fields: false, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED)
      if attachment.content_type == PDF_CONTENT_TYPE
        if extract_fields && data.size < MAX_FLATTEN_FILE_SIZE
          pdf = HexaPDF::Document.new(io: StringIO.new(data))

          fields = Templates::FindAcroFields.call(pdf, attachment)
        end

        generate_pdf_preview_images(attachment, data, pdf, max_pages:)

        attachment.metadata['pdf']['fields'] = fields if fields
      elsif attachment.image?
        generate_preview_image(attachment, data)
      end

      attachment
    end

    def generate_preview_image(attachment, data)
      ActiveStorage::Attachment.where(name: ATTACHMENT_NAME, record: attachment).destroy_all

      image = Vips::Image.new_from_buffer(data, '')
      image = image.autorot.resize(MAX_WIDTH / image.width.to_f)

      bitdepth = 2**image.stats.to_a[1..3].pluck(2).uniq.size

      io = StringIO.new(image.write_to_buffer(FORMAT, compression: 7, filter: 0, bitdepth:,
                                                      palette: true, Q: bitdepth == 8 ? Q : 5, dither: 0))

      ActiveStorage::Attachment.create!(
        blob: ActiveStorage::Blob.create_and_upload!(
          io:, filename: "0#{FORMAT}",
          metadata: { analyzed: true, identified: true, width: image.width, height: image.height }
        ),
        name: ATTACHMENT_NAME,
        record: attachment
      )
    end

    def generate_pdf_preview_images(attachment, data, pdf = nil, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED)
      ActiveStorage::Attachment.where(name: ATTACHMENT_NAME, record: attachment).destroy_all

      pdf ||= HexaPDF::Document.new(io: StringIO.new(data))
      number_of_pages = pdf.pages.size

      data = maybe_flatten_form(data, pdf)

      attachment.metadata['pdf'] ||= {}
      attachment.metadata['pdf']['number_of_pages'] = number_of_pages

      ApplicationRecord.no_touching do
        attachment.save!
      end

      max_pages_to_process = data.size < GENERATE_PREVIEW_SIZE_LIMIT ? max_pages : 1

      generate_document_preview_images(attachment, data, (0..[number_of_pages - 1, max_pages_to_process].min))
    end

    def generate_document_preview_images(attachment, data, range, concurrency: CONCURRENCY)
      pool = Concurrent::FixedThreadPool.new(concurrency)

      promises =
        range.map do |page_number|
          Concurrent::Promise.execute(executor: pool) { build_and_upload_blob(data, page_number) }
        end

      Concurrent::Promise.zip(*promises).value!.each do |blob|
        next unless blob

        ApplicationRecord.no_touching do
          ActiveStorage::Attachment.create!(
            blob:,
            name: ATTACHMENT_NAME,
            record: attachment
          )
        end
      end

      pool.kill
    end

    def build_and_upload_blob(data, page_number)
      page = Vips::Image.new_from_buffer(data, '', dpi: DPI, page: page_number)
      page = page.resize(MAX_WIDTH / page.width.to_f)

      bitdepth = 2**page.stats.to_a[1..3].pluck(2).uniq.size

      io = StringIO.new(page.write_to_buffer(FORMAT, compression: 7, filter: 0, bitdepth:,
                                                     palette: true, Q: bitdepth == 8 ? Q : 5, dither: 0))

      blob = ActiveStorage::Blob.new(
        filename: "#{page_number}#{FORMAT}",
        metadata: { analyzed: true, identified: true, width: page.width, height: page.height }
      )

      blob.upload(io)

      blob
    rescue Vips::Error => e
      Rollbar.warning(e) if defined?(Rollbar)

      nil
    end

    def maybe_flatten_form(data, pdf)
      return data if data.size > MAX_FLATTEN_FILE_SIZE
      return data if pdf.acro_form.blank?

      io = StringIO.new

      pdf.acro_form.each_field do |field|
        next if field.field_type != :Ch ||
                field[:Opt].blank? ||
                %i[combo_box editable_combo_box].exclude?(field.concrete_field_type) ||
                !field.field_value.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP)

        field[:V] = ''
      end

      pdf.acro_form.create_appearances(force: true) if pdf.acro_form[:NeedAppearances]
      pdf.acro_form.flatten

      pdf.write(io, incremental: false, validate: false)

      io.string
    rescue StandardError
      raise if Rails.env.development?

      data
    end

    def normalize_attachment_fields(template, attachments = template.documents)
      attachments.flat_map do |a|
        pdf_fields = a.metadata['pdf'].delete('fields').to_a if a.metadata['pdf'].present?

        next [] if pdf_fields.blank?

        pdf_fields.each { |f| f['submitter_uuid'] = template.submitters.first['uuid'] }

        pdf_fields
      end
    end

    def generate_pdf_preview_from_file(attachment, file_path, page_number)
      io = StringIO.new

      command = [
        'pdftocairo', '-jpeg', '-jpegopt', "progressive=y,quality=#{JPEG_Q},optimize=y",
        '-scale-to-x', MAX_WIDTH, '-scale-to-y', '-1',
        '-r', DPI, '-f', page_number + 1, '-l', page_number + 1,
        '-singlefile', Shellwords.escape(file_path), '-'
      ].join(' ')

      Open3.popen3(command) do |_, stdout, _, _|
        io.write(stdout.read)

        io.rewind
      end

      page = Vips::Image.new_from_buffer(io.read, '')

      io.rewind

      ApplicationRecord.no_touching do
        ActiveStorage::Attachment.create!(
          blob: ActiveStorage::Blob.create_and_upload!(
            io:, filename: "#{page_number}.jpg",
            metadata: { analyzed: true, identified: true, width: page.width, height: page.height }
          ),
          name: ATTACHMENT_NAME,
          record: attachment
        )
      end
    end
  end
end