process pdf v2

lib/pdf_utils.rb

@@ -6,24 +6,6 @@ module PdfUtils
 
   module_function
 
-  def encrypted?(data, password: nil)
-    HexaPDF::Document.new(io: StringIO.new(data), decryption_opts: { password: })
-
-    false
-  rescue HexaPDF::EncryptionError
-    true
-  end
-
-  def decrypt(data, password)
-    decrypted_io = StringIO.new
-
-    Pdfium::Document.open_bytes(data, password) do |doc|
-      doc.save(decrypted_io, flags: Pdfium::FPDF_REMOVE_SECURITY)
-    end
-
-    decrypted_io.tap(&:rewind).read
-  end
-
   def merge(io_files)
     merged_content = StringIO.new
 

lib/templates/build_annotations.rb

@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-module Templates
-  module BuildAnnotations
-    module_function
-
-    def call(data)
-      pdf = HexaPDF::Document.new(io: StringIO.new(data))
-
-      pdf.pages.flat_map.with_index do |page, index|
-        (page[:Annots] || []).filter_map do |annot|
-          next if annot.blank?
-          next if annot.is_a?(Integer) || annot.is_a?(Symbol) || annot.is_a?(HexaPDF::PDFArray)
-          next if annot[:A].blank? || annot[:A][:URI].blank?
-          next unless annot[:Subtype] == :Link
-          next if !annot[:A][:URI].starts_with?('https://') && !annot[:A][:URI].starts_with?('http://')
-
-          build_external_link_hash(page, annot).merge('page' => index)
-        end
-      end
-    rescue StandardError => e
-      Rollbar.error(e) if defined?(Rollbar)
-
-      []
-    end
-
-    def build_external_link_hash(page, annot)
-      left, bottom, right, top = annot[:Rect]
-
-      {
-        'type' => 'external_link',
-        'value' => annot[:A][:URI],
-        'x' => left / page.box.width.to_f,
-        'y' => (page.box.height - top) / page.box.height.to_f,
-        'w' => (right - left) / page.box.width.to_f,
-        'h' => (top - bottom) / page.box.height.to_f
-      }
-    end
-  end
-end

lib/templates/create_attachments.rb

@@ -39,23 +39,6 @@ module Templates
     end
 
     def handle_pdf_or_image(template, file, document_data = nil, params = {}, extract_fields: false, metadata: {})
-      return handle_pdf_or_image_v2(template, file, document_data, params, extract_fields:, metadata:) if v2?
-
-      document_data ||= file.read
-
-      if file.content_type == PDF_CONTENT_TYPE
-        document_data = maybe_decrypt_pdf_or_raise(document_data, params)
-
-        annotations =
-          document_data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildAnnotations.call(document_data) : []
-      end
-
-      document = create_document(template, file, document_data, metadata, annotations)
-
-      Templates::ProcessDocument.call(document, document_data, extract_fields:)
-    end
-
-    def handle_pdf_or_image_v2(template, file, document_data = nil, params = {}, extract_fields: false, metadata: {})
       document_data ||= file.read
 
       unless file.content_type == PDF_CONTENT_TYPE
@@ -80,16 +63,6 @@ module Templates
       doc&.close
     end
 
-    def maybe_decrypt_pdf_or_raise(data, params)
-      if data.size < ANNOTATIONS_SIZE_LIMIT && PdfUtils.encrypted?(data)
-        PdfUtils.decrypt(data, params[:password])
-      else
-        data
-      end
-    rescue Pdfium::PasswordError
-      raise PdfEncrypted
-    end
-
     def decrypt_document(doc)
       io = StringIO.new
 
@@ -163,9 +136,5 @@ module Templates
 
       raise InvalidFileType, "#{file.content_type}/#{dynamic}"
     end
-
-    def v2?
-      true
-    end
   end
 end

lib/templates/find_acro_fields.rb

@@ -1,288 +0,0 @@
-# frozen_string_literal: true
-
-module Templates
-  module FindAcroFields
-    PDF_CONTENT_TYPE = 'application/pdf'
-
-    FIELD_NAME_REGEXP = /\A(?=.*\p{L})[\p{L}\d\s-]+\z/
-    SKIP_FIELD_DESCRIPTION = %w[undefined].freeze
-    SELECT_PLACEHOLDER_REGEXP = /\b(
-      Select      |
-      Choose      |
-      Wählen      |
-      Auswählen   |
-      Sélectionner|
-      Choisir     |
-      Seleccionar |
-      Elegir      |
-      Seleziona   |
-      Scegliere   |
-      Selecionar  |
-      Escolher
-    )\b/ix
-
-    DATE_FORMAT_REGEXP = %r{[myd]{2,4}[-\\/\s.][myd]{2,4}[-\\/\s.][myd]{2,4}}i
-
-    FIELD_ALIGNMENT = {
-      0 => 'left',
-      1 => 'center',
-      2 => 'right'
-    }.freeze
-
-    module_function
-
-    # rubocop:disable Metrics
-    def call(pdf, attachment, data)
-      return [] if pdf.acro_form.blank? && data.exclude?('/Form')
-
-      fields, annots_index = build_fields_with_pages(pdf)
-
-      fields.filter_map do |field|
-        areas = Array.wrap(field[:Kids] || field).filter_map do |child_field|
-          page = annots_index[child_field.hash]
-
-          next unless page
-
-          media_box = page[:CropBox] || page[:MediaBox]
-          crop_box = page[:CropBox] || media_box
-
-          media_box_start = [media_box[0], media_box[1]]
-          crop_shift = [crop_box[0] - media_box[0], crop_box[1] - media_box[1]]
-
-          next unless child_field[:Rect]
-
-          x0, y0, x1, y1 = child_field[:Rect]
-
-          x0, y0 = correct_coordinates(x0, y0, crop_shift, media_box_start)
-          x1, y1 = correct_coordinates(x1, y1, crop_shift, media_box_start)
-
-          page_width = media_box[2] - media_box[0]
-          page_height = media_box[3] - media_box[1]
-
-          x = x0
-          y = y0
-          w = x1 - x0
-          h = y1 - y0
-
-          transformed_y = page_height - y - h
-
-          attrs = {
-            page: page.index,
-            x: x / page_width.to_f,
-            y: transformed_y / page_height.to_f,
-            w: w / page_width.to_f,
-            h: h / page_height.to_f,
-            attachment_uuid: attachment.uuid
-          }
-
-          next if attrs[:w].zero? || attrs[:h].zero?
-
-          if child_field[:MaxLen] && child_field.try(:concrete_field_type) == :comb_text_field
-            attrs[:cell_w] = w / page_width.to_f / child_field[:MaxLen].to_f
-          end
-
-          attrs
-        end
-
-        next if areas.blank?
-
-        field_properties = build_field_properties(field)
-
-        next if field_properties.blank?
-        next if field_properties[:default_value].present?
-
-        if field_properties[:type].in?(%w[radio multiple])
-          if areas.size != field_properties[:options].size
-            field_properties[:options] = build_options(Array.new(areas.size, ''))
-          end
-
-          areas.each_with_index do |area, index|
-            area[:option_uuid] = field_properties[:options][index][:uuid]
-          end
-        end
-
-        {
-          uuid: SecureRandom.uuid,
-          required: field.flags.include?(:required),
-          preferences: {},
-          areas:,
-          **field_properties
-        }
-      end
-    rescue StandardError => e
-      raise if Rails.env.local?
-
-      Rollbar.error(e) if defined?(Rollbar)
-
-      []
-    end
-
-    def correct_coordinates(x_coord, y_coord, shift, media_box_start)
-      corrected_x = x_coord + shift[0] - media_box_start[0]
-      corrected_y = y_coord + shift[1] - media_box_start[1]
-
-      [corrected_x, corrected_y]
-    end
-
-    def build_field_properties(field)
-      field_name = field.full_field_name if field.full_field_name.to_s.match?(FIELD_NAME_REGEXP)
-
-      field_name = field_name&.encode('utf-8', invalid: :replace, undef: :replace, replace: '')
-
-      attrs = { name: field_name.to_s }
-      attrs[:description] = field[:TU] if field[:TU].present? &&
-                                          field[:TU] != field.full_field_name &&
-                                          !field[:TU].in?(SKIP_FIELD_DESCRIPTION)
-
-      if field[:Q].present? && field.field_type == :Tx
-        attrs[:preferences] ||= {}
-        attrs[:preferences][:align] = FIELD_ALIGNMENT.fetch(field[:Q], 'left')
-      end
-
-      if field.field_type == :Btn && field.concrete_field_type == :radio_button && field[:Opt].present?
-        selected_option_index = (field.allowed_values || []).find_index(field.field_value)
-        selected_option = field[:Opt][selected_option_index] if selected_option_index
-
-        {
-          **attrs,
-          type: 'radio',
-          options: build_options(field[:Opt], 'radio'),
-          default_value: selected_option
-        }
-      elsif field.field_type == :Btn && %i[check_box radio_button].include?(field.concrete_field_type) &&
-            field[:Kids].present? && field[:Kids].size > 1 && field.allowed_values.size > 1
-        selected_option = (field.allowed_values || []).find { |v| v == field.field_value }
-
-        return {} if field.allowed_values.include?(:BBox)
-
-        {
-          **attrs,
-          type: 'radio',
-          options: build_options(field.allowed_values, 'radio'),
-          default_value: selected_option
-        }
-      elsif field.field_type == :Btn && %i[check_box radio_button].include?(field.concrete_field_type)
-        {
-          **attrs,
-          type: 'checkbox',
-          default_value: field.field_value.present?
-        }
-      elsif field.field_type == :Ch &&
-            %i[combo_box editable_combo_box].include?(field.concrete_field_type) && field[:Opt].present?
-        {
-          **attrs,
-          type: 'select',
-          options: build_options(field[:Opt], 'select'),
-          default_value: field.field_value.to_s.match?(SELECT_PLACEHOLDER_REGEXP) ? nil : field.field_value.presence
-        }
-      elsif field.field_type == :Ch && field.concrete_field_type == :multi_select && field[:Opt].present?
-        {
-          **attrs,
-          type: 'multiple',
-          options: build_options(field[:Opt], 'multiple'),
-          default_value: field.field_value.presence
-        }
-      elsif field.field_type == :Tx && field.concrete_field_type == :comb_text_field
-        {
-          **attrs,
-          type: 'cells',
-          default_value: field.field_value.presence
-        }
-      elsif field.field_type == :Tx
-        if field[:AA] && ((field[:AA][:F] && field[:AA][:F][:JS].include?('AFDate_')) ||
-            (field[:AA][:K] && field[:AA][:K][:JS].include?('AFDate_')))
-          if (format = field[:AA][:F][:JS][DATE_FORMAT_REGEXP])
-            attrs[:preferences] ||= {}
-            attrs[:preferences][:format] = format.upcase
-          end
-
-          {
-            **attrs,
-            type: 'date',
-            default_value: field.field_value.presence
-          }
-        else
-          {
-            **attrs,
-            type: 'text',
-            default_value: field.field_value.presence
-          }
-        end
-      elsif field.field_type == :Sig
-        {
-          **attrs,
-          type: field.try(:field_name).to_s.downcase.include?('initials') ? 'initials' : 'signature'
-        }
-      else
-        {}
-      end.compact
-    end
-
-    def build_options(values, type = nil)
-      is_skip_single_value = type.in?(%w[radio multiple]) && values.uniq.size == 1
-
-      values.filter_map do |option|
-        is_option_number = option.is_a?(Symbol) && option.to_s.match?(/\A\d+\z/)
-
-        option = option[1] if option.is_a?(Array) && option.size == 2
-
-        if option.is_a?(String) || option.is_a?(Symbol)
-          option = option.to_s.encode('utf-8', invalid: :replace, undef: :replace, replace: '')
-        end
-
-        next if type == 'select' && option.to_s.match?(SELECT_PLACEHOLDER_REGEXP)
-
-        {
-          uuid: SecureRandom.uuid,
-          value: is_option_number || is_skip_single_value ? '' : option.presence
-        }
-      end
-    end
-
-    def build_fields_with_pages(pdf)
-      fields_index = {}
-      annots_index = {}
-
-      pdf.pages.each do |page|
-        page.each_annotation do |annot|
-          annots_index[annot.hash] = page
-
-          if !annot.key?(:Parent) && annot.key?(:FT)
-            fields_index[annot.hash] ||= HexaPDF::Type::AcroForm::Field.wrap(pdf, annot)
-          elsif annot.key?(:Parent)
-            field = annot[:Parent]
-            seen = Set.new.compare_by_identity
-
-            field = field[:Parent] while field[:Parent] && seen.add?(field.value)
-
-            annots_index[field.hash] ||= page
-            fields_index[field.hash] ||= HexaPDF::Type::AcroForm::Field.wrap(pdf, field)
-          end
-        end
-      end
-
-      [process_fields_array(pdf, fields_index.values), annots_index]
-    end
-
-    def process_fields_array(pdf, array, acc = [], seen = Set.new.compare_by_identity)
-      array.each_with_index do |field, index|
-        next if field.nil?
-
-        unless field.respond_to?(:type) && field.type == :XXAcroFormField
-          array[index] = field = HexaPDF::Type::AcroForm::Field.wrap(pdf, field)
-        end
-
-        next unless seen.add?(field.value)
-
-        if field.terminal_field?
-          acc << field
-        else
-          process_fields_array(pdf, field[:Kids], acc, seen)
-        end
-      end
-
-      acc
-    end
-    # rubocop:enable Metrics
-  end
-end

lib/templates/find_pdfium_acro_fields.rb

@@ -6,6 +6,31 @@ module Templates
     SKIP_FIELD_TYPES = %i[unknown pushbutton].freeze
     TEXT_OPERATOR_REGEXP = /\bT[jJ]\b/
 
+    FIELD_NAME_REGEXP = /\A(?=.*\p{L})[\p{L}\d\s-]+\z/
+    SKIP_FIELD_DESCRIPTION = %w[undefined].freeze
+    SELECT_PLACEHOLDER_REGEXP = /\b(
+      Select      |
+      Choose      |
+      Wählen      |
+      Auswählen   |
+      Sélectionner|
+      Choisir     |
+      Seleccionar |
+      Elegir      |
+      Seleziona   |
+      Scegliere   |
+      Selecionar  |
+      Escolher
+    )\b/ix
+
+    DATE_FORMAT_REGEXP = %r{[myd]{2,4}[-\\/\s.][myd]{2,4}[-\\/\s.][myd]{2,4}}i
+
+    FIELD_ALIGNMENT = {
+      0 => 'left',
+      1 => 'center',
+      2 => 'right'
+    }.freeze
+
     module_function
 
     def call(attachment, doc, data)
@@ -108,12 +133,12 @@ module Templates
     def build_field_properties(widgets)
       field = widgets.first.field
 
-      field_name = field.name if field.name.match?(FindAcroFields::FIELD_NAME_REGEXP)
+      field_name = field.name if field.name.match?(FIELD_NAME_REGEXP)
 
       attrs = { name: field_name.to_s }
       attrs[:description] = field.alternate_name if field.alternate_name.present? &&
                                                     field.alternate_name != field.name &&
-                                                    !field.alternate_name.in?(FindAcroFields::SKIP_FIELD_DESCRIPTION)
+                                                    !field.alternate_name.in?(SKIP_FIELD_DESCRIPTION)
 
       case field.type
       when :checkbox, :radio
@@ -172,7 +197,7 @@ module Templates
         **attrs,
         type: 'select',
         options: build_options(field.options, 'select'),
-        default_value: value.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP) ? nil : value
+        default_value: value.to_s.match?(SELECT_PLACEHOLDER_REGEXP) ? nil : value
       }
     end
 
@@ -183,7 +208,7 @@ module Templates
     end
 
     def build_text_properties(attrs, field)
-      preferences = { align: FindAcroFields::FIELD_ALIGNMENT.fetch(field.quadding.to_i, 'left') }
+      preferences = { align: FIELD_ALIGNMENT.fetch(field.quadding.to_i, 'left') }
 
       attrs = { **attrs, preferences: }
 
@@ -191,7 +216,7 @@ module Templates
         { **attrs, type: 'cells', default_value: field.value.presence }
       elsif date?(field)
         format = [field.format_js, field.keystroke_js].compact
-                                                      .filter_map { |js| js[FindAcroFields::DATE_FORMAT_REGEXP] }
+                                                      .filter_map { |js| js[DATE_FORMAT_REGEXP] }
                                                       .first
 
         preferences[:format] = format.upcase if format
@@ -218,7 +243,7 @@ module Templates
           option = option.to_s.encode('utf-8', invalid: :replace, undef: :replace, replace: '')
         end
 
-        next if type == 'select' && option.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP)
+        next if type == 'select' && option.to_s.match?(SELECT_PLACEHOLDER_REGEXP)
 
         {
           uuid: SecureRandom.uuid,

lib/templates/modify_documents.rb

@@ -435,7 +435,9 @@ module Templates
     end
 
     def save_document(template, old_attachment, data)
-      annotations = data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildAnnotations.call(data) : []
+      doc = Pdfium::Document.open_io(StringIO.new(data))
+
+      annotations = data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildPdfiumAnnotations.call(doc) : []
       sha256 = Base64.urlsafe_encode64(Digest::SHA256.digest(data))
 
       blob = ActiveStorage::Blob.create_and_upload!(
@@ -448,7 +450,9 @@ module Templates
 
       document = template.documents.create!(blob:)
 
-      Templates::ProcessDocument.call(document, data)
+      Templates::ProcessDocument.call(document, data, doc:)
+    ensure
+      doc&.close
     end
 
     def remap_fields(template, mapping)

lib/templates/process_document.rb

@@ -22,16 +22,10 @@ module Templates
     def call(attachment, data, extract_fields: false, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED, doc: nil)
       if attachment.content_type == PDF_CONTENT_TYPE
         if extract_fields && data.size < MAX_FLATTEN_FILE_SIZE
-          if doc
-            fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data)
-          else
-            pdf = HexaPDF::Document.new(io: StringIO.new(data))
-
-            fields = Templates::FindAcroFields.call(pdf, attachment, data)
-          end
+          fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data)
         end
 
-        generate_pdf_preview_images(attachment, data, pdf, max_pages:, doc:)
+        generate_pdf_preview_images(attachment, data, max_pages:, doc:)
 
         attachment.metadata['pdf']['fields'] = fields if fields
       elsif attachment.image?
@@ -41,27 +35,13 @@ module Templates
       attachment
     end
 
-    def process(attachment, data, extract_fields: false, doc: nil)
+    def process(attachment, data, doc:, extract_fields: false)
       if attachment.content_type == PDF_CONTENT_TYPE && extract_fields && data.size < MAX_FLATTEN_FILE_SIZE
-        if doc
-          fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data)
-        else
-          pdf = HexaPDF::Document.new(io: StringIO.new(data))
-
-          fields = Templates::FindAcroFields.call(pdf, attachment, data)
-        end
-      end
-
-      if doc
-        number_of_pages = doc.page_count
-      else
-        pdf ||= HexaPDF::Document.new(io: StringIO.new(data))
-
-        number_of_pages = pdf.pages.size
+        fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data)
       end
 
       attachment.metadata['pdf'] ||= {}
-      attachment.metadata['pdf']['number_of_pages'] = number_of_pages
+      attachment.metadata['pdf']['number_of_pages'] = doc.page_count
       attachment.metadata['pdf']['fields'] = fields if fields
 
       attachment
@@ -88,17 +68,10 @@ module Templates
       )
     end
 
-    def generate_pdf_preview_images(attachment, data, pdf = nil, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED, doc: nil)
+    def generate_pdf_preview_images(attachment, data, doc:, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED)
       ActiveStorage::Attachment.where(name: ATTACHMENT_NAME, record: attachment).destroy_all
 
-      if doc
-        number_of_pages = doc.page_count
-      else
-        pdf ||= HexaPDF::Document.new(io: StringIO.new(data))
-        number_of_pages = pdf.pages.size
-
-        data = maybe_flatten_form(data, pdf)
-      end
+      number_of_pages = doc.page_count
 
       attachment.metadata['pdf'] ||= {}
       attachment.metadata['pdf']['number_of_pages'] = number_of_pages
@@ -109,20 +82,19 @@ module Templates
 
       max_pages_to_process = data.size < GENERATE_PREVIEW_SIZE_LIMIT ? max_pages : 1
 
-      generate_document_preview_images(attachment, data, 0..[number_of_pages - 1, max_pages_to_process].min, doc:)
+      generate_document_preview_images(attachment, 0..[number_of_pages - 1, max_pages_to_process].min, doc:)
     end
 
-    def generate_document_preview_images(attachment, data, range, concurrency: CONCURRENCY, doc: nil)
-      flatten_pages = doc&.form?
-      pdfium_doc = doc || Pdfium::Document.open_bytes(data)
+    def generate_document_preview_images(attachment, range, doc:, concurrency: CONCURRENCY)
+      flatten_pages = doc.form?
 
       pool = Concurrent::FixedThreadPool.new(concurrency)
 
       promises =
         range.map do |page_number|
-          doc_page = pdfium_doc.get_page(page_number)
+          doc_page = doc.get_page(page_number)
 
-          hide_placeholder_widgets(doc_page, hide_empty: !flatten_pages) if doc
+          hide_placeholder_widgets(doc_page, hide_empty: !flatten_pages)
           doc_page.flatten if flatten_pages
 
           bytes, width, height = doc_page.render_to_bitmap(width: MAX_WIDTH)
@@ -146,7 +118,6 @@ module Templates
         end
       end
     ensure
-      pdfium_doc&.close if doc.nil?
       pool&.kill
     end
 
@@ -162,7 +133,7 @@ module Templates
           if value.blank?
             next unless hide_empty
           elsif handle.option_labels.blank? ||
-                !value.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP)
+                !value.match?(FindPdfiumAcroFields::SELECT_PLACEHOLDER_REGEXP)
             next
           end
 
@@ -194,33 +165,6 @@ module Templates
       blob
     end
 
-    def maybe_flatten_form(data, pdf)
-      return data if data.size > MAX_FLATTEN_FILE_SIZE
-      return data if pdf.acro_form.blank?
-
-      io = StringIO.new
-
-      pdf.acro_form.each_field do |field|
-        next if field.field_type != :Ch ||
-                field[:Opt].blank? ||
-                %i[combo_box editable_combo_box].exclude?(field.concrete_field_type) ||
-                !field.field_value.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP)
-
-        field[:V] = ''
-      end
-
-      pdf.acro_form.create_appearances(force: true) if pdf.acro_form[:NeedAppearances]
-      pdf.acro_form.flatten
-
-      pdf.write(io, incremental: false, validate: false)
-
-      io.string
-    rescue StandardError
-      raise if Rails.env.development?
-
-      data
-    end
-
     def normalize_attachment_fields(template, attachments = template.documents)
       attachments.flat_map do |a|
         pdf_fields = a.metadata['pdf'].delete('fields').to_a if a.metadata['pdf'].present?