process attachment v2

lib/templates/create_attachments.rb

@@ -39,6 +39,8 @@ module Templates
     end
 
     def handle_pdf_or_image(template, file, document_data = nil, params = {}, extract_fields: false, metadata: {})
+      return handle_pdf_or_image_v2(template, file, document_data, params, extract_fields:, metadata:) if v2?
+
       document_data ||= file.read
 
       if file.content_type == PDF_CONTENT_TYPE
@@ -48,6 +50,55 @@ module Templates
           document_data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildAnnotations.call(document_data) : []
       end
 
+      document = create_document(template, file, document_data, metadata, annotations)
+
+      Templates::ProcessDocument.call(document, document_data, extract_fields:)
+    end
+
+    def handle_pdf_or_image_v2(template, file, document_data = nil, params = {}, extract_fields: false, metadata: {})
+      document_data ||= file.read
+
+      unless file.content_type == PDF_CONTENT_TYPE
+        document = create_document(template, file, document_data, metadata)
+
+        return Templates::ProcessDocument.call(document, document_data, extract_fields:)
+      end
+
+      doc = Pdfium::Document.open_bytes(document_data, params[:password])
+
+      document_data = decrypt_document(doc) if doc.encrypted?
+
+      annotations =
+        document_data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildPdfiumAnnotations.call(doc) : []
+
+      document = create_document(template, file, document_data, metadata, annotations)
+
+      Templates::ProcessDocument.call(document, document_data, extract_fields:, doc:)
+    rescue Pdfium::PasswordError
+      raise PdfEncrypted
+    ensure
+      doc&.close
+    end
+
+    def maybe_decrypt_pdf_or_raise(data, params)
+      if data.size < ANNOTATIONS_SIZE_LIMIT && PdfUtils.encrypted?(data)
+        PdfUtils.decrypt(data, params[:password])
+      else
+        data
+      end
+    rescue Pdfium::PasswordError
+      raise PdfEncrypted
+    end
+
+    def decrypt_document(doc)
+      io = StringIO.new
+
+      doc.save(io, flags: Pdfium::FPDF_REMOVE_SECURITY)
+
+      io.tap(&:rewind).read
+    end
+
+    def create_document(template, file, document_data, metadata, annotations = nil)
       sha256 = Base64.urlsafe_encode64(Digest::SHA256.digest(document_data))
 
       blob = ActiveStorage::Blob.create_and_upload!(
@@ -62,19 +113,7 @@ module Templates
         content_type: file.content_type
       )
 
-      document = template.documents.create!(blob:)
-
-      Templates::ProcessDocument.call(document, document_data, extract_fields:)
-    end
-
-    def maybe_decrypt_pdf_or_raise(data, params)
-      if data.size < ANNOTATIONS_SIZE_LIMIT && PdfUtils.encrypted?(data)
-        PdfUtils.decrypt(data, params[:password])
-      else
-        data
-      end
-    rescue HexaPDF::EncryptionError
-      raise PdfEncrypted
+      template.documents.create!(blob:)
     end
 
     def extract_zip_files(files)
@@ -124,5 +163,9 @@ module Templates
 
       raise InvalidFileType, "#{file.content_type}/#{dynamic}"
     end
+
+    def v2?
+      true
+    end
   end
 end

lib/templates/process_document.rb

@@ -19,15 +19,19 @@ module Templates
 
     module_function
 
-    def call(attachment, data, extract_fields: false, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED)
+    def call(attachment, data, extract_fields: false, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED, doc: nil)
       if attachment.content_type == PDF_CONTENT_TYPE
         if extract_fields && data.size < MAX_FLATTEN_FILE_SIZE
+          if doc
+            fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data)
+          else
             pdf = HexaPDF::Document.new(io: StringIO.new(data))
 
             fields = Templates::FindAcroFields.call(pdf, attachment, data)
           end
+        end
 
-        generate_pdf_preview_images(attachment, data, pdf, max_pages:)
+        generate_pdf_preview_images(attachment, data, pdf, max_pages:, doc:)
 
         attachment.metadata['pdf']['fields'] = fields if fields
       elsif attachment.image?
@@ -76,13 +80,17 @@ module Templates
       )
     end
 
-    def generate_pdf_preview_images(attachment, data, pdf = nil, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED)
+    def generate_pdf_preview_images(attachment, data, pdf = nil, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED, doc: nil)
       ActiveStorage::Attachment.where(name: ATTACHMENT_NAME, record: attachment).destroy_all
 
+      if doc
+        number_of_pages = doc.page_count
+      else
         pdf ||= HexaPDF::Document.new(io: StringIO.new(data))
         number_of_pages = pdf.pages.size
 
         data = maybe_flatten_form(data, pdf)
+      end
 
       attachment.metadata['pdf'] ||= {}
       attachment.metadata['pdf']['number_of_pages'] = number_of_pages
@@ -93,17 +101,21 @@ module Templates
 
       max_pages_to_process = data.size < GENERATE_PREVIEW_SIZE_LIMIT ? max_pages : 1
 
-      generate_document_preview_images(attachment, data, 0..[number_of_pages - 1, max_pages_to_process].min)
+      generate_document_preview_images(attachment, data, 0..[number_of_pages - 1, max_pages_to_process].min, doc:)
     end
 
-    def generate_document_preview_images(attachment, data, range, concurrency: CONCURRENCY)
-      doc = Pdfium::Document.open_bytes(data)
+    def generate_document_preview_images(attachment, data, range, concurrency: CONCURRENCY, doc: nil)
+      flatten_pages = doc&.form?
+      pdfium_doc = doc || Pdfium::Document.open_bytes(data)
 
       pool = Concurrent::FixedThreadPool.new(concurrency)
 
       promises =
         range.map do |page_number|
-          doc_page = doc.get_page(page_number)
+          doc_page = pdfium_doc.get_page(page_number)
+
+          hide_placeholder_widgets(doc_page, hide_empty: !flatten_pages) if doc
+          doc_page.flatten if flatten_pages
 
           bytes, width, height = doc_page.render_to_bitmap(width: MAX_WIDTH)
 
@@ -126,10 +138,31 @@ module Templates
         end
       end
     ensure
-      doc&.close
+      pdfium_doc&.close if doc.nil?
       pool&.kill
     end
 
+    def hide_placeholder_widgets(page, hide_empty: true)
+      page.annotations.each do |annotation|
+        next unless annotation.widget?
+
+        page.with_annotation(annotation.index) do |handle|
+          next if handle.field_type != Pdfium::FPDF_FORMFIELD_COMBOBOX
+
+          value = handle.field_value.to_s
+
+          if value.blank?
+            next unless hide_empty
+          elsif handle.option_labels.blank? ||
+                !value.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP)
+            next
+          end
+
+          handle.hide!
+        end
+      end
+    end
+
     def build_and_upload_blob(image, page_number, format = FORMAT)
       image = image.copy(interpretation: :srgb)