diff --git a/app/controllers/templates_debug_controller.rb b/app/controllers/templates_debug_controller.rb index 333e8847..551d167f 100644 --- a/app/controllers/templates_debug_controller.rb +++ b/app/controllers/templates_debug_controller.rb @@ -13,9 +13,10 @@ class TemplatesDebugController < ApplicationController data = attachment.download unless attachment.image? - pdf = HexaPDF::Document.new(io: StringIO.new(data)) - - fields = Templates::FindAcroFields.call(pdf, attachment, data) + fields = + Pdfium::Document.open_io(StringIO.new(data)) do |doc| + Templates::FindPdfiumAcroFields.call(attachment, doc, data) + end end # fields, = Templates::DetectFields.call(StringIO.new(data), attachment:) if fields.blank? diff --git a/lib/pdf_utils.rb b/lib/pdf_utils.rb index 71595479..1e7c55a2 100644 --- a/lib/pdf_utils.rb +++ b/lib/pdf_utils.rb @@ -6,24 +6,6 @@ module PdfUtils module_function - def encrypted?(data, password: nil) - HexaPDF::Document.new(io: StringIO.new(data), decryption_opts: { password: }) - - false - rescue HexaPDF::EncryptionError - true - end - - def decrypt(data, password) - decrypted_io = StringIO.new - - Pdfium::Document.open_bytes(data, password) do |doc| - doc.save(decrypted_io, flags: Pdfium::FPDF_REMOVE_SECURITY) - end - - decrypted_io.tap(&:rewind).read - end - def merge(io_files) merged_content = StringIO.new diff --git a/lib/templates/build_annotations.rb b/lib/templates/build_annotations.rb deleted file mode 100644 index 8ce566d1..00000000 --- a/lib/templates/build_annotations.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Templates - module BuildAnnotations - module_function - - def call(data) - pdf = HexaPDF::Document.new(io: StringIO.new(data)) - - pdf.pages.flat_map.with_index do |page, index| - (page[:Annots] || []).filter_map do |annot| - next if annot.blank? - next if annot.is_a?(Integer) || annot.is_a?(Symbol) || annot.is_a?(HexaPDF::PDFArray) - next if annot[:A].blank? || annot[:A][:URI].blank? - next unless annot[:Subtype] == :Link - next if !annot[:A][:URI].starts_with?('https://') && !annot[:A][:URI].starts_with?('http://') - - build_external_link_hash(page, annot).merge('page' => index) - end - end - rescue StandardError => e - Rollbar.error(e) if defined?(Rollbar) - - [] - end - - def build_external_link_hash(page, annot) - left, bottom, right, top = annot[:Rect] - - { - 'type' => 'external_link', - 'value' => annot[:A][:URI], - 'x' => left / page.box.width.to_f, - 'y' => (page.box.height - top) / page.box.height.to_f, - 'w' => (right - left) / page.box.width.to_f, - 'h' => (top - bottom) / page.box.height.to_f - } - end - end -end diff --git a/lib/templates/create_attachments.rb b/lib/templates/create_attachments.rb index 8b3a555f..f5b47fd9 100644 --- a/lib/templates/create_attachments.rb +++ b/lib/templates/create_attachments.rb @@ -39,23 +39,6 @@ module Templates end def handle_pdf_or_image(template, file, document_data = nil, params = {}, extract_fields: false, metadata: {}) - return handle_pdf_or_image_v2(template, file, document_data, params, extract_fields:, metadata:) if v2? - - document_data ||= file.read - - if file.content_type == PDF_CONTENT_TYPE - document_data = maybe_decrypt_pdf_or_raise(document_data, params) - - annotations = - document_data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildAnnotations.call(document_data) : [] - end - - document = create_document(template, file, document_data, metadata, annotations) - - Templates::ProcessDocument.call(document, document_data, extract_fields:) - end - - def handle_pdf_or_image_v2(template, file, document_data = nil, params = {}, extract_fields: false, metadata: {}) document_data ||= file.read unless file.content_type == PDF_CONTENT_TYPE @@ -80,16 +63,6 @@ module Templates doc&.close end - def maybe_decrypt_pdf_or_raise(data, params) - if data.size < ANNOTATIONS_SIZE_LIMIT && PdfUtils.encrypted?(data) - PdfUtils.decrypt(data, params[:password]) - else - data - end - rescue Pdfium::PasswordError - raise PdfEncrypted - end - def decrypt_document(doc) io = StringIO.new @@ -163,9 +136,5 @@ module Templates raise InvalidFileType, "#{file.content_type}/#{dynamic}" end - - def v2? - true - end end end diff --git a/lib/templates/find_acro_fields.rb b/lib/templates/find_acro_fields.rb deleted file mode 100644 index 6da0bf72..00000000 --- a/lib/templates/find_acro_fields.rb +++ /dev/null @@ -1,288 +0,0 @@ -# frozen_string_literal: true - -module Templates - module FindAcroFields - PDF_CONTENT_TYPE = 'application/pdf' - - FIELD_NAME_REGEXP = /\A(?=.*\p{L})[\p{L}\d\s-]+\z/ - SKIP_FIELD_DESCRIPTION = %w[undefined].freeze - SELECT_PLACEHOLDER_REGEXP = /\b( - Select | - Choose | - Wählen | - Auswählen | - Sélectionner| - Choisir | - Seleccionar | - Elegir | - Seleziona | - Scegliere | - Selecionar | - Escolher - )\b/ix - - DATE_FORMAT_REGEXP = %r{[myd]{2,4}[-\\/\s.][myd]{2,4}[-\\/\s.][myd]{2,4}}i - - FIELD_ALIGNMENT = { - 0 => 'left', - 1 => 'center', - 2 => 'right' - }.freeze - - module_function - - # rubocop:disable Metrics - def call(pdf, attachment, data) - return [] if pdf.acro_form.blank? && data.exclude?('/Form') - - fields, annots_index = build_fields_with_pages(pdf) - - fields.filter_map do |field| - areas = Array.wrap(field[:Kids] || field).filter_map do |child_field| - page = annots_index[child_field.hash] - - next unless page - - media_box = page[:CropBox] || page[:MediaBox] - crop_box = page[:CropBox] || media_box - - media_box_start = [media_box[0], media_box[1]] - crop_shift = [crop_box[0] - media_box[0], crop_box[1] - media_box[1]] - - next unless child_field[:Rect] - - x0, y0, x1, y1 = child_field[:Rect] - - x0, y0 = correct_coordinates(x0, y0, crop_shift, media_box_start) - x1, y1 = correct_coordinates(x1, y1, crop_shift, media_box_start) - - page_width = media_box[2] - media_box[0] - page_height = media_box[3] - media_box[1] - - x = x0 - y = y0 - w = x1 - x0 - h = y1 - y0 - - transformed_y = page_height - y - h - - attrs = { - page: page.index, - x: x / page_width.to_f, - y: transformed_y / page_height.to_f, - w: w / page_width.to_f, - h: h / page_height.to_f, - attachment_uuid: attachment.uuid - } - - next if attrs[:w].zero? || attrs[:h].zero? - - if child_field[:MaxLen] && child_field.try(:concrete_field_type) == :comb_text_field - attrs[:cell_w] = w / page_width.to_f / child_field[:MaxLen].to_f - end - - attrs - end - - next if areas.blank? - - field_properties = build_field_properties(field) - - next if field_properties.blank? - next if field_properties[:default_value].present? - - if field_properties[:type].in?(%w[radio multiple]) - if areas.size != field_properties[:options].size - field_properties[:options] = build_options(Array.new(areas.size, '')) - end - - areas.each_with_index do |area, index| - area[:option_uuid] = field_properties[:options][index][:uuid] - end - end - - { - uuid: SecureRandom.uuid, - required: field.flags.include?(:required), - preferences: {}, - areas:, - **field_properties - } - end - rescue StandardError => e - raise if Rails.env.local? - - Rollbar.error(e) if defined?(Rollbar) - - [] - end - - def correct_coordinates(x_coord, y_coord, shift, media_box_start) - corrected_x = x_coord + shift[0] - media_box_start[0] - corrected_y = y_coord + shift[1] - media_box_start[1] - - [corrected_x, corrected_y] - end - - def build_field_properties(field) - field_name = field.full_field_name if field.full_field_name.to_s.match?(FIELD_NAME_REGEXP) - - field_name = field_name&.encode('utf-8', invalid: :replace, undef: :replace, replace: '') - - attrs = { name: field_name.to_s } - attrs[:description] = field[:TU] if field[:TU].present? && - field[:TU] != field.full_field_name && - !field[:TU].in?(SKIP_FIELD_DESCRIPTION) - - if field[:Q].present? && field.field_type == :Tx - attrs[:preferences] ||= {} - attrs[:preferences][:align] = FIELD_ALIGNMENT.fetch(field[:Q], 'left') - end - - if field.field_type == :Btn && field.concrete_field_type == :radio_button && field[:Opt].present? - selected_option_index = (field.allowed_values || []).find_index(field.field_value) - selected_option = field[:Opt][selected_option_index] if selected_option_index - - { - **attrs, - type: 'radio', - options: build_options(field[:Opt], 'radio'), - default_value: selected_option - } - elsif field.field_type == :Btn && %i[check_box radio_button].include?(field.concrete_field_type) && - field[:Kids].present? && field[:Kids].size > 1 && field.allowed_values.size > 1 - selected_option = (field.allowed_values || []).find { |v| v == field.field_value } - - return {} if field.allowed_values.include?(:BBox) - - { - **attrs, - type: 'radio', - options: build_options(field.allowed_values, 'radio'), - default_value: selected_option - } - elsif field.field_type == :Btn && %i[check_box radio_button].include?(field.concrete_field_type) - { - **attrs, - type: 'checkbox', - default_value: field.field_value.present? - } - elsif field.field_type == :Ch && - %i[combo_box editable_combo_box].include?(field.concrete_field_type) && field[:Opt].present? - { - **attrs, - type: 'select', - options: build_options(field[:Opt], 'select'), - default_value: field.field_value.to_s.match?(SELECT_PLACEHOLDER_REGEXP) ? nil : field.field_value.presence - } - elsif field.field_type == :Ch && field.concrete_field_type == :multi_select && field[:Opt].present? - { - **attrs, - type: 'multiple', - options: build_options(field[:Opt], 'multiple'), - default_value: field.field_value.presence - } - elsif field.field_type == :Tx && field.concrete_field_type == :comb_text_field - { - **attrs, - type: 'cells', - default_value: field.field_value.presence - } - elsif field.field_type == :Tx - if field[:AA] && ((field[:AA][:F] && field[:AA][:F][:JS].include?('AFDate_')) || - (field[:AA][:K] && field[:AA][:K][:JS].include?('AFDate_'))) - if (format = field[:AA][:F][:JS][DATE_FORMAT_REGEXP]) - attrs[:preferences] ||= {} - attrs[:preferences][:format] = format.upcase - end - - { - **attrs, - type: 'date', - default_value: field.field_value.presence - } - else - { - **attrs, - type: 'text', - default_value: field.field_value.presence - } - end - elsif field.field_type == :Sig - { - **attrs, - type: field.try(:field_name).to_s.downcase.include?('initials') ? 'initials' : 'signature' - } - else - {} - end.compact - end - - def build_options(values, type = nil) - is_skip_single_value = type.in?(%w[radio multiple]) && values.uniq.size == 1 - - values.filter_map do |option| - is_option_number = option.is_a?(Symbol) && option.to_s.match?(/\A\d+\z/) - - option = option[1] if option.is_a?(Array) && option.size == 2 - - if option.is_a?(String) || option.is_a?(Symbol) - option = option.to_s.encode('utf-8', invalid: :replace, undef: :replace, replace: '') - end - - next if type == 'select' && option.to_s.match?(SELECT_PLACEHOLDER_REGEXP) - - { - uuid: SecureRandom.uuid, - value: is_option_number || is_skip_single_value ? '' : option.presence - } - end - end - - def build_fields_with_pages(pdf) - fields_index = {} - annots_index = {} - - pdf.pages.each do |page| - page.each_annotation do |annot| - annots_index[annot.hash] = page - - if !annot.key?(:Parent) && annot.key?(:FT) - fields_index[annot.hash] ||= HexaPDF::Type::AcroForm::Field.wrap(pdf, annot) - elsif annot.key?(:Parent) - field = annot[:Parent] - seen = Set.new.compare_by_identity - - field = field[:Parent] while field[:Parent] && seen.add?(field.value) - - annots_index[field.hash] ||= page - fields_index[field.hash] ||= HexaPDF::Type::AcroForm::Field.wrap(pdf, field) - end - end - end - - [process_fields_array(pdf, fields_index.values), annots_index] - end - - def process_fields_array(pdf, array, acc = [], seen = Set.new.compare_by_identity) - array.each_with_index do |field, index| - next if field.nil? - - unless field.respond_to?(:type) && field.type == :XXAcroFormField - array[index] = field = HexaPDF::Type::AcroForm::Field.wrap(pdf, field) - end - - next unless seen.add?(field.value) - - if field.terminal_field? - acc << field - else - process_fields_array(pdf, field[:Kids], acc, seen) - end - end - - acc - end - # rubocop:enable Metrics - end -end diff --git a/lib/templates/find_pdfium_acro_fields.rb b/lib/templates/find_pdfium_acro_fields.rb index d9133aaa..805ed75c 100644 --- a/lib/templates/find_pdfium_acro_fields.rb +++ b/lib/templates/find_pdfium_acro_fields.rb @@ -6,6 +6,31 @@ module Templates SKIP_FIELD_TYPES = %i[unknown pushbutton].freeze TEXT_OPERATOR_REGEXP = /\bT[jJ]\b/ + FIELD_NAME_REGEXP = /\A(?=.*\p{L})[\p{L}\d\s-]+\z/ + SKIP_FIELD_DESCRIPTION = %w[undefined].freeze + SELECT_PLACEHOLDER_REGEXP = /\b( + Select | + Choose | + Wählen | + Auswählen | + Sélectionner| + Choisir | + Seleccionar | + Elegir | + Seleziona | + Scegliere | + Selecionar | + Escolher + )\b/ix + + DATE_FORMAT_REGEXP = %r{[myd]{2,4}[-\\/\s.][myd]{2,4}[-\\/\s.][myd]{2,4}}i + + FIELD_ALIGNMENT = { + 0 => 'left', + 1 => 'center', + 2 => 'right' + }.freeze + module_function def call(attachment, doc, data) @@ -108,12 +133,12 @@ module Templates def build_field_properties(widgets) field = widgets.first.field - field_name = field.name if field.name.match?(FindAcroFields::FIELD_NAME_REGEXP) + field_name = field.name if field.name.match?(FIELD_NAME_REGEXP) attrs = { name: field_name.to_s } attrs[:description] = field.alternate_name if field.alternate_name.present? && field.alternate_name != field.name && - !field.alternate_name.in?(FindAcroFields::SKIP_FIELD_DESCRIPTION) + !field.alternate_name.in?(SKIP_FIELD_DESCRIPTION) case field.type when :checkbox, :radio @@ -172,7 +197,7 @@ module Templates **attrs, type: 'select', options: build_options(field.options, 'select'), - default_value: value.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP) ? nil : value + default_value: value.to_s.match?(SELECT_PLACEHOLDER_REGEXP) ? nil : value } end @@ -183,7 +208,7 @@ module Templates end def build_text_properties(attrs, field) - preferences = { align: FindAcroFields::FIELD_ALIGNMENT.fetch(field.quadding.to_i, 'left') } + preferences = { align: FIELD_ALIGNMENT.fetch(field.quadding.to_i, 'left') } attrs = { **attrs, preferences: } @@ -191,7 +216,7 @@ module Templates { **attrs, type: 'cells', default_value: field.value.presence } elsif date?(field) format = [field.format_js, field.keystroke_js].compact - .filter_map { |js| js[FindAcroFields::DATE_FORMAT_REGEXP] } + .filter_map { |js| js[DATE_FORMAT_REGEXP] } .first preferences[:format] = format.upcase if format @@ -218,7 +243,7 @@ module Templates option = option.to_s.encode('utf-8', invalid: :replace, undef: :replace, replace: '') end - next if type == 'select' && option.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP) + next if type == 'select' && option.to_s.match?(SELECT_PLACEHOLDER_REGEXP) { uuid: SecureRandom.uuid, diff --git a/lib/templates/modify_documents.rb b/lib/templates/modify_documents.rb index 8136520e..5be3b07b 100644 --- a/lib/templates/modify_documents.rb +++ b/lib/templates/modify_documents.rb @@ -435,7 +435,9 @@ module Templates end def save_document(template, old_attachment, data) - annotations = data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildAnnotations.call(data) : [] + doc = Pdfium::Document.open_io(StringIO.new(data)) + + annotations = data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildPdfiumAnnotations.call(doc) : [] sha256 = Base64.urlsafe_encode64(Digest::SHA256.digest(data)) blob = ActiveStorage::Blob.create_and_upload!( @@ -448,7 +450,9 @@ module Templates document = template.documents.create!(blob:) - Templates::ProcessDocument.call(document, data) + Templates::ProcessDocument.call(document, data, doc:) + ensure + doc&.close end def remap_fields(template, mapping) diff --git a/lib/templates/process_document.rb b/lib/templates/process_document.rb index 2d60665e..e8f138f3 100644 --- a/lib/templates/process_document.rb +++ b/lib/templates/process_document.rb @@ -22,16 +22,10 @@ module Templates def call(attachment, data, extract_fields: false, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED, doc: nil) if attachment.content_type == PDF_CONTENT_TYPE if extract_fields && data.size < MAX_FLATTEN_FILE_SIZE - if doc - fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data) - else - pdf = HexaPDF::Document.new(io: StringIO.new(data)) - - fields = Templates::FindAcroFields.call(pdf, attachment, data) - end + fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data) end - generate_pdf_preview_images(attachment, data, pdf, max_pages:, doc:) + generate_pdf_preview_images(attachment, data, max_pages:, doc:) attachment.metadata['pdf']['fields'] = fields if fields elsif attachment.image? @@ -41,27 +35,13 @@ module Templates attachment end - def process(attachment, data, extract_fields: false, doc: nil) + def process(attachment, data, doc:, extract_fields: false) if attachment.content_type == PDF_CONTENT_TYPE && extract_fields && data.size < MAX_FLATTEN_FILE_SIZE - if doc - fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data) - else - pdf = HexaPDF::Document.new(io: StringIO.new(data)) - - fields = Templates::FindAcroFields.call(pdf, attachment, data) - end - end - - if doc - number_of_pages = doc.page_count - else - pdf ||= HexaPDF::Document.new(io: StringIO.new(data)) - - number_of_pages = pdf.pages.size + fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data) end attachment.metadata['pdf'] ||= {} - attachment.metadata['pdf']['number_of_pages'] = number_of_pages + attachment.metadata['pdf']['number_of_pages'] = doc.page_count attachment.metadata['pdf']['fields'] = fields if fields attachment @@ -88,17 +68,10 @@ module Templates ) end - def generate_pdf_preview_images(attachment, data, pdf = nil, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED, doc: nil) + def generate_pdf_preview_images(attachment, data, doc:, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED) ActiveStorage::Attachment.where(name: ATTACHMENT_NAME, record: attachment).destroy_all - if doc - number_of_pages = doc.page_count - else - pdf ||= HexaPDF::Document.new(io: StringIO.new(data)) - number_of_pages = pdf.pages.size - - data = maybe_flatten_form(data, pdf) - end + number_of_pages = doc.page_count attachment.metadata['pdf'] ||= {} attachment.metadata['pdf']['number_of_pages'] = number_of_pages @@ -109,20 +82,19 @@ module Templates max_pages_to_process = data.size < GENERATE_PREVIEW_SIZE_LIMIT ? max_pages : 1 - generate_document_preview_images(attachment, data, 0..[number_of_pages - 1, max_pages_to_process].min, doc:) + generate_document_preview_images(attachment, 0..[number_of_pages - 1, max_pages_to_process].min, doc:) end - def generate_document_preview_images(attachment, data, range, concurrency: CONCURRENCY, doc: nil) - flatten_pages = doc&.form? - pdfium_doc = doc || Pdfium::Document.open_bytes(data) + def generate_document_preview_images(attachment, range, doc:, concurrency: CONCURRENCY) + flatten_pages = doc.form? pool = Concurrent::FixedThreadPool.new(concurrency) promises = range.map do |page_number| - doc_page = pdfium_doc.get_page(page_number) + doc_page = doc.get_page(page_number) - hide_placeholder_widgets(doc_page, hide_empty: !flatten_pages) if doc + hide_placeholder_widgets(doc_page, hide_empty: !flatten_pages) doc_page.flatten if flatten_pages bytes, width, height = doc_page.render_to_bitmap(width: MAX_WIDTH) @@ -146,7 +118,6 @@ module Templates end end ensure - pdfium_doc&.close if doc.nil? pool&.kill end @@ -162,7 +133,7 @@ module Templates if value.blank? next unless hide_empty elsif handle.option_labels.blank? || - !value.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP) + !value.match?(FindPdfiumAcroFields::SELECT_PLACEHOLDER_REGEXP) next end @@ -194,33 +165,6 @@ module Templates blob end - def maybe_flatten_form(data, pdf) - return data if data.size > MAX_FLATTEN_FILE_SIZE - return data if pdf.acro_form.blank? - - io = StringIO.new - - pdf.acro_form.each_field do |field| - next if field.field_type != :Ch || - field[:Opt].blank? || - %i[combo_box editable_combo_box].exclude?(field.concrete_field_type) || - !field.field_value.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP) - - field[:V] = '' - end - - pdf.acro_form.create_appearances(force: true) if pdf.acro_form[:NeedAppearances] - pdf.acro_form.flatten - - pdf.write(io, incremental: false, validate: false) - - io.string - rescue StandardError - raise if Rails.env.development? - - data - end - def normalize_attachment_fields(template, attachments = template.documents) attachments.flat_map do |a| pdf_fields = a.metadata['pdf'].delete('fields').to_a if a.metadata['pdf'].present? diff --git a/spec/factories/templates.rb b/spec/factories/templates.rb index 33988385..09d5a3fd 100644 --- a/spec/factories/templates.rb +++ b/spec/factories/templates.rb @@ -42,7 +42,11 @@ FactoryBot.define do record: template ) - Templates::ProcessDocument.call(attachment, attachment.download) + data = attachment.download + + Pdfium::Document.open_io(StringIO.new(data)) do |doc| + Templates::ProcessDocument.call(attachment, data, doc:) + end template.schema << { attachment_uuid: attachment.uuid,