process pdf v2

pull/721/head
Pete Matsyburka 2 weeks ago
parent 785eb789ad
commit f9669b0e16

@ -13,9 +13,10 @@ class TemplatesDebugController < ApplicationController
data = attachment.download
unless attachment.image?
pdf = HexaPDF::Document.new(io: StringIO.new(data))
fields = Templates::FindAcroFields.call(pdf, attachment, data)
fields =
Pdfium::Document.open_io(StringIO.new(data)) do |doc|
Templates::FindPdfiumAcroFields.call(attachment, doc, data)
end
end
# fields, = Templates::DetectFields.call(StringIO.new(data), attachment:) if fields.blank?

@ -6,24 +6,6 @@ module PdfUtils
module_function
def encrypted?(data, password: nil)
HexaPDF::Document.new(io: StringIO.new(data), decryption_opts: { password: })
false
rescue HexaPDF::EncryptionError
true
end
def decrypt(data, password)
decrypted_io = StringIO.new
Pdfium::Document.open_bytes(data, password) do |doc|
doc.save(decrypted_io, flags: Pdfium::FPDF_REMOVE_SECURITY)
end
decrypted_io.tap(&:rewind).read
end
def merge(io_files)
merged_content = StringIO.new

@ -1,40 +0,0 @@
# frozen_string_literal: true
module Templates
module BuildAnnotations
module_function
def call(data)
pdf = HexaPDF::Document.new(io: StringIO.new(data))
pdf.pages.flat_map.with_index do |page, index|
(page[:Annots] || []).filter_map do |annot|
next if annot.blank?
next if annot.is_a?(Integer) || annot.is_a?(Symbol) || annot.is_a?(HexaPDF::PDFArray)
next if annot[:A].blank? || annot[:A][:URI].blank?
next unless annot[:Subtype] == :Link
next if !annot[:A][:URI].starts_with?('https://') && !annot[:A][:URI].starts_with?('http://')
build_external_link_hash(page, annot).merge('page' => index)
end
end
rescue StandardError => e
Rollbar.error(e) if defined?(Rollbar)
[]
end
def build_external_link_hash(page, annot)
left, bottom, right, top = annot[:Rect]
{
'type' => 'external_link',
'value' => annot[:A][:URI],
'x' => left / page.box.width.to_f,
'y' => (page.box.height - top) / page.box.height.to_f,
'w' => (right - left) / page.box.width.to_f,
'h' => (top - bottom) / page.box.height.to_f
}
end
end
end

@ -39,23 +39,6 @@ module Templates
end
def handle_pdf_or_image(template, file, document_data = nil, params = {}, extract_fields: false, metadata: {})
return handle_pdf_or_image_v2(template, file, document_data, params, extract_fields:, metadata:) if v2?
document_data ||= file.read
if file.content_type == PDF_CONTENT_TYPE
document_data = maybe_decrypt_pdf_or_raise(document_data, params)
annotations =
document_data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildAnnotations.call(document_data) : []
end
document = create_document(template, file, document_data, metadata, annotations)
Templates::ProcessDocument.call(document, document_data, extract_fields:)
end
def handle_pdf_or_image_v2(template, file, document_data = nil, params = {}, extract_fields: false, metadata: {})
document_data ||= file.read
unless file.content_type == PDF_CONTENT_TYPE
@ -80,16 +63,6 @@ module Templates
doc&.close
end
def maybe_decrypt_pdf_or_raise(data, params)
if data.size < ANNOTATIONS_SIZE_LIMIT && PdfUtils.encrypted?(data)
PdfUtils.decrypt(data, params[:password])
else
data
end
rescue Pdfium::PasswordError
raise PdfEncrypted
end
def decrypt_document(doc)
io = StringIO.new
@ -163,9 +136,5 @@ module Templates
raise InvalidFileType, "#{file.content_type}/#{dynamic}"
end
def v2?
true
end
end
end

@ -1,288 +0,0 @@
# frozen_string_literal: true
module Templates
module FindAcroFields
PDF_CONTENT_TYPE = 'application/pdf'
FIELD_NAME_REGEXP = /\A(?=.*\p{L})[\p{L}\d\s-]+\z/
SKIP_FIELD_DESCRIPTION = %w[undefined].freeze
SELECT_PLACEHOLDER_REGEXP = /\b(
Select |
Choose |
Wählen |
Auswählen |
Sélectionner|
Choisir |
Seleccionar |
Elegir |
Seleziona |
Scegliere |
Selecionar |
Escolher
)\b/ix
DATE_FORMAT_REGEXP = %r{[myd]{2,4}[-\\/\s.][myd]{2,4}[-\\/\s.][myd]{2,4}}i
FIELD_ALIGNMENT = {
0 => 'left',
1 => 'center',
2 => 'right'
}.freeze
module_function
# rubocop:disable Metrics
def call(pdf, attachment, data)
return [] if pdf.acro_form.blank? && data.exclude?('/Form')
fields, annots_index = build_fields_with_pages(pdf)
fields.filter_map do |field|
areas = Array.wrap(field[:Kids] || field).filter_map do |child_field|
page = annots_index[child_field.hash]
next unless page
media_box = page[:CropBox] || page[:MediaBox]
crop_box = page[:CropBox] || media_box
media_box_start = [media_box[0], media_box[1]]
crop_shift = [crop_box[0] - media_box[0], crop_box[1] - media_box[1]]
next unless child_field[:Rect]
x0, y0, x1, y1 = child_field[:Rect]
x0, y0 = correct_coordinates(x0, y0, crop_shift, media_box_start)
x1, y1 = correct_coordinates(x1, y1, crop_shift, media_box_start)
page_width = media_box[2] - media_box[0]
page_height = media_box[3] - media_box[1]
x = x0
y = y0
w = x1 - x0
h = y1 - y0
transformed_y = page_height - y - h
attrs = {
page: page.index,
x: x / page_width.to_f,
y: transformed_y / page_height.to_f,
w: w / page_width.to_f,
h: h / page_height.to_f,
attachment_uuid: attachment.uuid
}
next if attrs[:w].zero? || attrs[:h].zero?
if child_field[:MaxLen] && child_field.try(:concrete_field_type) == :comb_text_field
attrs[:cell_w] = w / page_width.to_f / child_field[:MaxLen].to_f
end
attrs
end
next if areas.blank?
field_properties = build_field_properties(field)
next if field_properties.blank?
next if field_properties[:default_value].present?
if field_properties[:type].in?(%w[radio multiple])
if areas.size != field_properties[:options].size
field_properties[:options] = build_options(Array.new(areas.size, ''))
end
areas.each_with_index do |area, index|
area[:option_uuid] = field_properties[:options][index][:uuid]
end
end
{
uuid: SecureRandom.uuid,
required: field.flags.include?(:required),
preferences: {},
areas:,
**field_properties
}
end
rescue StandardError => e
raise if Rails.env.local?
Rollbar.error(e) if defined?(Rollbar)
[]
end
def correct_coordinates(x_coord, y_coord, shift, media_box_start)
corrected_x = x_coord + shift[0] - media_box_start[0]
corrected_y = y_coord + shift[1] - media_box_start[1]
[corrected_x, corrected_y]
end
def build_field_properties(field)
field_name = field.full_field_name if field.full_field_name.to_s.match?(FIELD_NAME_REGEXP)
field_name = field_name&.encode('utf-8', invalid: :replace, undef: :replace, replace: '')
attrs = { name: field_name.to_s }
attrs[:description] = field[:TU] if field[:TU].present? &&
field[:TU] != field.full_field_name &&
!field[:TU].in?(SKIP_FIELD_DESCRIPTION)
if field[:Q].present? && field.field_type == :Tx
attrs[:preferences] ||= {}
attrs[:preferences][:align] = FIELD_ALIGNMENT.fetch(field[:Q], 'left')
end
if field.field_type == :Btn && field.concrete_field_type == :radio_button && field[:Opt].present?
selected_option_index = (field.allowed_values || []).find_index(field.field_value)
selected_option = field[:Opt][selected_option_index] if selected_option_index
{
**attrs,
type: 'radio',
options: build_options(field[:Opt], 'radio'),
default_value: selected_option
}
elsif field.field_type == :Btn && %i[check_box radio_button].include?(field.concrete_field_type) &&
field[:Kids].present? && field[:Kids].size > 1 && field.allowed_values.size > 1
selected_option = (field.allowed_values || []).find { |v| v == field.field_value }
return {} if field.allowed_values.include?(:BBox)
{
**attrs,
type: 'radio',
options: build_options(field.allowed_values, 'radio'),
default_value: selected_option
}
elsif field.field_type == :Btn && %i[check_box radio_button].include?(field.concrete_field_type)
{
**attrs,
type: 'checkbox',
default_value: field.field_value.present?
}
elsif field.field_type == :Ch &&
%i[combo_box editable_combo_box].include?(field.concrete_field_type) && field[:Opt].present?
{
**attrs,
type: 'select',
options: build_options(field[:Opt], 'select'),
default_value: field.field_value.to_s.match?(SELECT_PLACEHOLDER_REGEXP) ? nil : field.field_value.presence
}
elsif field.field_type == :Ch && field.concrete_field_type == :multi_select && field[:Opt].present?
{
**attrs,
type: 'multiple',
options: build_options(field[:Opt], 'multiple'),
default_value: field.field_value.presence
}
elsif field.field_type == :Tx && field.concrete_field_type == :comb_text_field
{
**attrs,
type: 'cells',
default_value: field.field_value.presence
}
elsif field.field_type == :Tx
if field[:AA] && ((field[:AA][:F] && field[:AA][:F][:JS].include?('AFDate_')) ||
(field[:AA][:K] && field[:AA][:K][:JS].include?('AFDate_')))
if (format = field[:AA][:F][:JS][DATE_FORMAT_REGEXP])
attrs[:preferences] ||= {}
attrs[:preferences][:format] = format.upcase
end
{
**attrs,
type: 'date',
default_value: field.field_value.presence
}
else
{
**attrs,
type: 'text',
default_value: field.field_value.presence
}
end
elsif field.field_type == :Sig
{
**attrs,
type: field.try(:field_name).to_s.downcase.include?('initials') ? 'initials' : 'signature'
}
else
{}
end.compact
end
def build_options(values, type = nil)
is_skip_single_value = type.in?(%w[radio multiple]) && values.uniq.size == 1
values.filter_map do |option|
is_option_number = option.is_a?(Symbol) && option.to_s.match?(/\A\d+\z/)
option = option[1] if option.is_a?(Array) && option.size == 2
if option.is_a?(String) || option.is_a?(Symbol)
option = option.to_s.encode('utf-8', invalid: :replace, undef: :replace, replace: '')
end
next if type == 'select' && option.to_s.match?(SELECT_PLACEHOLDER_REGEXP)
{
uuid: SecureRandom.uuid,
value: is_option_number || is_skip_single_value ? '' : option.presence
}
end
end
def build_fields_with_pages(pdf)
fields_index = {}
annots_index = {}
pdf.pages.each do |page|
page.each_annotation do |annot|
annots_index[annot.hash] = page
if !annot.key?(:Parent) && annot.key?(:FT)
fields_index[annot.hash] ||= HexaPDF::Type::AcroForm::Field.wrap(pdf, annot)
elsif annot.key?(:Parent)
field = annot[:Parent]
seen = Set.new.compare_by_identity
field = field[:Parent] while field[:Parent] && seen.add?(field.value)
annots_index[field.hash] ||= page
fields_index[field.hash] ||= HexaPDF::Type::AcroForm::Field.wrap(pdf, field)
end
end
end
[process_fields_array(pdf, fields_index.values), annots_index]
end
def process_fields_array(pdf, array, acc = [], seen = Set.new.compare_by_identity)
array.each_with_index do |field, index|
next if field.nil?
unless field.respond_to?(:type) && field.type == :XXAcroFormField
array[index] = field = HexaPDF::Type::AcroForm::Field.wrap(pdf, field)
end
next unless seen.add?(field.value)
if field.terminal_field?
acc << field
else
process_fields_array(pdf, field[:Kids], acc, seen)
end
end
acc
end
# rubocop:enable Metrics
end
end

@ -6,6 +6,31 @@ module Templates
SKIP_FIELD_TYPES = %i[unknown pushbutton].freeze
TEXT_OPERATOR_REGEXP = /\bT[jJ]\b/
FIELD_NAME_REGEXP = /\A(?=.*\p{L})[\p{L}\d\s-]+\z/
SKIP_FIELD_DESCRIPTION = %w[undefined].freeze
SELECT_PLACEHOLDER_REGEXP = /\b(
Select |
Choose |
Wählen |
Auswählen |
Sélectionner|
Choisir |
Seleccionar |
Elegir |
Seleziona |
Scegliere |
Selecionar |
Escolher
)\b/ix
DATE_FORMAT_REGEXP = %r{[myd]{2,4}[-\\/\s.][myd]{2,4}[-\\/\s.][myd]{2,4}}i
FIELD_ALIGNMENT = {
0 => 'left',
1 => 'center',
2 => 'right'
}.freeze
module_function
def call(attachment, doc, data)
@ -108,12 +133,12 @@ module Templates
def build_field_properties(widgets)
field = widgets.first.field
field_name = field.name if field.name.match?(FindAcroFields::FIELD_NAME_REGEXP)
field_name = field.name if field.name.match?(FIELD_NAME_REGEXP)
attrs = { name: field_name.to_s }
attrs[:description] = field.alternate_name if field.alternate_name.present? &&
field.alternate_name != field.name &&
!field.alternate_name.in?(FindAcroFields::SKIP_FIELD_DESCRIPTION)
!field.alternate_name.in?(SKIP_FIELD_DESCRIPTION)
case field.type
when :checkbox, :radio
@ -172,7 +197,7 @@ module Templates
**attrs,
type: 'select',
options: build_options(field.options, 'select'),
default_value: value.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP) ? nil : value
default_value: value.to_s.match?(SELECT_PLACEHOLDER_REGEXP) ? nil : value
}
end
@ -183,7 +208,7 @@ module Templates
end
def build_text_properties(attrs, field)
preferences = { align: FindAcroFields::FIELD_ALIGNMENT.fetch(field.quadding.to_i, 'left') }
preferences = { align: FIELD_ALIGNMENT.fetch(field.quadding.to_i, 'left') }
attrs = { **attrs, preferences: }
@ -191,7 +216,7 @@ module Templates
{ **attrs, type: 'cells', default_value: field.value.presence }
elsif date?(field)
format = [field.format_js, field.keystroke_js].compact
.filter_map { |js| js[FindAcroFields::DATE_FORMAT_REGEXP] }
.filter_map { |js| js[DATE_FORMAT_REGEXP] }
.first
preferences[:format] = format.upcase if format
@ -218,7 +243,7 @@ module Templates
option = option.to_s.encode('utf-8', invalid: :replace, undef: :replace, replace: '')
end
next if type == 'select' && option.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP)
next if type == 'select' && option.to_s.match?(SELECT_PLACEHOLDER_REGEXP)
{
uuid: SecureRandom.uuid,

@ -435,7 +435,9 @@ module Templates
end
def save_document(template, old_attachment, data)
annotations = data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildAnnotations.call(data) : []
doc = Pdfium::Document.open_io(StringIO.new(data))
annotations = data.size < ANNOTATIONS_SIZE_LIMIT ? Templates::BuildPdfiumAnnotations.call(doc) : []
sha256 = Base64.urlsafe_encode64(Digest::SHA256.digest(data))
blob = ActiveStorage::Blob.create_and_upload!(
@ -448,7 +450,9 @@ module Templates
document = template.documents.create!(blob:)
Templates::ProcessDocument.call(document, data)
Templates::ProcessDocument.call(document, data, doc:)
ensure
doc&.close
end
def remap_fields(template, mapping)

@ -22,16 +22,10 @@ module Templates
def call(attachment, data, extract_fields: false, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED, doc: nil)
if attachment.content_type == PDF_CONTENT_TYPE
if extract_fields && data.size < MAX_FLATTEN_FILE_SIZE
if doc
fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data)
else
pdf = HexaPDF::Document.new(io: StringIO.new(data))
fields = Templates::FindAcroFields.call(pdf, attachment, data)
end
fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data)
end
generate_pdf_preview_images(attachment, data, pdf, max_pages:, doc:)
generate_pdf_preview_images(attachment, data, max_pages:, doc:)
attachment.metadata['pdf']['fields'] = fields if fields
elsif attachment.image?
@ -41,27 +35,13 @@ module Templates
attachment
end
def process(attachment, data, extract_fields: false, doc: nil)
def process(attachment, data, doc:, extract_fields: false)
if attachment.content_type == PDF_CONTENT_TYPE && extract_fields && data.size < MAX_FLATTEN_FILE_SIZE
if doc
fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data)
else
pdf = HexaPDF::Document.new(io: StringIO.new(data))
fields = Templates::FindAcroFields.call(pdf, attachment, data)
end
end
if doc
number_of_pages = doc.page_count
else
pdf ||= HexaPDF::Document.new(io: StringIO.new(data))
number_of_pages = pdf.pages.size
fields = Templates::FindPdfiumAcroFields.call(attachment, doc, data)
end
attachment.metadata['pdf'] ||= {}
attachment.metadata['pdf']['number_of_pages'] = number_of_pages
attachment.metadata['pdf']['number_of_pages'] = doc.page_count
attachment.metadata['pdf']['fields'] = fields if fields
attachment
@ -88,17 +68,10 @@ module Templates
)
end
def generate_pdf_preview_images(attachment, data, pdf = nil, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED, doc: nil)
def generate_pdf_preview_images(attachment, data, doc:, max_pages: MAX_NUMBER_OF_PAGES_PROCESSED)
ActiveStorage::Attachment.where(name: ATTACHMENT_NAME, record: attachment).destroy_all
if doc
number_of_pages = doc.page_count
else
pdf ||= HexaPDF::Document.new(io: StringIO.new(data))
number_of_pages = pdf.pages.size
data = maybe_flatten_form(data, pdf)
end
number_of_pages = doc.page_count
attachment.metadata['pdf'] ||= {}
attachment.metadata['pdf']['number_of_pages'] = number_of_pages
@ -109,20 +82,19 @@ module Templates
max_pages_to_process = data.size < GENERATE_PREVIEW_SIZE_LIMIT ? max_pages : 1
generate_document_preview_images(attachment, data, 0..[number_of_pages - 1, max_pages_to_process].min, doc:)
generate_document_preview_images(attachment, 0..[number_of_pages - 1, max_pages_to_process].min, doc:)
end
def generate_document_preview_images(attachment, data, range, concurrency: CONCURRENCY, doc: nil)
flatten_pages = doc&.form?
pdfium_doc = doc || Pdfium::Document.open_bytes(data)
def generate_document_preview_images(attachment, range, doc:, concurrency: CONCURRENCY)
flatten_pages = doc.form?
pool = Concurrent::FixedThreadPool.new(concurrency)
promises =
range.map do |page_number|
doc_page = pdfium_doc.get_page(page_number)
doc_page = doc.get_page(page_number)
hide_placeholder_widgets(doc_page, hide_empty: !flatten_pages) if doc
hide_placeholder_widgets(doc_page, hide_empty: !flatten_pages)
doc_page.flatten if flatten_pages
bytes, width, height = doc_page.render_to_bitmap(width: MAX_WIDTH)
@ -146,7 +118,6 @@ module Templates
end
end
ensure
pdfium_doc&.close if doc.nil?
pool&.kill
end
@ -162,7 +133,7 @@ module Templates
if value.blank?
next unless hide_empty
elsif handle.option_labels.blank? ||
!value.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP)
!value.match?(FindPdfiumAcroFields::SELECT_PLACEHOLDER_REGEXP)
next
end
@ -194,33 +165,6 @@ module Templates
blob
end
def maybe_flatten_form(data, pdf)
return data if data.size > MAX_FLATTEN_FILE_SIZE
return data if pdf.acro_form.blank?
io = StringIO.new
pdf.acro_form.each_field do |field|
next if field.field_type != :Ch ||
field[:Opt].blank? ||
%i[combo_box editable_combo_box].exclude?(field.concrete_field_type) ||
!field.field_value.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP)
field[:V] = ''
end
pdf.acro_form.create_appearances(force: true) if pdf.acro_form[:NeedAppearances]
pdf.acro_form.flatten
pdf.write(io, incremental: false, validate: false)
io.string
rescue StandardError
raise if Rails.env.development?
data
end
def normalize_attachment_fields(template, attachments = template.documents)
attachments.flat_map do |a|
pdf_fields = a.metadata['pdf'].delete('fields').to_a if a.metadata['pdf'].present?

@ -42,7 +42,11 @@ FactoryBot.define do
record: template
)
Templates::ProcessDocument.call(attachment, attachment.download)
data = attachment.download
Pdfium::Document.open_io(StringIO.new(data)) do |doc|
Templates::ProcessDocument.call(attachment, data, doc:)
end
template.schema << {
attachment_uuid: attachment.uuid,

Loading…
Cancel
Save