{
+ if (this.timeLeftSeconds === null) return
+
+ this.timeLeftSeconds -= 1
+
+ if (this.timeLeftSeconds <= 0) {
+ this.handleTimeout()
+ }
+ }, 1000)
+ },
+ handleTimeout () {
+ this.clearCountdown()
+
+ this.questions = null
+ this.token = null
+ this.reference = null
+ this.answers = {}
+ this.currentQuestionIndex = 0
+ this.error = `Knowledge Based Authentication timed out. You only have ${this.kbaTimeLimitSeconds} seconds to complete the verification. Please retry.`
+ },
nextQuestion () {
if (this.isLastQuestion) {
this.$emit('submit')
@@ -405,6 +462,8 @@ export default {
}
},
restartKba () {
+ this.clearCountdown()
+
this.questions = null
this.token = null
this.reference = null
@@ -415,6 +474,7 @@ export default {
async startKba () {
this.isLoading = true
this.error = null
+ this.clearCountdown()
try {
const payload = { ...this.form, submitter_slug: this.submitterSlug }
@@ -457,6 +517,8 @@ export default {
this.questions.forEach(q => {
this.answers[q.id] = null
})
+
+ this.startCountdown()
} else {
throw new Error('Invalid KBA response')
}
@@ -467,6 +529,8 @@ export default {
}
},
async submit () {
+ this.clearCountdown()
+
this.isSubmitting = true
this.error = null
diff --git a/app/javascript/template_builder/builder.vue b/app/javascript/template_builder/builder.vue
index 8a09371b..13af471a 100644
--- a/app/javascript/template_builder/builder.vue
+++ b/app/javascript/template_builder/builder.vue
@@ -1116,14 +1116,17 @@ export default {
const sortArea = (aArea, bArea) => {
if (aArea.attachment_uuid === bArea.attachment_uuid) {
if (aArea.page === bArea.page) {
- if (Math.abs(aArea.y - bArea.y) < 0.01) {
+ const aY = aArea.y + aArea.h
+ const bY = bArea.y + bArea.h
+
+ if (Math.abs(aY - bY) < 0.01 || (aArea.h < bArea.h ? (aArea.y >= bArea.y && aY <= bY) : (bArea.y >= aArea.y && bY <= aY))) {
if (aArea.x === bArea.x) {
return 0
} else {
return aArea.x - bArea.x
}
} else {
- return aArea.y - bArea.y
+ return (aArea.y + aArea.h) - (bArea.y + bArea.h)
}
} else {
return aArea.page - bArea.page
@@ -1173,6 +1176,72 @@ export default {
this.save()
}
},
+ findFieldInsertIndex (field) {
+ if (!field.areas?.length) return -1
+
+ const area = field.areas[0]
+
+ const attachmentUuidsIndex = this.template.schema.reduce((acc, e, index) => {
+ acc[e.attachment_uuid] = index
+
+ return acc
+ }, {})
+
+ const compareAreas = (a, b) => {
+ const aAttIdx = attachmentUuidsIndex[a.attachment_uuid]
+ const bAttIdx = attachmentUuidsIndex[b.attachment_uuid]
+
+ if (aAttIdx !== bAttIdx) return aAttIdx - bAttIdx
+ if (a.page !== b.page) return a.page - b.page
+
+ const aY = a.y + a.h
+ const bY = b.y + b.h
+
+ if (Math.abs(aY - bY) < 0.01) return a.x - b.x
+ if (a.h < b.h ? a.y >= b.y && aY <= bY : b.y >= a.y && bY <= aY) return a.x - b.x
+
+ return aY - bY
+ }
+
+ let closestBeforeIndex = -1
+ let closestBeforeArea = null
+ let closestAfterIndex = -1
+ let closestAfterArea = null
+
+ this.template.fields.forEach((f, index) => {
+ if (f.submitter_uuid === field.submitter_uuid) {
+ (f.areas || []).forEach((a) => {
+ const cmp = compareAreas(a, area)
+
+ if (cmp < 0) {
+ if (!closestBeforeArea || (compareAreas(a, closestBeforeArea) > 0 && closestBeforeIndex < index)) {
+ closestBeforeIndex = index
+ closestBeforeArea = a
+ }
+ } else {
+ if (!closestAfterArea || (compareAreas(a, closestAfterArea) < 0 && closestAfterIndex > index)) {
+ closestAfterIndex = index
+ closestAfterArea = a
+ }
+ }
+ })
+ }
+ })
+
+ if (closestBeforeIndex !== -1) return closestBeforeIndex + 1
+ if (closestAfterIndex !== -1) return closestAfterIndex
+
+ return -1
+ },
+ insertField (field) {
+ const insertIndex = this.findFieldInsertIndex(field)
+
+ if (insertIndex !== -1) {
+ this.template.fields.splice(insertIndex, 0, field)
+ } else {
+ this.template.fields.push(field)
+ }
+ },
closeDropdown () {
document.activeElement.blur()
},
@@ -1226,7 +1295,7 @@ export default {
field.preferences.with_signature_id = this.withSignatureId
}
- this.template.fields.push(field)
+ this.insertField(field)
this.save()
},
@@ -1455,11 +1524,13 @@ export default {
field.areas.push(area)
} else {
- this.template.fields.push({
+ const newField = {
...JSON.parse(JSON.stringify(field)),
uuid: v4(),
areas: [area]
- })
+ }
+
+ this.insertField(newField)
}
this.selectedAreaRef.value = area
@@ -1482,7 +1553,7 @@ export default {
const documentRef = this.documentRefs.find((e) => e.document.uuid === area.attachment_uuid)
const pageMask = documentRef.pageRefs[area.page].$refs.mask
- if (type === 'checkbox') {
+ if (type === 'checkbox' || type === 'radio' || type === 'multiple') {
area.w = pageMask.clientWidth / 30 / pageMask.clientWidth
area.h = (pageMask.clientWidth / 30 / pageMask.clientWidth) * (pageMask.clientWidth / pageMask.clientHeight)
} else if (type === 'image') {
@@ -1544,7 +1615,7 @@ export default {
}
if (this.template.fields.indexOf(this.drawField) === -1) {
- this.template.fields.push(this.drawField)
+ this.insertField(this.drawField)
}
this.drawField = null
@@ -1572,8 +1643,8 @@ export default {
const previousArea = previousField?.areas?.[previousField.areas.length - 1]
if (previousArea || area.w) {
- const areaW = previousArea?.w || (30 / pageMask.clientWidth)
- const areaH = previousArea?.h || (30 / pageMask.clientHeight)
+ const areaW = previousArea?.w || area.w || (30 / pageMask.clientWidth)
+ const areaH = previousArea?.h || area.h || (30 / pageMask.clientHeight)
if ((pageMask.clientWidth * area.w) < 5) {
area.x = area.x - (areaW / 2)
@@ -1683,7 +1754,7 @@ export default {
this.selectedAreaRef.value = fieldArea
if (this.template.fields.indexOf(field) === -1) {
- this.template.fields.push(field)
+ this.insertField(field)
}
this.save()
@@ -1713,7 +1784,7 @@ export default {
} else if (previousField?.areas?.length) {
baseArea = previousField.areas[previousField.areas.length - 1]
} else {
- if (['checkbox'].includes(fieldType)) {
+ if (['checkbox', 'radio', 'multiple'].includes(fieldType)) {
baseArea = {
w: area.maskW / 30 / area.maskW,
h: area.maskW / 30 / area.maskW * (area.maskW / area.maskH)
@@ -1845,7 +1916,7 @@ export default {
attachment.metadata.pdf.fields.forEach((field) => {
field.submitter_uuid = this.selectedSubmitter.uuid
- this.template.fields.push(field)
+ this.insertField(field)
})
}
})
diff --git a/app/javascript/template_builder/field_type.vue b/app/javascript/template_builder/field_type.vue
index 72267956..55633e08 100644
--- a/app/javascript/template_builder/field_type.vue
+++ b/app/javascript/template_builder/field_type.vue
@@ -160,7 +160,7 @@ export default {
payment: IconCreditCard,
phone: IconPhoneCheck,
verification: IconId,
- kba: IconUserScan,
+ kba: IconUserScan
}
},
skipTypes () {
diff --git a/app/models/submission.rb b/app/models/submission.rb
index 04a7ad85..5630af76 100644
--- a/app/models/submission.rb
+++ b/app/models/submission.rb
@@ -63,6 +63,8 @@ class Submission < ApplicationRecord
has_one_attached :audit_trail
has_one_attached :combined_document
+ has_one_attached :merged_document
+ has_one_attached :preview_merged_document
has_many_attached :preview_documents
has_many_attached :documents
diff --git a/app/views/accounts/show.html.erb b/app/views/accounts/show.html.erb
index 5ca2d649..5093d7bf 100644
--- a/app/views/accounts/show.html.erb
+++ b/app/views/accounts/show.html.erb
@@ -235,7 +235,7 @@
<% end %>
<% end %>
<% end %>
- <% if can?(:manage, :personalization_advanced) %>
+ <% if !Docuseal.multitenant? || can?(:manage, :personalization_advanced) %>
<% account_config = AccountConfig.find_or_initialize_by(account: current_account, key: AccountConfig::WITH_FILE_LINKS_KEY) %>
<% if can?(:manage, account_config) %>
<%= form_for account_config, url: account_configs_path, method: :post do |f| %>
diff --git a/app/views/templates/_file_form.html.erb b/app/views/templates/_file_form.html.erb
index d0c59485..eaedc346 100644
--- a/app/views/templates/_file_form.html.erb
+++ b/app/views/templates/_file_form.html.erb
@@ -1,12 +1,4 @@
<%= form_for @template, data: { turbo_frame: :_top }, html: { autocomplete: :off } do |f| %>
- <% if @base_template %>
- <%= hidden_field_tag :base_template_id, @base_template.id %>
- <% end %>
- <% if @base_template && (can?(:manage, :tenants) || true_user != current_user) && true_user.account.linked_accounts.active.accessible_by(current_ability).exists? %>
-
- <%= select_tag :account_id, options_for_select([true_user.account, *true_user.account.linked_accounts.active.accessible_by(current_ability)].uniq.map { |e| [e.name, e.id] }, current_account.id), required: true, class: 'base-select' %>
-
- <% end %>
<%= f.text_field :name, required: true, placeholder: t('document_name'), class: 'base-input', dir: 'auto' %>
@@ -16,7 +8,7 @@
-
+
- <%= f.button button_title(title: @base_template ? t('submit') : t('create'), disabled_with: t('creating')), class: 'base-button' %>
+ <%= f.button button_title(title: t('create'), disabled_with: t('creating')), class: 'base-button' %>
<% end %>
diff --git a/app/views/templates/_template.html.erb b/app/views/templates/_template.html.erb
index 1495be02..2ba35ebd 100644
--- a/app/views/templates/_template.html.erb
+++ b/app/views/templates/_template.html.erb
@@ -53,7 +53,7 @@
<% end %>
<% if can?(:create, template) %>
diff --git a/app/views/templates/_title.html.erb b/app/views/templates/_title.html.erb
index 99f11044..5d563e22 100644
--- a/app/views/templates/_title.html.erb
+++ b/app/views/templates/_title.html.erb
@@ -63,7 +63,7 @@
<%= button_to button_title(title: t('archive'), disabled_with: t('archiving'), title_class: 'inline', icon: svg_icon('archive', class: 'w-6 h-6')), template_path(template), class: 'btn btn-outline btn-sm w-full', form_class: 'flex-1', method: :delete, data: { turbo_confirm: t('are_you_sure_') } %>
<% end %>
<% if can?(:create, current_account.templates.new(author: current_user)) %>
- <%= link_to new_template_path(base_template_id: template.id), class: 'btn btn-outline btn-sm flex-1', data: { turbo_frame: :modal } do %>
+ <%= link_to new_template_clone_path(template), class: 'btn btn-outline btn-sm flex-1', data: { turbo_frame: :modal } do %>
<%= svg_icon('copy', class: 'w-6 h-6') %>
<%= t('clone') %>
diff --git a/app/views/templates_clone/_form.html.erb b/app/views/templates_clone/_form.html.erb
new file mode 100644
index 00000000..b2fb88b5
--- /dev/null
+++ b/app/views/templates_clone/_form.html.erb
@@ -0,0 +1,29 @@
+<%= form_for @template, url: template_clone_index_path(@base_template), data: { turbo_frame: :_top }, html: { autocomplete: :off } do |f| %>
+ <% accounts = Account.accessible_by(true_ability).where.not(id: true_user.account.testing_accounts).where(User.where(User.arel_table[:account_id].eq(Account.arel_table[:id])).arel.exists).active %>
+ <% if (can?(:manage, :tenants) || true_user != current_user) && accounts.where.not(id: current_account.id).exists? %>
+
+ <%= select_tag :account_id, options_for_select([current_account, *accounts.order(:name)].uniq.map { |e| [e.name, e.id] }, current_account.id), required: true, class: 'base-select' %>
+
+ <% end %>
+
+ <%= f.text_field :name, required: true, placeholder: t('document_name'), class: 'base-input', dir: 'auto' %>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <%= f.button button_title(title: t('submit'), disabled_with: t('creating')), class: 'base-button' %>
+
+<% end %>
diff --git a/app/views/templates_clone/new.html.erb b/app/views/templates_clone/new.html.erb
new file mode 100644
index 00000000..728c6d48
--- /dev/null
+++ b/app/views/templates_clone/new.html.erb
@@ -0,0 +1,3 @@
+<%= render 'shared/turbo_modal', title: t('clone_template') do %>
+ <%= render 'templates_clone/form' %>
+<% end %>
diff --git a/config/dotenv.rb b/config/dotenv.rb
index 319785e7..c12e6b2c 100644
--- a/config/dotenv.rb
+++ b/config/dotenv.rb
@@ -1,6 +1,20 @@
# frozen_string_literal: true
if ENV['RAILS_ENV'] == 'production'
+ if Process.uid.zero?
+ begin
+ workdir = ENV.fetch('WORKDIR', '.')
+
+ if File.exist?(workdir) && File.stat(workdir).uid != 2000
+ puts 'Changing the owner of the docuseal directory...' unless Dir.empty?(workdir)
+
+ FileUtils.chown_R(2000, 2000, workdir)
+ end
+ rescue StandardError
+ puts 'Unable to change docuseal directory owner'
+ end
+ end
+
if !ENV['AWS_SECRET_MANAGER_ID'].to_s.empty?
require 'aws-sdk-secretsmanager'
@@ -30,12 +44,30 @@ if ENV['RAILS_ENV'] == 'production'
File.write(dotenv_path, default_env)
end
+ if Process.uid.zero?
+ begin
+ File.chown(0, 0, dotenv_path)
+ File.chmod(0o600, dotenv_path)
+ rescue StandardError
+ puts 'Unable to set dotenv mod'
+ end
+ end
+
database_url = ENV.fetch('DATABASE_URL', nil)
Dotenv.load(dotenv_path)
ENV['DATABASE_URL'] = ENV['DATABASE_URL'].to_s.empty? ? database_url : ENV.fetch('DATABASE_URL', nil)
end
+
+ unless Process.uid == 2000
+ begin
+ Process::Sys.setgid(2000)
+ Process::Sys.setuid(2000)
+ rescue StandardError
+ puts 'Unable to run as 2000:2000'
+ end
+ end
end
if ENV['DATABASE_URL'].to_s.split('@').last.to_s.split('/').first.to_s.include?('_')
diff --git a/config/routes.rb b/config/routes.rb
index fa510839..26b6d617 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -92,6 +92,7 @@ Rails.application.routes.draw do
resources :templates, only: %i[index], controller: 'templates_dashboard'
resources :submissions_filters, only: %i[show], param: 'name'
resources :templates, only: %i[new create edit update show destroy] do
+ resources :clone, only: %i[new create], controller: 'templates_clone'
resource :debug, only: %i[show], controller: 'templates_debug' if Rails.env.development?
resources :documents, only: %i[index create], controller: 'template_documents'
resources :clone_and_replace, only: %i[create], controller: 'templates_clone_and_replace'
diff --git a/lib/submissions/generate_combined_attachment.rb b/lib/submissions/generate_combined_attachment.rb
index faf55471..bafd20a7 100644
--- a/lib/submissions/generate_combined_attachment.rb
+++ b/lib/submissions/generate_combined_attachment.rb
@@ -4,8 +4,8 @@ module Submissions
module GenerateCombinedAttachment
module_function
- def call(submitter)
- pdf = build_combined_pdf(submitter)
+ def call(submitter, with_audit: true)
+ pdf = build_combined_pdf(submitter, with_audit:)
submission = submitter.submission
account = submission.account
@@ -39,7 +39,7 @@ module Submissions
blob: ActiveStorage::Blob.create_and_upload!(
io: io.tap(&:rewind), filename: "#{submission.name || submission.template.name}.pdf"
),
- name: 'combined_document',
+ name: with_audit ? 'combined_document' : 'merged_document',
record: submission
)
end
@@ -58,14 +58,16 @@ module Submissions
pdf.sign(io, write_options: { validate: false }, **sign_params)
end
- def build_combined_pdf(submitter)
+ def build_combined_pdf(submitter, with_audit:)
pdfs_index = Submissions::GenerateResultAttachments.generate_pdfs(submitter)
- audit_trail = I18n.with_locale(submitter.account.locale) do
- Submissions::GenerateAuditTrail.build_audit_trail(submitter.submission)
- end
+ if with_audit
+ audit_trail = I18n.with_locale(submitter.account.locale) do
+ Submissions::GenerateAuditTrail.build_audit_trail(submitter.submission)
+ end
- audit_trail.dispatch_message(:complete_objects)
+ audit_trail.dispatch_message(:complete_objects)
+ end
result = HexaPDF::Document.new
@@ -79,7 +81,7 @@ module Submissions
pdf.pages.each { |page| result.pages << result.import(page) }
end
- audit_trail.pages.each { |page| result.pages << result.import(page) }
+ audit_trail&.pages&.each { |page| result.pages << result.import(page) }
result
end
diff --git a/lib/submissions/generate_preview_attachments.rb b/lib/submissions/generate_preview_attachments.rb
index b57b80a1..ff5a85a8 100644
--- a/lib/submissions/generate_preview_attachments.rb
+++ b/lib/submissions/generate_preview_attachments.rb
@@ -5,7 +5,7 @@ module Submissions
module_function
# rubocop:disable Metrics
- def call(submission, values_hash: nil, submitter: nil)
+ def call(submission, values_hash: nil, submitter: nil, merge: false)
values_hash ||= if submitter
build_submitter_values_hash(submitter)
else
@@ -42,48 +42,74 @@ module Submissions
template = submission.template
- image_pdfs = []
- original_documents = submission.schema_documents.preload(:blob)
+ if merge
+ result = HexaPDF::Document.new
- result_attachments =
- (submission.template_schema || template.schema).filter_map do |item|
+ (submission.template_schema || template.schema).each do |item|
pdf = pdfs_index[item['attachment_uuid']]
- next if pdf.nil?
+ next unless pdf
- if original_documents.find { |a| a.uuid == item['attachment_uuid'] }.image?
- pdf = GenerateResultAttachments.normalize_image_pdf(pdf)
+ pdf.dispatch_message(:complete_objects)
- image_pdfs << pdf
- end
-
- build_pdf_attachment(pdf:, submission:, submitter:,
- uuid: item['attachment_uuid'],
- values_hash:,
- name: item['name'])
- end
-
- return ApplicationRecord.no_touching { result_attachments.map { |e| e.tap(&:save!) } } if image_pdfs.size < 2
-
- images_pdf =
- image_pdfs.each_with_object(HexaPDF::Document.new) do |pdf, doc|
- pdf.pages.each { |page| doc.pages << doc.import(page) }
+ pdf.pages.each { |page| result.pages << result.import(page) }
end
- images_pdf = GenerateResultAttachments.normalize_image_pdf(images_pdf)
-
- images_pdf_attachment =
- build_pdf_attachment(
- pdf: images_pdf,
+ attachment = build_pdf_attachment(
+ pdf: result,
submission:,
- submitter:,
- uuid: GenerateResultAttachments.images_pdf_uuid(original_documents.select(&:image?)),
values_hash:,
- name: submission.name || template.name
+ name: 'preview_merged_document',
+ filename: "#{submission.name || template.name}.pdf"
)
- ApplicationRecord.no_touching do
- (result_attachments + [images_pdf_attachment]).map { |e| e.tap(&:save!) }
+ ApplicationRecord.no_touching { attachment.save! }
+
+ [attachment]
+ else
+ image_pdfs = []
+ original_documents = submission.schema_documents.preload(:blob)
+
+ result_attachments =
+ (submission.template_schema || template.schema).filter_map do |item|
+ pdf = pdfs_index[item['attachment_uuid']]
+
+ next if pdf.nil?
+
+ if original_documents.find { |a| a.uuid == item['attachment_uuid'] }.image?
+ pdf = GenerateResultAttachments.normalize_image_pdf(pdf)
+
+ image_pdfs << pdf
+ end
+
+ build_pdf_attachment(pdf:, submission:, submitter:,
+ uuid: item['attachment_uuid'],
+ values_hash:,
+ filename: "#{item['name']}.pdf")
+ end
+
+ return ApplicationRecord.no_touching { result_attachments.map { |e| e.tap(&:save!) } } if image_pdfs.size < 2
+
+ images_pdf =
+ image_pdfs.each_with_object(HexaPDF::Document.new) do |pdf, doc|
+ pdf.pages.each { |page| doc.pages << doc.import(page) }
+ end
+
+ images_pdf = GenerateResultAttachments.normalize_image_pdf(images_pdf)
+
+ images_pdf_attachment =
+ build_pdf_attachment(
+ pdf: images_pdf,
+ submission:,
+ submitter:,
+ uuid: GenerateResultAttachments.images_pdf_uuid(original_documents.select(&:image?)),
+ values_hash:,
+ filename: "#{submission.name || template.name}.pdf"
+ )
+
+ ApplicationRecord.no_touching do
+ (result_attachments + [images_pdf_attachment]).map { |e| e.tap(&:save!) }
+ end
end
end
@@ -102,7 +128,8 @@ module Submissions
)
end
- def build_pdf_attachment(pdf:, submission:, submitter:, uuid:, name:, values_hash:)
+ def build_pdf_attachment(pdf:, submission:, filename:, values_hash:, submitter: nil, uuid: nil,
+ name: 'preview_documents')
io = StringIO.new
begin
@@ -114,13 +141,13 @@ module Submissions
end
ActiveStorage::Attachment.new(
- blob: ActiveStorage::Blob.create_and_upload!(io: io.tap(&:rewind), filename: "#{name}.pdf"),
+ blob: ActiveStorage::Blob.create_and_upload!(io: io.tap(&:rewind), filename:),
io_data: io.string,
metadata: { original_uuid: uuid,
values_hash:,
analyzed: true,
- sha256: Base64.urlsafe_encode64(Digest::SHA256.digest(io.string)) },
- name: 'preview_documents',
+ sha256: Base64.urlsafe_encode64(Digest::SHA256.digest(io.string)) }.compact,
+ name: name,
record: submitter || submission
)
end
diff --git a/lib/submissions/normalize_param_utils.rb b/lib/submissions/normalize_param_utils.rb
index eedb4cc4..02b7e6a7 100644
--- a/lib/submissions/normalize_param_utils.rb
+++ b/lib/submissions/normalize_param_utils.rb
@@ -33,14 +33,18 @@ module Submissions
return submitter_params if default_values.blank?
values, new_attachments, new_fields =
- Submitters::NormalizeValues.call(template,
- default_values,
- submitter_name: submitter_params[:role] ||
- template.submitters.dig(index, 'name'),
- role_names: submitter_params[:roles],
- for_submitter:,
- add_fields:,
- throw_errors: !with_values)
+ Submitters::NormalizeValues.call(
+ template,
+ default_values,
+ submitter_name: submitter_params[:role] ||
+ (submitter_params[:uuid] &&
+ template.submitters.find { |s| s['uuid'] == submitter_params[:uuid] }&.dig('name')) ||
+ template.submitters.dig(index, 'name'),
+ role_names: submitter_params[:roles],
+ for_submitter:,
+ add_fields:,
+ throw_errors: !with_values
+ )
submitter_params[:values] = values
diff --git a/lib/templates/detect_fields.rb b/lib/templates/detect_fields.rb
index 0b7f444f..9cf4a095 100755
--- a/lib/templates/detect_fields.rb
+++ b/lib/templates/detect_fields.rb
@@ -60,24 +60,24 @@ module Templates
# rubocop:disable Metrics, Style
def call(io, attachment: nil, confidence: 0.3, temperature: 1, inference: Templates::ImageToFields, nms: 0.1,
- split_page: false, aspect_ratio: true, padding: 20, regexp_type: true, &)
+ nmm: 0.5, split_page: false, aspect_ratio: true, padding: 20, regexp_type: true, &)
fields, head_node =
if attachment&.image?
- process_image_attachment(io, attachment:, confidence:, nms:, split_page:, inference:,
+ process_image_attachment(io, attachment:, confidence:, nms:, nmm:, split_page:, inference:,
temperature:, aspect_ratio:, padding:, &)
else
- process_pdf_attachment(io, attachment:, confidence:, nms:, split_page:, inference:,
+ process_pdf_attachment(io, attachment:, confidence:, nms:, nmm:, split_page:, inference:,
temperature:, aspect_ratio:, regexp_type:, padding:, &)
end
[fields, head_node]
end
- def process_image_attachment(io, attachment:, confidence:, nms:, temperature:, inference:,
+ def process_image_attachment(io, attachment:, confidence:, nms:, nmm:, temperature:, inference:,
split_page: false, aspect_ratio: false, padding: nil)
image = Vips::Image.new_from_buffer(io.read, '')
- fields = inference.call(image, confidence:, nms:, split_page:,
+ fields = inference.call(image, confidence:, nms:, nmm:, split_page:,
temperature:, aspect_ratio:, padding:)
fields = sort_fields(fields, y_threshold: 10.0 / image.height)
@@ -104,7 +104,7 @@ module Templates
[fields, nil]
end
- def process_pdf_attachment(io, attachment:, confidence:, nms:, temperature:, inference:,
+ def process_pdf_attachment(io, attachment:, confidence:, nms:, nmm:, temperature:, inference:,
split_page: false, aspect_ratio: false, padding: nil, regexp_type: false)
doc = Pdfium::Document.open_bytes(io.read)
@@ -121,7 +121,7 @@ module Templates
image = Vips::Image.new_from_memory(data, width, height, 4, :uchar)
- fields = inference.call(image, confidence: confidence / 3.0, nms:, split_page:,
+ fields = inference.call(image, confidence: confidence / 3.0, nms:, nmm:, split_page:,
temperature:, aspect_ratio:, padding:)
text_fields = extract_text_fields_from_page(page)
diff --git a/lib/templates/image_to_fields.rb b/lib/templates/image_to_fields.rb
index e1c37135..2da4b925 100755
--- a/lib/templates/image_to_fields.rb
+++ b/lib/templates/image_to_fields.rb
@@ -26,7 +26,7 @@ module Templates
CPU_THREADS = Etc.nprocessors
# rubocop:disable Metrics
- def call(image, confidence: 0.3, nms: 0.1, temperature: 1,
+ def call(image, confidence: 0.3, nms: 0.1, nmm: 0.9, temperature: 1,
split_page: false, aspect_ratio: true, padding: nil, resolution: self.resolution)
image = image.extract_band(0, n: 3) if image.bands > 3
@@ -68,7 +68,7 @@ module Templates
detections = postprocess_outputs(boxes, logits, transform_info, confidence:, temperature:, resolution:)
end
- detections = apply_nms(detections, nms)
+ detections = apply_nms_nmm(detections, nms_threshold: nms, nmm_threshold: nmm)
build_fields_from_detections(detections, image)
end
@@ -245,7 +245,7 @@ module Templates
left, top, trim_width, trim_height = image.find_trim(threshold: 10, background: [255, 255, 255])
- trim_width = [trim_width, image.width * 0.7].max
+ trim_width = [trim_width, image.width - (left * 1.9)].max
padded_left = [left - padding, 0].max
padded_top = [top - padding, 0].max
@@ -297,7 +297,10 @@ module Templates
[img_array.reshape(1, 3, resolution, resolution), transform_info]
end
- def nms(boxes, scores, iou_threshold = 0.5)
+ def nms(detections, iou_threshold = 0.5)
+ boxes = detections[:xyxy]
+ scores = detections[:confidence]
+
return Numo::Int32[] if boxes.shape[0].zero?
x1 = boxes[true, 0]
@@ -333,7 +336,73 @@ module Templates
order = order[inds + 1]
end
- Numo::Int32.cast(keep)
+ {
+ xyxy: detections[:xyxy][keep, true],
+ confidence: detections[:confidence][keep],
+ class_id: detections[:class_id][keep]
+ }
+ end
+
+ def nmm(detections, overlap_threshold = 0.9, confidence: 0.3)
+ boxes = detections[:xyxy]
+ scores = detections[:confidence]
+ classes = detections[:class_id]
+
+ return detections if boxes.shape[0].zero?
+
+ x1 = boxes[true, 0]
+ y1 = boxes[true, 1]
+ x2 = boxes[true, 2]
+ y2 = boxes[true, 3]
+
+ areas = (x2 - x1) * (y2 - y1)
+ order = areas.sort_index.reverse
+
+ keep = []
+
+ while order.size.positive?
+ i = order[0]
+ keep << i
+ break if order.size == 1
+
+ xx1 = Numo::SFloat.maximum(x1[i], x1[order[1..]])
+ yy1 = Numo::SFloat.maximum(y1[i], y1[order[1..]])
+ xx2 = Numo::SFloat.minimum(x2[i], x2[order[1..]])
+ yy2 = Numo::SFloat.minimum(y2[i], y2[order[1..]])
+
+ w = Numo::SFloat.maximum(0.0, xx2 - xx1)
+ h = Numo::SFloat.maximum(0.0, yy2 - yy1)
+
+ intersection = w * h
+
+ overlap = intersection / areas[order[1..]]
+
+ merge_mask = scores[i] > confidence ? (overlap.gt(overlap_threshold) & classes[order[1..]].eq(classes[i])) : nil
+
+ if merge_mask && (merge_inds = merge_mask.where).size.positive?
+ candidates = order[merge_inds + 1]
+
+ scores[i] = [scores[i], scores[candidates].max].max
+
+ x1[i] = [x1[i], x1[candidates].min].min
+ y1[i] = [y1[i], y1[candidates].min].min
+ x2[i] = [x2[i], x2[candidates].max].max
+ y2[i] = [y2[i], y2[candidates].max].max
+ end
+
+ if merge_mask
+ inds = (~merge_mask).where
+ order = order[inds + 1]
+ else
+ order = order[1..]
+ end
+ end
+
+ {
+ xyxy: detections[:xyxy][keep, true],
+ confidence: detections[:confidence][keep],
+ class_id: detections[:class_id][keep]
+ }
end
def postprocess_outputs(boxes, logits, transform_info, detections = nil, confidence: 0.3, temperature: 1,
@@ -429,25 +498,21 @@ module Templates
end
end
- def apply_nms(detections, threshold = 0.5)
+ def apply_nms_nmm(detections, nms_threshold: 0.5, nmm_threshold: 0.7, confidence: 0.3)
return detections if detections[:xyxy].shape[0].zero?
- keep_indices = nms(detections[:xyxy], detections[:confidence], threshold)
+ nms_result = nms(detections, nms_threshold)
- {
- xyxy: detections[:xyxy][keep_indices, true],
- confidence: detections[:confidence][keep_indices],
- class_id: detections[:class_id][keep_indices]
- }
+ nmm(nms_result, nmm_threshold, confidence:)
end
def model
@model ||= OnnxRuntime::Model.new(
MODEL_PATH.to_s,
- inter_op_num_threads: CPU_THREADS,
+ inter_op_num_threads: 1,
intra_op_num_threads: CPU_THREADS,
enable_mem_pattern: false,
- enable_cpu_mem_arena: false,
+ enable_cpu_mem_arena: Docuseal.multitenant? || Rails.env.development?,
providers: ['CPUExecutionProvider']
)
end