diff --git a/.gitignore b/.gitignore index 5f01e718..d14f4595 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ yarn-debug.log* /docuseal /ee dump.rdb +*.onnx diff --git a/Dockerfile b/Dockerfile index 1e397cf0..b0be901f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ RUN apk --no-cache add fontforge wget && \ wget https://cdn.jsdelivr.net/gh/notofonts/notofonts.github.io/fonts/NotoSansSymbols2/hinted/ttf/NotoSansSymbols2-Regular.ttf && \ wget https://github.com/Maxattax97/gnu-freefont/raw/master/ttf/FreeSans.ttf && \ wget https://github.com/impallari/DancingScript/raw/master/OFL.txt && \ + wget -O /model.onnx "https://github.com/docusealco/fields-detection/releases/download/1.0.0/model_704_int8.onnx" && \ wget -O pdfium-linux.tgz "https://github.com/docusealco/pdfium-binaries/releases/latest/download/pdfium-linux-$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/').tgz" && \ mkdir -p /pdfium-linux && \ tar -xzf pdfium-linux.tgz -C /pdfium-linux @@ -50,7 +51,7 @@ ENV OPENSSL_CONF=/app/openssl_legacy.cnf WORKDIR /app -RUN echo '@edge https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories && apk add --no-cache sqlite-dev libpq-dev mariadb-dev vips-dev@edge yaml-dev redis libheif@edge vips-heif@edge gcompat ttf-freefont && mkdir /fonts && rm /usr/share/fonts/freefont/FreeSans.otf +RUN apk add --no-cache sqlite-dev libpq-dev mariadb-dev vips-dev yaml-dev redis libheif vips-heif gcompat ttf-freefont && mkdir /fonts && rm /usr/share/fonts/freefont/FreeSans.otf RUN echo $'.include = /etc/ssl/openssl.cnf\n\ \n\ @@ -66,7 +67,9 @@ activate = 1' >> /app/openssl_legacy.cnf COPY ./Gemfile ./Gemfile.lock ./ -RUN apk add --no-cache build-base && bundle install && apk del --no-cache build-base && rm -rf ~/.bundle /usr/local/bundle/cache && ruby -e "puts Dir['/usr/local/bundle/**/{spec,rdoc,resources/shared,resources/collation,resources/locales}']" | xargs rm -rf +RUN apk add --no-cache build-base && bundle install && apk del --no-cache build-base && rm -rf ~/.bundle /usr/local/bundle/cache && ruby -e "puts Dir['/usr/local/bundle/**/{spec,rdoc,resources/shared,resources/collation,resources/locales}']" | xargs rm -rf && ln -sf /usr/lib/libonnxruntime.so.1 $(ruby -e "print Dir[Gem::Specification.find_by_name('onnxruntime').gem_dir + '/vendor/*.so'].first") + +RUN echo 'https://dl-cdn.alpinelinux.org/alpine/edge/main' >> /etc/apk/repositories && echo 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories && apk add --no-cache onnxruntime COPY ./bin ./bin COPY ./app ./app @@ -83,6 +86,7 @@ COPY --from=download /fonts/GoNotoKurrent-Regular.ttf /fonts/GoNotoKurrent-Bold. COPY --from=download /fonts/FreeSans.ttf /usr/share/fonts/freefont COPY --from=download /pdfium-linux/lib/libpdfium.so /usr/lib/libpdfium.so COPY --from=download /pdfium-linux/licenses/pdfium.txt /usr/lib/libpdfium-LICENSE.txt +COPY --from=download /model.onnx /app/tmp/model.onnx COPY --from=webpack /app/public/packs ./public/packs RUN ln -s /fonts /app/public/fonts diff --git a/Gemfile b/Gemfile index 3a704a0d..b0974208 100644 --- a/Gemfile +++ b/Gemfile @@ -24,7 +24,9 @@ gem 'image_processing' gem 'jwt' gem 'lograge' gem 'mysql2', require: false +gem 'numo-narray' gem 'oj' +gem 'onnxruntime' gem 'pagy' gem 'pg', require: false gem 'premailer-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 84d527e6..f8f8b1ee 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -357,9 +357,18 @@ GEM racc (~> 1.4) nokogiri (1.18.9-x86_64-linux-musl) racc (~> 1.4) + numo-narray (0.9.2.1) oj (3.16.11) bigdecimal (>= 3.0) ostruct (>= 0.2) + onnxruntime (0.10.1) + ffi + onnxruntime (0.10.1-aarch64-linux) + ffi + onnxruntime (0.10.1-arm64-darwin) + ffi + onnxruntime (0.10.1-x86_64-linux) + ffi openssl (3.3.0) orm_adapter (0.5.0) os (1.1.4) @@ -638,7 +647,9 @@ DEPENDENCIES letter_opener_web lograge mysql2 + numo-narray oj + onnxruntime pagy pg premailer-rails diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 7a6587a5..0cd12d52 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -47,6 +47,7 @@ class AccountsController < ApplicationController authorize!(:manage, current_account) true_user.update!(locked_at: Time.current, email: true_user.email.sub('@', '+removed@')) + true_user.account.update!(archived_at: Time.current) # rubocop:disable Layout/LineLength render turbo_stream: turbo_stream.replace( diff --git a/app/controllers/api/attachments_controller.rb b/app/controllers/api/attachments_controller.rb index 5051cfeb..9d86b923 100644 --- a/app/controllers/api/attachments_controller.rb +++ b/app/controllers/api/attachments_controller.rb @@ -34,6 +34,10 @@ module Api end render json: attachment.as_json(only: %i[uuid created_at], methods: %i[url filename content_type]) + rescue Submitters::MaliciousFileExtension => e + Rollbar.error(e) if defined?(Rollbar) + + render json: { error: e.message }, status: :unprocessable_entity end def build_new_cookie_signatures_json(submitter, attachment) diff --git a/app/controllers/console_redirect_controller.rb b/app/controllers/console_redirect_controller.rb index dd80e9fe..4b910263 100644 --- a/app/controllers/console_redirect_controller.rb +++ b/app/controllers/console_redirect_controller.rb @@ -17,8 +17,10 @@ class ConsoleRedirectController < ApplicationController scope: :console, exp: 1.minute.from_now.to_i) - path = Addressable::URI.parse(params[:redir]).path if params[:redir].to_s.starts_with?(Docuseal::CONSOLE_URL) + redir_uri = Addressable::URI.parse(params[:redir]) + path = redir_uri.path if params[:redir].to_s.starts_with?(Docuseal::CONSOLE_URL) - redirect_to("#{Docuseal::CONSOLE_URL}#{path}?#{{ auth: }.to_query}", allow_other_host: true) + redirect_to "#{Docuseal::CONSOLE_URL}#{path}?#{{ **redir_uri&.query_values, 'auth' => auth }.to_query}", + allow_other_host: true end end diff --git a/app/controllers/send_submission_email_controller.rb b/app/controllers/send_submission_email_controller.rb index 45852360..c3a95158 100644 --- a/app/controllers/send_submission_email_controller.rb +++ b/app/controllers/send_submission_email_controller.rb @@ -29,7 +29,7 @@ class SendSubmissionEmailController < ApplicationController RateLimit.call("send-email-#{@submitter.id}", limit: 2, ttl: 5.minutes) - SubmitterMailer.documents_copy_email(@submitter, sig: true).deliver_later! unless already_sent?(@submitter) + SubmitterMailer.documents_copy_email(@submitter, sig: true).deliver_later! if can_send?(@submitter) respond_to do |f| f.html { render :success } @@ -39,8 +39,11 @@ class SendSubmissionEmailController < ApplicationController private - def already_sent?(submitter) - EmailEvent.exists?(tag: :submitter_documents_copy, email: submitter.email, emailable: submitter, - event_type: :send, created_at: SEND_DURATION.ago..Time.current) + def can_send?(submitter) + return false if submitter.account.archived_at? + return false if EmailEvent.exists?(tag: :submitter_documents_copy, email: submitter.email, emailable: submitter, + event_type: :send, created_at: SEND_DURATION.ago..Time.current) + + true end end diff --git a/app/controllers/submissions_preview_controller.rb b/app/controllers/submissions_preview_controller.rb index 40130e19..2ac30745 100644 --- a/app/controllers/submissions_preview_controller.rb +++ b/app/controllers/submissions_preview_controller.rb @@ -41,6 +41,9 @@ class SubmissionsPreviewController < ApplicationController def completed @submission = Submission.find_by!(slug: params[:submissions_preview_slug]) + + raise ActionController::RoutingError, I18n.t('not_found') if @submission.account.archived_at? + @template = @submission.template render :completed, layout: 'form' diff --git a/app/controllers/submit_form_controller.rb b/app/controllers/submit_form_controller.rb index f7bf565e..963594fa 100644 --- a/app/controllers/submit_form_controller.rb +++ b/app/controllers/submit_form_controller.rb @@ -75,7 +75,9 @@ class SubmitFormController < ApplicationController render json: { error: e.message }, status: :unprocessable_content end - def completed; end + def completed + raise ActionController::RoutingError, I18n.t('not_found') if @submitter.account.archived_at? + end def success; end diff --git a/app/controllers/templates_debug_controller.rb b/app/controllers/templates_debug_controller.rb index 676c2f64..edaedff0 100644 --- a/app/controllers/templates_debug_controller.rb +++ b/app/controllers/templates_debug_controller.rb @@ -6,12 +6,18 @@ class TemplatesDebugController < ApplicationController DEBUG_FILE = '' def show - attachment = @template.documents.first + schema_uuids = @template.schema.index_by { |e| e['attachment_uuid'] } + attachment = @template.documents.find { |a| schema_uuids[a.uuid] } data = attachment.download - pdf = HexaPDF::Document.new(io: StringIO.new(data)) - fields = Templates::FindAcroFields.call(pdf, attachment, data) + unless attachment.image? + pdf = HexaPDF::Document.new(io: StringIO.new(data)) + + fields = Templates::FindAcroFields.call(pdf, attachment, data) + end + + fields = Templates::DetectFields.call(StringIO.new(data), attachment:) if fields.blank? attachment.metadata['pdf'] ||= {} attachment.metadata['pdf']['fields'] = fields diff --git a/app/controllers/templates_detect_fields_controller.rb b/app/controllers/templates_detect_fields_controller.rb new file mode 100644 index 00000000..8355dcb2 --- /dev/null +++ b/app/controllers/templates_detect_fields_controller.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class TemplatesDetectFieldsController < ApplicationController + include ActionController::Live + + load_and_authorize_resource :template + + def create + response.headers['Content-Type'] = 'text/event-stream' + + sse = SSE.new(response.stream) + + documents = @template.schema_documents.preload(:blob) + + documents.each do |document| + io = StringIO.new(document.download) + + Templates::DetectFields.call(io, attachment: document) do |(attachment_uuid, page, fields)| + sse.write({ attachment_uuid:, page:, fields: }) + end + end + + sse.write({ completed: true }) + ensure + response.stream.close + end +end diff --git a/app/javascript/application.js b/app/javascript/application.js index 31e28407..9b17172b 100644 --- a/app/javascript/application.js +++ b/app/javascript/application.js @@ -156,6 +156,7 @@ safeRegisterElement('template-builder', class extends HTMLElement { withPhone: this.dataset.withPhone === 'true', withVerification: ['true', 'false'].includes(this.dataset.withVerification) ? this.dataset.withVerification === 'true' : null, withLogo: this.dataset.withLogo !== 'false', + withFieldsDetection: this.dataset.withFieldsDetection === 'true', editable: this.dataset.editable !== 'false', authenticityToken: document.querySelector('meta[name="csrf-token"]')?.content, withPayment: this.dataset.withPayment === 'true', diff --git a/app/javascript/submission_form/area.vue b/app/javascript/submission_form/area.vue index 2a2fd20e..d3a4eaf6 100644 --- a/app/javascript/submission_form/area.vue +++ b/app/javascript/submission_form/area.vue @@ -553,6 +553,10 @@ export default { style.color = this.field.preferences.color } + if (this.field.preferences?.background) { + style.background = this.field.preferences.background + } + return style }, isNarrow () { diff --git a/app/javascript/submission_form/dropzone.vue b/app/javascript/submission_form/dropzone.vue index 89e01b3b..275b0f30 100644 --- a/app/javascript/submission_form/dropzone.vue +++ b/app/javascript/submission_form/dropzone.vue @@ -163,12 +163,20 @@ export default { return fetch(this.baseUrl + '/api/attachments', { method: 'POST', body: formData - }).then(resp => resp.json()).then((data) => { - return data + }).then(async (resp) => { + const data = await resp.json() + + if (resp.status === 422) { + alert(data.error) + } else { + return data + } }) } })).then((result) => { - this.$emit('upload', result) + if (result && result[0]) { + this.$emit('upload', result) + } }).finally(() => { this.isLoading = false }) diff --git a/app/javascript/submission_form/initials_step.vue b/app/javascript/submission_form/initials_step.vue index 376fb7e9..dad1a944 100644 --- a/app/javascript/submission_form/initials_step.vue +++ b/app/javascript/submission_form/initials_step.vue @@ -123,6 +123,11 @@ v-if="!isDrawInitials" class="absolute top-0 right-0 left-0 bottom-0" /> +