diff --git a/app/controllers/api/submissions_controller.rb b/app/controllers/api/submissions_controller.rb index fc89c1d3..41b73fb9 100644 --- a/app/controllers/api/submissions_controller.rb +++ b/app/controllers/api/submissions_controller.rb @@ -10,7 +10,7 @@ module Api end def index - submissions = Submissions.search(@submissions, params[:q]) + submissions = Submissions.search(current_user, @submissions, params[:q]) submissions = filter_submissions(submissions, params) submissions = paginate(submissions.preload(:created_by_user, :submitters, @@ -80,6 +80,8 @@ module Api end end + SearchEntries.enqueue_reindex(submissions) + render json: build_create_json(submissions) rescue Submitters::NormalizeValues::BaseError, Submissions::CreateFromSubmitters::BaseError, DownloadUtils::UnableToDownload => e diff --git a/app/controllers/api/submitters_controller.rb b/app/controllers/api/submitters_controller.rb index 4d53d3cb..1731968f 100644 --- a/app/controllers/api/submitters_controller.rb +++ b/app/controllers/api/submitters_controller.rb @@ -5,7 +5,7 @@ module Api load_and_authorize_resource :submitter def index - submitters = Submitters.search(@submitters, params[:q]) + submitters = Submitters.search(current_user, @submitters, params[:q]) submitters = filter_submitters(submitters, params) @@ -65,6 +65,8 @@ module Api Submitters.send_signature_requests([@submitter]) end + SearchEntries.enqueue_reindex(@submitter) + render json: Submitters::SerializeForApi.call(@submitter, with_template: false, with_urls: true, with_events: false, diff --git a/app/controllers/api/templates_clone_controller.rb b/app/controllers/api/templates_clone_controller.rb index bf9aa671..2619a243 100644 --- a/app/controllers/api/templates_clone_controller.rb +++ b/app/controllers/api/templates_clone_controller.rb @@ -33,6 +33,8 @@ module Api 'webhook_url_id' => webhook_url.id) end + SearchEntries.enqueue_reindex(cloned_template) + render json: Templates::SerializeForApi.call(cloned_template, schema_documents) end end diff --git a/app/controllers/api/templates_controller.rb b/app/controllers/api/templates_controller.rb index 25c0c537..a36732a3 100644 --- a/app/controllers/api/templates_controller.rb +++ b/app/controllers/api/templates_controller.rb @@ -65,6 +65,8 @@ module Api @template.update!(template_params) + SearchEntries.enqueue_reindex(@template) + WebhookUrls.for_account_id(@template.account_id, 'template.updated').each do |webhook_url| SendTemplateUpdatedWebhookRequestJob.perform_async('template_id' => @template.id, 'webhook_url_id' => webhook_url.id) @@ -86,7 +88,7 @@ module Api private def filter_templates(templates, params) - templates = Templates.search(templates, params[:q]) + templates = Templates.search(current_user, templates, params[:q]) templates = params[:archived].in?(['true', true]) ? templates.archived : templates.active templates = templates.where(external_id: params[:application_key]) if params[:application_key].present? templates = templates.where(external_id: params[:external_id]) if params[:external_id].present? diff --git a/app/controllers/start_form_controller.rb b/app/controllers/start_form_controller.rb index b2957c1c..fb2a9786 100644 --- a/app/controllers/start_form_controller.rb +++ b/app/controllers/start_form_controller.rb @@ -53,6 +53,8 @@ class StartFormController < ApplicationController if is_new_record enqueue_submission_create_webhooks(@submitter) + SearchEntries.enqueue_reindex(@submitter) + if @submitter.submission.expire_at? ProcessSubmissionExpiredJob.perform_at(@submitter.submission.expire_at, 'submission_id' => @submitter.submission_id) diff --git a/app/controllers/submissions_archived_controller.rb b/app/controllers/submissions_archived_controller.rb index 9bd67da2..793da755 100644 --- a/app/controllers/submissions_archived_controller.rb +++ b/app/controllers/submissions_archived_controller.rb @@ -9,7 +9,7 @@ class SubmissionsArchivedController < ApplicationController .or(@submissions.where.not(templates: { archived_at: nil })) .preload(:template_accesses, :created_by_user, template: :author) - @submissions = Submissions.search(@submissions, params[:q], search_template: true) + @submissions = Submissions.search(current_user, @submissions, params[:q], search_template: true) @submissions = Submissions::Filter.call(@submissions, current_user, params) @submissions = if params[:completed_at_from].present? || params[:completed_at_to].present? diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index 0240b5be..98d7cb63 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -8,6 +8,10 @@ class SubmissionsController < ApplicationController prepend_before_action :maybe_redirect_com, only: %i[show] + before_action only: :create do + authorize!(:create, Submission) + end + def show @submission = Submissions.preload_with_pages(@submission) @@ -26,8 +30,6 @@ class SubmissionsController < ApplicationController end def create - authorize!(:create, Submission) - save_template_message(@template, params) if params[:save_message] == '1' if params[:is_custom_message] != '1' @@ -56,6 +58,8 @@ class SubmissionsController < ApplicationController Submissions.send_signature_requests(submissions) + SearchEntries.enqueue_reindex(submissions) + redirect_to template_path(@template), notice: I18n.t('new_recipients_have_been_added') rescue Submissions::CreateFromSubmitters::BaseError => e render turbo_stream: turbo_stream.replace(:submitters_error, diff --git a/app/controllers/submissions_dashboard_controller.rb b/app/controllers/submissions_dashboard_controller.rb index 3d0950a6..f0851741 100644 --- a/app/controllers/submissions_dashboard_controller.rb +++ b/app/controllers/submissions_dashboard_controller.rb @@ -10,7 +10,7 @@ class SubmissionsDashboardController < ApplicationController .where(templates: { archived_at: nil }) .preload(:template_accesses, :created_by_user, template: :author) - @submissions = Submissions.search(@submissions, params[:q], search_template: true) + @submissions = Submissions.search(current_user, @submissions, params[:q], search_template: true) @submissions = Submissions::Filter.call(@submissions, current_user, params) @submissions = if params[:completed_at_from].present? || params[:completed_at_to].present? diff --git a/app/controllers/submitters_autocomplete_controller.rb b/app/controllers/submitters_autocomplete_controller.rb index e86e129e..70f02747 100644 --- a/app/controllers/submitters_autocomplete_controller.rb +++ b/app/controllers/submitters_autocomplete_controller.rb @@ -22,13 +22,17 @@ class SubmittersAutocompleteController < ApplicationController def search_submitters(submitters) if SELECT_COLUMNS.include?(params[:field]) - column = Submitter.arel_table[params[:field].to_sym] + if Docuseal.fulltext_search?(current_user) + Submitters.fulltext_search_field(current_user, submitters, params[:q], params[:field]) + else + column = Submitter.arel_table[params[:field].to_sym] - term = "#{params[:q].downcase}%" + term = "#{params[:q].downcase}%" - submitters.where(column.matches(term)) + submitters.where(column.matches(term)) + end else - Submitters.search(submitters, params[:q]) + Submitters.search(current_user, submitters, params[:q]) end end end diff --git a/app/controllers/submitters_controller.rb b/app/controllers/submitters_controller.rb index 3adb3c89..697b3dc1 100644 --- a/app/controllers/submitters_controller.rb +++ b/app/controllers/submitters_controller.rb @@ -38,6 +38,8 @@ class SubmittersController < ApplicationController if @submitter.save maybe_resend_email_sms(@submitter, params) + SearchEntries.enqueue_reindex(@submitter) + redirect_back fallback_location: submission_path(submission), notice: I18n.t('changes_have_been_saved') else redirect_back fallback_location: submission_path(submission), alert: I18n.t('unable_to_save') diff --git a/app/controllers/template_folders_controller.rb b/app/controllers/template_folders_controller.rb index c22f2fed..25dad2c1 100644 --- a/app/controllers/template_folders_controller.rb +++ b/app/controllers/template_folders_controller.rb @@ -6,7 +6,7 @@ class TemplateFoldersController < ApplicationController def show @templates = @template_folder.templates.active.accessible_by(current_ability) .preload(:author, :template_accesses) - @templates = Templates.search(@templates, params[:q]) + @templates = Templates.search(current_user, @templates, params[:q]) @templates = Templates::Order.call(@templates, current_user, cookies.permanent[:dashboard_templates_order]) @pagy, @templates = pagy_auto(@templates, limit: 12) diff --git a/app/controllers/templates_archived_controller.rb b/app/controllers/templates_archived_controller.rb index f52e5af7..34d2b601 100644 --- a/app/controllers/templates_archived_controller.rb +++ b/app/controllers/templates_archived_controller.rb @@ -5,7 +5,7 @@ class TemplatesArchivedController < ApplicationController def index @templates = @templates.where.not(archived_at: nil).preload(:author, :folder, :template_accesses).order(id: :desc) - @templates = Templates.search(@templates, params[:q]) + @templates = Templates.search(current_user, @templates, params[:q]) @pagy, @templates = pagy_auto(@templates, limit: 12) end diff --git a/app/controllers/templates_archived_submissions_controller.rb b/app/controllers/templates_archived_submissions_controller.rb index 9c9da083..39aa0877 100644 --- a/app/controllers/templates_archived_submissions_controller.rb +++ b/app/controllers/templates_archived_submissions_controller.rb @@ -6,7 +6,7 @@ class TemplatesArchivedSubmissionsController < ApplicationController def index @submissions = @submissions.where.not(archived_at: nil) - @submissions = Submissions.search(@submissions, params[:q], search_values: true) + @submissions = Submissions.search(current_user, @submissions, params[:q], search_values: true) @submissions = Submissions::Filter.call(@submissions, current_user, params) @submissions = if params[:completed_at_from].present? || params[:completed_at_to].present? diff --git a/app/controllers/templates_clone_and_replace_controller.rb b/app/controllers/templates_clone_and_replace_controller.rb index 0d75416b..21989067 100644 --- a/app/controllers/templates_clone_and_replace_controller.rb +++ b/app/controllers/templates_clone_and_replace_controller.rb @@ -22,6 +22,8 @@ class TemplatesCloneAndReplaceController < ApplicationController Templates::CloneAttachments.call(template: cloned_template, original_template: @template, excluded_attachment_uuids: documents.map(&:uuid)) + SearchEntries.enqueue_reindex(cloned_template) + respond_to do |f| f.html { redirect_to edit_template_path(cloned_template) } f.json { render json: { id: cloned_template.id } } diff --git a/app/controllers/templates_controller.rb b/app/controllers/templates_controller.rb index b3d0ccec..40c8dbd1 100644 --- a/app/controllers/templates_controller.rb +++ b/app/controllers/templates_controller.rb @@ -8,7 +8,7 @@ class TemplatesController < ApplicationController def show submissions = @template.submissions.accessible_by(current_ability) submissions = submissions.active if @template.archived_at.blank? - submissions = Submissions.search(submissions, params[:q], search_values: true) + submissions = Submissions.search(current_user, submissions, params[:q], search_values: true) submissions = Submissions::Filter.call(submissions, current_user, params.except(:status)) @base_submissions = submissions @@ -72,6 +72,8 @@ class TemplatesController < ApplicationController if @template.save Templates::CloneAttachments.call(template: @template, original_template: @base_template) if @base_template + SearchEntries.enqueue_reindex(@template) + enqueue_template_created_webhooks(@template) maybe_redirect_to_template(@template) @@ -81,7 +83,13 @@ class TemplatesController < ApplicationController end def update - @template.update!(template_params) + @template.assign_attributes(template_params) + + is_name_changed = @template.name_changed? + + @template.save! + + SearchEntries.enqueue_reindex(@template) if is_name_changed enqueue_template_updated_webhooks(@template) diff --git a/app/controllers/templates_dashboard_controller.rb b/app/controllers/templates_dashboard_controller.rb index b4806fbc..3649ac0f 100644 --- a/app/controllers/templates_dashboard_controller.rb +++ b/app/controllers/templates_dashboard_controller.rb @@ -57,7 +57,7 @@ class TemplatesDashboardController < ApplicationController end end - Templates.search(rel, params[:q]) + Templates.search(current_user, rel, params[:q]) end def sort_template_folders(template_folders, current_user, order) diff --git a/app/controllers/templates_uploads_controller.rb b/app/controllers/templates_uploads_controller.rb index 4cd0072f..45403a7a 100644 --- a/app/controllers/templates_uploads_controller.rb +++ b/app/controllers/templates_uploads_controller.rb @@ -25,6 +25,8 @@ class TemplatesUploadsController < ApplicationController enqueue_template_created_webhooks(@template) + SearchEntries.enqueue_reindex(@template) + redirect_to edit_template_path(@template) rescue Templates::CreateAttachments::PdfEncrypted render turbo_stream: turbo_stream.append(params[:form_id], html: helpers.tag.prompt_password) diff --git a/app/javascript/elements/submitter_autocomplete.js b/app/javascript/elements/submitter_autocomplete.js index 5fc539bf..bf24739a 100644 --- a/app/javascript/elements/submitter_autocomplete.js +++ b/app/javascript/elements/submitter_autocomplete.js @@ -36,7 +36,9 @@ export default class extends HTMLElement { } fetch = (text, resolve) => { - const q = text.split(/[;,\s]+/).pop().trim() + const q = this.dataset.field === 'email' + ? text.split(/[;,\s]+/).pop().trim() + : text if (q) { const queryParams = new URLSearchParams({ q, field: this.dataset.field }) diff --git a/app/jobs/reindex_search_entry_job.rb b/app/jobs/reindex_search_entry_job.rb new file mode 100644 index 00000000..c5650021 --- /dev/null +++ b/app/jobs/reindex_search_entry_job.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ReindexSearchEntryJob + include Sidekiq::Job + + InvalidFormat = Class.new(StandardError) + + def perform(params = {}) + entry = SearchEntry.find_or_initialize_by(params.slice('record_type', 'record_id')) + + SearchEntries.reindex_record(entry.record) + end +end diff --git a/app/models/search_entry.rb b/app/models/search_entry.rb new file mode 100644 index 00000000..bc641669 --- /dev/null +++ b/app/models/search_entry.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: search_entries +# +# id :bigint not null, primary key +# record_type :string not null +# tsvector :tsvector not null +# created_at :datetime not null +# updated_at :datetime not null +# account_id :bigint not null +# record_id :bigint not null +# +# Indexes +# +# index_search_entries_on_account_id_tsvector_submission (account_id,tsvector) WHERE ((record_type)::text = 'Submission'::text) USING gin +# index_search_entries_on_account_id_tsvector_submitter (account_id,tsvector) WHERE ((record_type)::text = 'Submitter'::text) USING gin +# index_search_entries_on_account_id_tsvector_template (account_id,tsvector) WHERE ((record_type)::text = 'Template'::text) USING gin +# index_search_entries_on_record_id_and_record_type (record_id,record_type) UNIQUE +# +class SearchEntry < ApplicationRecord + belongs_to :record, polymorphic: true + belongs_to :account +end diff --git a/app/models/submission.rb b/app/models/submission.rb index f0fe283c..31547d87 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -38,6 +38,8 @@ class Submission < ApplicationRecord belongs_to :account belongs_to :created_by_user, class_name: 'User', optional: true + has_one :search_entry, as: :record, inverse_of: :record, dependent: :destroy + has_many :submitters, dependent: :destroy has_many :submission_events, dependent: :destroy diff --git a/app/models/submitter.rb b/app/models/submitter.rb index 91322f96..fd9dfc6b 100644 --- a/app/models/submitter.rb +++ b/app/models/submitter.rb @@ -41,6 +41,7 @@ class Submitter < ApplicationRecord belongs_to :submission belongs_to :account has_one :template, through: :submission + has_one :search_entry, as: :record, inverse_of: :record, dependent: :destroy attribute :values, :string, default: -> { {} } attribute :preferences, :string, default: -> { {} } diff --git a/app/models/template.rb b/app/models/template.rb index faa9e696..dbfadb79 100644 --- a/app/models/template.rb +++ b/app/models/template.rb @@ -42,6 +42,8 @@ class Template < ApplicationRecord belongs_to :account belongs_to :folder, class_name: 'TemplateFolder' + has_one :search_entry, as: :record, inverse_of: :record, dependent: :destroy + before_validation :maybe_set_default_folder, on: :create attribute :preferences, :string, default: -> { {} } diff --git a/app/views/submissions_dashboard/index.html.erb b/app/views/submissions_dashboard/index.html.erb index 2ba16b18..286e0b91 100644 --- a/app/views/submissions_dashboard/index.html.erb +++ b/app/views/submissions_dashboard/index.html.erb @@ -26,7 +26,7 @@ <% view_archived_html = capture do %> - <% if current_account.submissions.where.not(archived_at: nil).exists? %> + <% if can?(:manage, :countless) || current_account.submissions.where.not(archived_at: nil).exists? %>
<%= t('view_archived') %>
diff --git a/app/views/templates/show.html.erb b/app/views/templates/show.html.erb index 18188523..c749ef1f 100644 --- a/app/views/templates/show.html.erb +++ b/app/views/templates/show.html.erb @@ -113,7 +113,7 @@ <% end %> <% view_archived_html = capture do %> - <% if @template.submissions.where.not(archived_at: nil).exists? && !@template.archived_at? %> + <% if (can?(:manage, :countless) || @template.submissions.where.not(archived_at: nil).exists?) && !@template.archived_at? %>
<%= t('view_archived') %>
diff --git a/app/views/templates_dashboard/index.html.erb b/app/views/templates_dashboard/index.html.erb index 99f4d80e..f1a4c3de 100644 --- a/app/views/templates_dashboard/index.html.erb +++ b/app/views/templates_dashboard/index.html.erb @@ -1,4 +1,4 @@ -<% has_archived = current_account.templates.where.not(archived_at: nil).exists? %> +<% has_archived = can?(:manage, :countless) || current_account.templates.where.not(archived_at: nil).exists? %> <% show_dropzone = params[:q].blank? && @pagy.pages == 1 && ((@template_folders.size < 10 && @templates.size.zero?) || (@template_folders.size < 7 && @templates.size < 4) || (@template_folders.size < 4 && @templates.size < 7)) %> <% if Docuseal.demo? %><%= render 'shared/demo_alert' %><% end %> diff --git a/db/migrate/20250603105556_create_search_enties.rb b/db/migrate/20250603105556_create_search_enties.rb new file mode 100644 index 00000000..2b219f84 --- /dev/null +++ b/db/migrate/20250603105556_create_search_enties.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class CreateSearchEnties < ActiveRecord::Migration[8.0] + def up + return unless adapter_name == 'PostgreSQL' + + enable_extension 'btree_gin' + + create_table :search_entries do |t| + t.references :record, null: false, polymorphic: true, index: false + t.bigint :account_id, null: false + t.tsvector :tsvector, null: false + + t.timestamps + end + + add_index :search_entries, %i[account_id tsvector], using: :gin, where: "record_type = 'Submitter'", + name: 'index_search_entries_on_account_id_tsvector_submitter' + add_index :search_entries, %i[account_id tsvector], using: :gin, where: "record_type = 'Submission'", + name: 'index_search_entries_on_account_id_tsvector_submission' + add_index :search_entries, %i[account_id tsvector], using: :gin, where: "record_type = 'Template'", + name: 'index_search_entries_on_account_id_tsvector_template' + add_index :search_entries, %i[record_id record_type], unique: true + end + + def down + return unless adapter_name == 'PostgreSQL' + + drop_table :search_entries + + disable_extension 'btree_gin' + end +end diff --git a/db/schema.rb b/db/schema.rb index 1e789e2b..ca4801e4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,8 +10,9 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2025_05_31_085328) do +ActiveRecord::Schema[8.0].define(version: 2025_06_03_105556) do # These are extensions that must be enabled in order to support this database + enable_extension "btree_gin" enable_extension "plpgsql" create_table "access_tokens", force: :cascade do |t| @@ -256,6 +257,19 @@ ActiveRecord::Schema[8.0].define(version: 2025_05_31_085328) do t.index ["uid"], name: "index_oauth_applications_on_uid", unique: true end + create_table "search_entries", force: :cascade do |t| + t.string "record_type", null: false + t.bigint "record_id", null: false + t.bigint "account_id", null: false + t.tsvector "tsvector", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id", "tsvector"], name: "index_search_entries_on_account_id_tsvector_submission", where: "((record_type)::text = 'Submission'::text)", using: :gin + t.index ["account_id", "tsvector"], name: "index_search_entries_on_account_id_tsvector_submitter", where: "((record_type)::text = 'Submitter'::text)", using: :gin + t.index ["account_id", "tsvector"], name: "index_search_entries_on_account_id_tsvector_template", where: "((record_type)::text = 'Template'::text)", using: :gin + t.index ["record_id", "record_type"], name: "index_search_entries_on_record_id_and_record_type", unique: true + end + create_table "submission_events", force: :cascade do |t| t.bigint "submission_id", null: false t.bigint "submitter_id" diff --git a/lib/docuseal.rb b/lib/docuseal.rb index 3c11deda..77d28a5e 100644 --- a/lib/docuseal.rb +++ b/lib/docuseal.rb @@ -73,6 +73,14 @@ module Docuseal @default_pkcs ||= GenerateCertificate.load_pkcs(Docuseal::CERTS) end + def fulltext_search?(_user) + return false unless SearchEntry.table_exists? + return true if Docuseal.multitenant? + return true if Rails.env.local? + + false + end + def enable_pwa? true end diff --git a/lib/phone_codes.rb b/lib/phone_codes.rb new file mode 100644 index 00000000..43f8ea14 --- /dev/null +++ b/lib/phone_codes.rb @@ -0,0 +1,228 @@ +# frozen_string_literal: true + +module PhoneCodes + ALL = [ + '+1', + '+93', + '+358', + '+355', + '+213', + '+1684', + '+376', + '+244', + '+1264', + '+1268', + '+54', + '+374', + '+297', + '+61', + '+43', + '+994', + '+1242', + '+973', + '+880', + '+1246', + '+32', + '+501', + '+229', + '+1441', + '+975', + '+591', + '+387', + '+267', + '+55', + '+246', + '+673', + '+359', + '+226', + '+257', + '+855', + '+237', + '+1', + '+238', + '+1345', + '+235', + '+56', + '+86', + '+61', + '+61', + '+57', + '+269', + '+243', + '+682', + '+506', + '+225', + '+385', + '+357', + '+420', + '+45', + '+253', + '+1767', + '+1849', + '+593', + '+20', + '+503', + '+240', + '+291', + '+372', + '+251', + '+500', + '+298', + '+679', + '+358', + '+33', + '+594', + '+689', + '+241', + '+220', + '+995', + '+49', + '+233', + '+350', + '+30', + '+299', + '+1473', + '+590', + '+1671', + '+502', + '+224', + '+245', + '+592', + '+509', + '+504', + '+852', + '+36', + '+354', + '+91', + '+62', + '+964', + '+353', + '+44', + '+972', + '+39', + '+1876', + '+81', + '+44', + '+962', + '+7', + '+254', + '+686', + '+82', + '+965', + '+996', + '+856', + '+371', + '+961', + '+266', + '+231', + '+423', + '+370', + '+352', + '+853', + '+389', + '+261', + '+265', + '+60', + '+960', + '+223', + '+356', + '+692', + '+596', + '+222', + '+230', + '+262', + '+52', + '+691', + '+373', + '+377', + '+976', + '+382', + '+1664', + '+212', + '+258', + '+264', + '+674', + '+977', + '+31', + '+687', + '+64', + '+227', + '+234', + '+683', + '+672', + '+1670', + '+47', + '+968', + '+92', + '+680', + '+507', + '+675', + '+595', + '+51', + '+63', + '+872', + '+48', + '+351', + '+1939', + '+974', + '+40', + '+250', + '+262', + '+590', + '+290', + '+1869', + '+1758', + '+590', + '+508', + '+1784', + '+685', + '+378', + '+239', + '+966', + '+221', + '+381', + '+248', + '+232', + '+65', + '+421', + '+386', + '+677', + '+27', + '+34', + '+94', + '+597', + '+47', + '+268', + '+46', + '+41', + '+886', + '+992', + '+255', + '+66', + '+670', + '+228', + '+690', + '+676', + '+1868', + '+216', + '+90', + '+993', + '+1649', + '+688', + '+256', + '+380', + '+971', + '+44', + '+598', + '+998', + '+678', + '+84', + '+1284', + '+1340', + '+681', + '+967', + '+260' + ].freeze + + REGEXP = /\A#{Regexp.union(ALL).source}/i +end diff --git a/lib/search_entries.rb b/lib/search_entries.rb new file mode 100644 index 00000000..6dbcc7f9 --- /dev/null +++ b/lib/search_entries.rb @@ -0,0 +1,167 @@ +# frozen_string_literal: true + +module SearchEntries + TRANSLITERATIONS = + I18n::Backend::Transliterator::HashTransliterator::DEFAULT_APPROXIMATIONS.reject { |_, v| v.length > 1 } + + MAX_VALUE_LENGTH = 100 + + UUID_REGEXP = /\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i + + FIELD_SEARCH_QUERY_SQL = <<~SQL.squish + tsvector @@ (quote_literal(coalesce((ts_lexize('english_stem', :keyword))[1], :keyword)) || ':*' || :weight)::tsquery + SQL + + module_function + + def reindex_all + Submitter.find_each { |submitter| index_submitter(submitter) } + Submission.find_each { |submission| index_submission(submission) } + Template.find_each { |template| index_template(template) } + end + + def enqueue_reindex(records) + return unless SearchEntry.table_exists? + + args = Array.wrap(records).map { |e| [{ 'record_type' => e.class.name, 'record_id' => e.id }] } + + ReindexSearchEntryJob.perform_bulk(args) + end + + def reindex_record(record) + case record + when Submitter + index_submitter(record) + when Template + index_template(record) + when Submission + index_submission(record) + + record.submitters.each do |submitter| + index_submitter(submitter) + end + else + raise ArgumentError, 'Invalid Record' + end + end + + def build_tsquery(keyword) + if keyword.match?(/\d/) && !keyword.match?(/\p{L}/) + number = keyword.gsub(/\D/, '') + + ["tsvector @@ ((quote_literal(?) || ':*')::tsquery || (quote_literal(?) || ':*')::tsquery || plainto_tsquery(?))", + number, number.length > 1 ? number.delete_prefix('0') : number, keyword] + elsif keyword.match?(/[^\p{L}\d&@._\-+]/) || keyword.match?(/\A['"].*['"]\z/) + ['tsvector @@ plainto_tsquery(?)', TextUtils.transliterate(keyword.downcase)] + else + [ + "tsvector @@ (quote_literal(coalesce((ts_lexize('english_stem', :keyword))[1], :keyword)) || ':*')::tsquery", + { keyword: TextUtils.transliterate(keyword.downcase).squish } + ] + end + end + + def build_weights_tsquery(terms, weight) + last_query = Arel.sql(<<~SQL.squish) + (quote_literal(coalesce((ts_lexize('english_stem', :term#{terms.size - 1}))[1], :term#{terms.size - 1})) || ':*' || :weight)::tsquery + SQL + + query = terms[..-2].reduce(last_query) do |acc, term| + index = terms.index(term) + + arel = Arel.sql(<<~SQL.squish) + (quote_literal(coalesce((ts_lexize('english_stem', :term#{index}))[1], :term#{index})) || ':' || :weight)::tsquery + SQL + + Arel::Nodes::InfixOperation.new('&&', arel, acc) + end + + ["tsvector @@ (#{query.to_sql})", terms.index_by.with_index { |_, index| :"term#{index}" }.merge(weight:)] + end + + def index_submitter(submitter) + return if submitter.email.blank? && submitter.phone.blank? && submitter.name.blank? + + sql = SearchEntry.sanitize_sql_array( + [ + "SELECT setweight(to_tsvector(?), 'A') || setweight(to_tsvector(?), 'B') || + setweight(to_tsvector(?), 'C') || setweight(to_tsvector(?), 'D')".squish, + [submitter.email.to_s, submitter.email.to_s.split('@').last].join(' ').downcase, + [submitter.phone.to_s.gsub(/\D/, ''), + submitter.phone.to_s.gsub(PhoneCodes::REGEXP, '').gsub(/\D/, '')].uniq.join(' '), + TextUtils.transliterate(submitter.name.to_s.downcase), + build_submitter_values_string(submitter) + ] + ) + + entry = submitter.search_entry || submitter.build_search_entry + + entry.account_id = submitter.account_id + entry.tsvector = SearchEntry.connection.select_value(sql) + + return if entry.tsvector.blank? + + entry.save! + + entry + rescue ActiveRecord::RecordNotUnique + submitter.reload + + retry + end + + def build_submitter_values_string(submitter) + values = + submitter.values.values.flatten.filter_map do |v| + next if !v.is_a?(String) || v.length > MAX_VALUE_LENGTH || UUID_REGEXP.match?(v) + + TextUtils.transliterate(v) + end + + values.uniq.join(' ') + end + + def index_template(template) + sql = SearchEntry.sanitize_sql_array( + ['SELECT to_tsvector(?)', TextUtils.transliterate(template.name.to_s.downcase)] + ) + + entry = template.search_entry || template.build_search_entry + + entry.account_id = template.account_id + entry.tsvector = SearchEntry.connection.select_value(sql) + + return if entry.tsvector.blank? + + entry.save! + + entry + rescue ActiveRecord::RecordNotUnique + template.reload + + retry + end + + def index_submission(submission) + return if submission.name.blank? + + sql = SearchEntry.sanitize_sql_array( + ['SELECT to_tsvector(?)', TextUtils.transliterate(submission.name.to_s.downcase)] + ) + + entry = submission.search_entry || submission.build_search_entry + + entry.account_id = submission.account_id + entry.tsvector = SearchEntry.connection.select_value(sql) + + return if entry.tsvector.blank? + + entry.save! + + entry + rescue ActiveRecord::RecordNotUnique + submission.reload + + retry + end +end diff --git a/lib/submissions.rb b/lib/submissions.rb index 33bbab5c..e6e3d6ab 100644 --- a/lib/submissions.rb +++ b/lib/submissions.rb @@ -7,7 +7,15 @@ module Submissions module_function - def search(submissions, keyword, search_values: false, search_template: false) + def search(current_user, submissions, keyword, search_values: false, search_template: false) + if Docuseal.fulltext_search?(current_user) + fulltext_search(current_user, submissions, keyword, search_template:) + else + plain_search(submissions, keyword, search_values:, search_template:) + end + end + + def plain_search(submissions, keyword, search_values: false, search_template: false) return submissions if keyword.blank? term = "%#{keyword.downcase}%" @@ -29,6 +37,36 @@ module Submissions submissions.joins(:submitters).where(arel).group(:id) end + def fulltext_search(current_user, submissions, keyword, search_template: false) + return submissions if keyword.blank? + + arel = SearchEntry.where(record_type: 'Submission') + .where(account_id: current_user.account_id) + .where(*SearchEntries.build_tsquery(keyword)) + .select(:record_id).arel + + if search_template + arel = Arel::Nodes::Union.new( + arel, + Submission.where( + template_id: SearchEntry.where(record_type: 'Template') + .where(account_id: current_user.account_id) + .where(*SearchEntries.build_tsquery(keyword)) + .select(:record_id) + ).select(:id).arel + ) + end + + arel = Arel::Nodes::Union.new( + arel, Submitter.joins(:search_entry) + .where(search_entry: { account_id: current_user.account_id }) + .where(*SearchEntries.build_tsquery(keyword)) + .select(:submission_id).arel + ) + + submissions.where(Submission.arel_table[:id].in(arel)) + end + def update_template_fields!(submission) submission.template_fields = submission.template.fields submission.template_schema = submission.template.schema diff --git a/lib/submitters.rb b/lib/submitters.rb index bb467068..91ae28b5 100644 --- a/lib/submitters.rb +++ b/lib/submitters.rb @@ -4,9 +4,71 @@ module Submitters TRUE_VALUES = ['1', 'true', true].freeze PRELOAD_ALL_PAGES_AMOUNT = 200 + FIELD_NAME_WEIGHTS = { + 'email' => 'A', + 'phone' => 'B', + 'name' => 'C', + 'values' => 'D' + }.freeze + module_function - def search(submitters, keyword) + def search(current_user, submitters, keyword) + if Docuseal.fulltext_search?(current_user) + fulltext_search(current_user, submitters, keyword) + else + plain_search(submitters, keyword) + end + end + + def fulltext_search(current_user, submitters, keyword) + return submitters if keyword.blank? + + submitters.where( + id: SearchEntry.where(record_type: 'Submitter') + .where(account_id: current_user.account_id) + .where(*SearchEntries.build_tsquery(keyword)) + .select(:record_id) + ) + end + + def fulltext_search_field(current_user, submitters, keyword, field_name) + return submitters if keyword.blank? + + weight = FIELD_NAME_WEIGHTS[field_name] + + return submitters if weight.blank? + + query = + if keyword.match?(/\d/) && !keyword.match?(/\p{L}/) + number = keyword.gsub(/\D/, '') + + ["tsvector @@ ((quote_literal(?) || ':*#{weight}')::tsquery || (quote_literal(?) || ':*#{weight}')::tsquery)", + number, number.length > 1 ? number.delete_prefix('0') : number] + elsif keyword.match?(/[^\p{L}\d&@._\-+]/) + terms = TextUtils.transliterate(keyword.downcase).split(/\b/).map(&:squish).compact_blank.uniq + + if terms.size > 1 + SearchEntries.build_weights_tsquery(terms, weight) + else + [ + SearchEntries::FIELD_SEARCH_QUERY_SQL, + { keyword: TextUtils.transliterate(keyword.downcase).squish, weight: } + ] + end + else + [SearchEntries::FIELD_SEARCH_QUERY_SQL, { keyword: TextUtils.transliterate(keyword.downcase).squish, weight: }] + end + + submitters.where( + id: SearchEntry.where(record_type: 'Submitter') + .where(account_id: current_user.account_id) + .where(*query) + .select(:record_id) + ) + end + + def plain_search(submitters, keyword) return submitters if keyword.blank? term = "%#{keyword.downcase}%" diff --git a/lib/submitters/submit_values.rb b/lib/submitters/submit_values.rb index 16f9738a..f8eab084 100644 --- a/lib/submitters/submit_values.rb +++ b/lib/submitters/submit_values.rb @@ -48,6 +48,8 @@ module Submitters submitter.save! end + SearchEntries.enqueue_reindex(submitter) if submitter.completed_at? + submitter end diff --git a/lib/templates.rb b/lib/templates.rb index 953ef034..aa476493 100644 --- a/lib/templates.rb +++ b/lib/templates.rb @@ -37,12 +37,31 @@ module Templates hash end - def search(templates, keyword) + def search(current_user, templates, keyword) + if Docuseal.fulltext_search?(current_user) + fulltext_search(current_user, templates, keyword) + else + plain_search(templates, keyword) + end + end + + def plain_search(templates, keyword) return templates if keyword.blank? templates.where(Template.arel_table[:name].lower.matches("%#{keyword.downcase}%")) end + def fulltext_search(current_user, templates, keyword) + return templates if keyword.blank? + + templates.where( + id: SearchEntry.where(record_type: 'Template') + .where(account_id: current_user.account_id) + .where(*SearchEntries.build_tsquery(keyword)) + .select(:record_id) + ) + end + def filter_undefined_submitters(template_submitters) template_submitters.to_a.select do |item| item['invite_by_uuid'].blank? && item['optional_invite_by_uuid'].blank? && diff --git a/lib/text_utils.rb b/lib/text_utils.rb index f286311f..38be7f92 100644 --- a/lib/text_utils.rb +++ b/lib/text_utils.rb @@ -5,6 +5,11 @@ module TextUtils MASK_REGEXP = /[^\s\-_\[\]\(\)\+\?\.\,]/ MASK_SYMBOL = 'X' + TRANSLITERATIONS = + I18n::Backend::Transliterator::HashTransliterator::DEFAULT_APPROXIMATIONS.reject { |_, v| v.length > 1 } + + TRANSLITERATION_REGEXP = Regexp.union(TRANSLITERATIONS.keys) + module_function def rtl?(text) @@ -15,6 +20,10 @@ module TextUtils false end + def transliterate(text) + text.to_s.gsub(TRANSLITERATION_REGEXP) { |e| TRANSLITERATIONS[e] } + end + def mask_value(text, unmask_size = 0) if unmask_size.is_a?(Numeric) && !unmask_size.zero? && unmask_size.abs < text.length if unmask_size.negative?