mirror of https://github.com/docusealco/docuseal
Pete Matsyburka
5 months ago
parent
9d1860f038
commit
33811945be
@ -0,0 +1,13 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class ReindexSearchEntryJob
|
||||
include Sidekiq::Job
|
||||
|
||||
InvalidFormat = Class.new(StandardError)
|
||||
|
||||
def perform(params = {})
|
||||
entry = SearchEntry.find_or_initialize_by(params.slice('record_type', 'record_id'))
|
||||
|
||||
SearchEntries.reindex_record(entry.record)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,25 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
# == Schema Information
|
||||
#
|
||||
# Table name: search_entries
|
||||
#
|
||||
# id :bigint not null, primary key
|
||||
# record_type :string not null
|
||||
# tsvector :tsvector not null
|
||||
# created_at :datetime not null
|
||||
# updated_at :datetime not null
|
||||
# account_id :bigint not null
|
||||
# record_id :bigint not null
|
||||
#
|
||||
# Indexes
|
||||
#
|
||||
# index_search_entries_on_account_id_tsvector_submission (account_id,tsvector) WHERE ((record_type)::text = 'Submission'::text) USING gin
|
||||
# index_search_entries_on_account_id_tsvector_submitter (account_id,tsvector) WHERE ((record_type)::text = 'Submitter'::text) USING gin
|
||||
# index_search_entries_on_account_id_tsvector_template (account_id,tsvector) WHERE ((record_type)::text = 'Template'::text) USING gin
|
||||
# index_search_entries_on_record_id_and_record_type (record_id,record_type) UNIQUE
|
||||
#
|
||||
class SearchEntry < ApplicationRecord
|
||||
belongs_to :record, polymorphic: true
|
||||
belongs_to :account
|
||||
end
|
||||
@ -0,0 +1,33 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class CreateSearchEnties < ActiveRecord::Migration[8.0]
|
||||
def up
|
||||
return unless adapter_name == 'PostgreSQL'
|
||||
|
||||
enable_extension 'btree_gin'
|
||||
|
||||
create_table :search_entries do |t|
|
||||
t.references :record, null: false, polymorphic: true, index: false
|
||||
t.bigint :account_id, null: false
|
||||
t.tsvector :tsvector, null: false
|
||||
|
||||
t.timestamps
|
||||
end
|
||||
|
||||
add_index :search_entries, %i[account_id tsvector], using: :gin, where: "record_type = 'Submitter'",
|
||||
name: 'index_search_entries_on_account_id_tsvector_submitter'
|
||||
add_index :search_entries, %i[account_id tsvector], using: :gin, where: "record_type = 'Submission'",
|
||||
name: 'index_search_entries_on_account_id_tsvector_submission'
|
||||
add_index :search_entries, %i[account_id tsvector], using: :gin, where: "record_type = 'Template'",
|
||||
name: 'index_search_entries_on_account_id_tsvector_template'
|
||||
add_index :search_entries, %i[record_id record_type], unique: true
|
||||
end
|
||||
|
||||
def down
|
||||
return unless adapter_name == 'PostgreSQL'
|
||||
|
||||
drop_table :search_entries
|
||||
|
||||
disable_extension 'btree_gin'
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,228 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module PhoneCodes
|
||||
ALL = [
|
||||
'+1',
|
||||
'+93',
|
||||
'+358',
|
||||
'+355',
|
||||
'+213',
|
||||
'+1684',
|
||||
'+376',
|
||||
'+244',
|
||||
'+1264',
|
||||
'+1268',
|
||||
'+54',
|
||||
'+374',
|
||||
'+297',
|
||||
'+61',
|
||||
'+43',
|
||||
'+994',
|
||||
'+1242',
|
||||
'+973',
|
||||
'+880',
|
||||
'+1246',
|
||||
'+32',
|
||||
'+501',
|
||||
'+229',
|
||||
'+1441',
|
||||
'+975',
|
||||
'+591',
|
||||
'+387',
|
||||
'+267',
|
||||
'+55',
|
||||
'+246',
|
||||
'+673',
|
||||
'+359',
|
||||
'+226',
|
||||
'+257',
|
||||
'+855',
|
||||
'+237',
|
||||
'+1',
|
||||
'+238',
|
||||
'+1345',
|
||||
'+235',
|
||||
'+56',
|
||||
'+86',
|
||||
'+61',
|
||||
'+61',
|
||||
'+57',
|
||||
'+269',
|
||||
'+243',
|
||||
'+682',
|
||||
'+506',
|
||||
'+225',
|
||||
'+385',
|
||||
'+357',
|
||||
'+420',
|
||||
'+45',
|
||||
'+253',
|
||||
'+1767',
|
||||
'+1849',
|
||||
'+593',
|
||||
'+20',
|
||||
'+503',
|
||||
'+240',
|
||||
'+291',
|
||||
'+372',
|
||||
'+251',
|
||||
'+500',
|
||||
'+298',
|
||||
'+679',
|
||||
'+358',
|
||||
'+33',
|
||||
'+594',
|
||||
'+689',
|
||||
'+241',
|
||||
'+220',
|
||||
'+995',
|
||||
'+49',
|
||||
'+233',
|
||||
'+350',
|
||||
'+30',
|
||||
'+299',
|
||||
'+1473',
|
||||
'+590',
|
||||
'+1671',
|
||||
'+502',
|
||||
'+224',
|
||||
'+245',
|
||||
'+592',
|
||||
'+509',
|
||||
'+504',
|
||||
'+852',
|
||||
'+36',
|
||||
'+354',
|
||||
'+91',
|
||||
'+62',
|
||||
'+964',
|
||||
'+353',
|
||||
'+44',
|
||||
'+972',
|
||||
'+39',
|
||||
'+1876',
|
||||
'+81',
|
||||
'+44',
|
||||
'+962',
|
||||
'+7',
|
||||
'+254',
|
||||
'+686',
|
||||
'+82',
|
||||
'+965',
|
||||
'+996',
|
||||
'+856',
|
||||
'+371',
|
||||
'+961',
|
||||
'+266',
|
||||
'+231',
|
||||
'+423',
|
||||
'+370',
|
||||
'+352',
|
||||
'+853',
|
||||
'+389',
|
||||
'+261',
|
||||
'+265',
|
||||
'+60',
|
||||
'+960',
|
||||
'+223',
|
||||
'+356',
|
||||
'+692',
|
||||
'+596',
|
||||
'+222',
|
||||
'+230',
|
||||
'+262',
|
||||
'+52',
|
||||
'+691',
|
||||
'+373',
|
||||
'+377',
|
||||
'+976',
|
||||
'+382',
|
||||
'+1664',
|
||||
'+212',
|
||||
'+258',
|
||||
'+264',
|
||||
'+674',
|
||||
'+977',
|
||||
'+31',
|
||||
'+687',
|
||||
'+64',
|
||||
'+227',
|
||||
'+234',
|
||||
'+683',
|
||||
'+672',
|
||||
'+1670',
|
||||
'+47',
|
||||
'+968',
|
||||
'+92',
|
||||
'+680',
|
||||
'+507',
|
||||
'+675',
|
||||
'+595',
|
||||
'+51',
|
||||
'+63',
|
||||
'+872',
|
||||
'+48',
|
||||
'+351',
|
||||
'+1939',
|
||||
'+974',
|
||||
'+40',
|
||||
'+250',
|
||||
'+262',
|
||||
'+590',
|
||||
'+290',
|
||||
'+1869',
|
||||
'+1758',
|
||||
'+590',
|
||||
'+508',
|
||||
'+1784',
|
||||
'+685',
|
||||
'+378',
|
||||
'+239',
|
||||
'+966',
|
||||
'+221',
|
||||
'+381',
|
||||
'+248',
|
||||
'+232',
|
||||
'+65',
|
||||
'+421',
|
||||
'+386',
|
||||
'+677',
|
||||
'+27',
|
||||
'+34',
|
||||
'+94',
|
||||
'+597',
|
||||
'+47',
|
||||
'+268',
|
||||
'+46',
|
||||
'+41',
|
||||
'+886',
|
||||
'+992',
|
||||
'+255',
|
||||
'+66',
|
||||
'+670',
|
||||
'+228',
|
||||
'+690',
|
||||
'+676',
|
||||
'+1868',
|
||||
'+216',
|
||||
'+90',
|
||||
'+993',
|
||||
'+1649',
|
||||
'+688',
|
||||
'+256',
|
||||
'+380',
|
||||
'+971',
|
||||
'+44',
|
||||
'+598',
|
||||
'+998',
|
||||
'+678',
|
||||
'+84',
|
||||
'+1284',
|
||||
'+1340',
|
||||
'+681',
|
||||
'+967',
|
||||
'+260'
|
||||
].freeze
|
||||
|
||||
REGEXP = /\A#{Regexp.union(ALL).source}/i
|
||||
end
|
||||
@ -0,0 +1,167 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module SearchEntries
|
||||
TRANSLITERATIONS =
|
||||
I18n::Backend::Transliterator::HashTransliterator::DEFAULT_APPROXIMATIONS.reject { |_, v| v.length > 1 }
|
||||
|
||||
MAX_VALUE_LENGTH = 100
|
||||
|
||||
UUID_REGEXP = /\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i
|
||||
|
||||
FIELD_SEARCH_QUERY_SQL = <<~SQL.squish
|
||||
tsvector @@ (quote_literal(coalesce((ts_lexize('english_stem', :keyword))[1], :keyword)) || ':*' || :weight)::tsquery
|
||||
SQL
|
||||
|
||||
module_function
|
||||
|
||||
def reindex_all
|
||||
Submitter.find_each { |submitter| index_submitter(submitter) }
|
||||
Submission.find_each { |submission| index_submission(submission) }
|
||||
Template.find_each { |template| index_template(template) }
|
||||
end
|
||||
|
||||
def enqueue_reindex(records)
|
||||
return unless SearchEntry.table_exists?
|
||||
|
||||
args = Array.wrap(records).map { |e| [{ 'record_type' => e.class.name, 'record_id' => e.id }] }
|
||||
|
||||
ReindexSearchEntryJob.perform_bulk(args)
|
||||
end
|
||||
|
||||
def reindex_record(record)
|
||||
case record
|
||||
when Submitter
|
||||
index_submitter(record)
|
||||
when Template
|
||||
index_template(record)
|
||||
when Submission
|
||||
index_submission(record)
|
||||
|
||||
record.submitters.each do |submitter|
|
||||
index_submitter(submitter)
|
||||
end
|
||||
else
|
||||
raise ArgumentError, 'Invalid Record'
|
||||
end
|
||||
end
|
||||
|
||||
def build_tsquery(keyword)
|
||||
if keyword.match?(/\d/) && !keyword.match?(/\p{L}/)
|
||||
number = keyword.gsub(/\D/, '')
|
||||
|
||||
["tsvector @@ ((quote_literal(?) || ':*')::tsquery || (quote_literal(?) || ':*')::tsquery || plainto_tsquery(?))",
|
||||
number, number.length > 1 ? number.delete_prefix('0') : number, keyword]
|
||||
elsif keyword.match?(/[^\p{L}\d&@._\-+]/) || keyword.match?(/\A['"].*['"]\z/)
|
||||
['tsvector @@ plainto_tsquery(?)', TextUtils.transliterate(keyword.downcase)]
|
||||
else
|
||||
[
|
||||
"tsvector @@ (quote_literal(coalesce((ts_lexize('english_stem', :keyword))[1], :keyword)) || ':*')::tsquery",
|
||||
{ keyword: TextUtils.transliterate(keyword.downcase).squish }
|
||||
]
|
||||
end
|
||||
end
|
||||
|
||||
def build_weights_tsquery(terms, weight)
|
||||
last_query = Arel.sql(<<~SQL.squish)
|
||||
(quote_literal(coalesce((ts_lexize('english_stem', :term#{terms.size - 1}))[1], :term#{terms.size - 1})) || ':*' || :weight)::tsquery
|
||||
SQL
|
||||
|
||||
query = terms[..-2].reduce(last_query) do |acc, term|
|
||||
index = terms.index(term)
|
||||
|
||||
arel = Arel.sql(<<~SQL.squish)
|
||||
(quote_literal(coalesce((ts_lexize('english_stem', :term#{index}))[1], :term#{index})) || ':' || :weight)::tsquery
|
||||
SQL
|
||||
|
||||
Arel::Nodes::InfixOperation.new('&&', arel, acc)
|
||||
end
|
||||
|
||||
["tsvector @@ (#{query.to_sql})", terms.index_by.with_index { |_, index| :"term#{index}" }.merge(weight:)]
|
||||
end
|
||||
|
||||
def index_submitter(submitter)
|
||||
return if submitter.email.blank? && submitter.phone.blank? && submitter.name.blank?
|
||||
|
||||
sql = SearchEntry.sanitize_sql_array(
|
||||
[
|
||||
"SELECT setweight(to_tsvector(?), 'A') || setweight(to_tsvector(?), 'B') ||
|
||||
setweight(to_tsvector(?), 'C') || setweight(to_tsvector(?), 'D')".squish,
|
||||
[submitter.email.to_s, submitter.email.to_s.split('@').last].join(' ').downcase,
|
||||
[submitter.phone.to_s.gsub(/\D/, ''),
|
||||
submitter.phone.to_s.gsub(PhoneCodes::REGEXP, '').gsub(/\D/, '')].uniq.join(' '),
|
||||
TextUtils.transliterate(submitter.name.to_s.downcase),
|
||||
build_submitter_values_string(submitter)
|
||||
]
|
||||
)
|
||||
|
||||
entry = submitter.search_entry || submitter.build_search_entry
|
||||
|
||||
entry.account_id = submitter.account_id
|
||||
entry.tsvector = SearchEntry.connection.select_value(sql)
|
||||
|
||||
return if entry.tsvector.blank?
|
||||
|
||||
entry.save!
|
||||
|
||||
entry
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
submitter.reload
|
||||
|
||||
retry
|
||||
end
|
||||
|
||||
def build_submitter_values_string(submitter)
|
||||
values =
|
||||
submitter.values.values.flatten.filter_map do |v|
|
||||
next if !v.is_a?(String) || v.length > MAX_VALUE_LENGTH || UUID_REGEXP.match?(v)
|
||||
|
||||
TextUtils.transliterate(v)
|
||||
end
|
||||
|
||||
values.uniq.join(' ')
|
||||
end
|
||||
|
||||
def index_template(template)
|
||||
sql = SearchEntry.sanitize_sql_array(
|
||||
['SELECT to_tsvector(?)', TextUtils.transliterate(template.name.to_s.downcase)]
|
||||
)
|
||||
|
||||
entry = template.search_entry || template.build_search_entry
|
||||
|
||||
entry.account_id = template.account_id
|
||||
entry.tsvector = SearchEntry.connection.select_value(sql)
|
||||
|
||||
return if entry.tsvector.blank?
|
||||
|
||||
entry.save!
|
||||
|
||||
entry
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
template.reload
|
||||
|
||||
retry
|
||||
end
|
||||
|
||||
def index_submission(submission)
|
||||
return if submission.name.blank?
|
||||
|
||||
sql = SearchEntry.sanitize_sql_array(
|
||||
['SELECT to_tsvector(?)', TextUtils.transliterate(submission.name.to_s.downcase)]
|
||||
)
|
||||
|
||||
entry = submission.search_entry || submission.build_search_entry
|
||||
|
||||
entry.account_id = submission.account_id
|
||||
entry.tsvector = SearchEntry.connection.select_value(sql)
|
||||
|
||||
return if entry.tsvector.blank?
|
||||
|
||||
entry.save!
|
||||
|
||||
entry
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
submission.reload
|
||||
|
||||
retry
|
||||
end
|
||||
end
|
||||
Loading…
Reference in new issue