block search bots

app/controllers/api/active_storage_blobs_proxy_controller.rb

@@ -34,6 +34,12 @@ module Api
       @record = attachment.record
       @record = @record.record if @record.is_a?(ActiveStorage::Attachment)
 
+      if DetectBrowserDevice.search_bot?(request.user_agent)
+        Rollbar.warning("Search bot: #{@record.class.name} #{@record.id}") if defined?(Rollbar)
+
+        raise ActionController::RoutingError, 'Not Found'
+      end
+
       authorization_check!(attachment, @record, exp)
 
       if request.headers['Range'].present?

app/controllers/submit_form_controller.rb

@@ -8,6 +8,7 @@ class SubmitFormController < ApplicationController
   skip_authorization_check
 
   before_action :load_submitter, only: %i[show update completed]
+  before_action :maybe_render_not_found_for_search_bot, only: %i[show completed]
   before_action :maybe_redirect_delegated, only: %i[show completed]
   before_action :maybe_render_locked_page, only: :show
   before_action :maybe_require_link_2fa, only: %i[show]
@@ -111,6 +112,14 @@ class SubmitFormController < ApplicationController
 
   private
 
+  def maybe_render_not_found_for_search_bot
+    return unless DetectBrowserDevice.search_bot?(request.user_agent)
+
+    Rollbar.warning("Search bot: submitter #{@submitter&.id}") if defined?(Rollbar)
+
+    raise ActionController::RoutingError, I18n.t('not_found')
+  end
+
   def maybe_require_link_2fa
     return if Submitters::AuthorizedForForm.pass_link_2fa?(@submitter, current_user, request)
 

lib/detect_browser_device.rb

@@ -63,6 +63,42 @@ module DetectBrowserDevice
     NetBSD
   /ix
 
+  SEARCH_BOT_USER_AGENT_REGEXP = /
+    Googlebot            |
+    Google-InspectionTool|
+    Storebot-Google      |
+    AdsBot-Google        |
+    Mediapartners-Google |
+    bingbot              |
+    BingPreview          |
+    adidxbot             |
+    msnbot               |
+    Slurp                |
+    DuckDuckBot          |
+    Baiduspider          |
+    YandexBot            |
+    YandexImages         |
+    Applebot             |
+    PetalBot             |
+    SeznamBot            |
+    Sogou                |
+    Exabot               |
+    Qwantify             |
+    YisouSpider          |
+    Bytespider           |
+    AhrefsBot            |
+    SemrushBot           |
+    MJ12bot              |
+    DotBot               |
+    GPTBot               |
+    OAI-SearchBot        |
+    ClaudeBot            |
+    Claude-SearchBot     |
+    PerplexityBot        |
+    CCBot                |
+    Amazonbot
+  /ix
+
   SDK_USER_AGENT_REGEXP = /\ADocuSeal (?<sdk>Ruby|Python|PHP|Java|C#|JS|Go|CLI) v/i
 
   def call(user_agent)
@@ -74,6 +110,10 @@ module DetectBrowserDevice
     'desktop'
   end
 
+  def search_bot?(user_agent)
+    SEARCH_BOT_USER_AGENT_REGEXP.match?(user_agent.to_s)
+  end
+
   def os(user_agent)
     return if user_agent.blank?