From 31acb611ab3392479e4419bc0e1ef3aa2beb2099 Mon Sep 17 00:00:00 2001 From: Gaurav agarwal Date: Thu, 11 Jun 2026 18:12:14 +0530 Subject: [PATCH 1/4] Upgrade to Ruby 3.4 / Rails 8.1 and ship Wave 0 SEO fixes. Unblock search crawlers via Rack::Attack throttles, add canonical URLs and JSON-LD, fix sitemap host/caching, and improve local dev (proxy auth, routes, vendored bitquery_logger). Co-authored-by: Cursor --- .ruby-version | 2 +- Dockerfile | 4 +- Gemfile | 15 +- Gemfile.lock | 460 ++++++++++-------- Rakefile | 1 + app/controllers/application_controller.rb | 53 +- .../proxy_eap_graphql_controller.rb | 2 +- app/controllers/proxy_graphql_controller.rb | 2 +- .../proxy_streaming_graphql_controller.rb | 2 +- app/helpers/seo_helper.rb | 66 +++ app/services/streaming_token_service.rb | 11 +- app/views/shared/_footer.html.erb | 6 - app/views/shared/_head.html.erb | 1 + app/views/shared/_seo_meta.html.erb | 16 + app/views/sitemaps/robots.text.erb | 2 +- app/views/utility/errors.html.erb | 8 +- bin/dev-server | 21 + config/application.rb | 2 +- config/environments/production.rb | 6 +- config/initializers/bitquery_logger.rb | 2 - config/initializers/canonical_host.rb | 8 + config/initializers/rack_attack.rb | 69 +-- config/routes.rb | 9 + lib/cached_query.rb | 32 ++ package.json | 2 +- ...ry_logger-233e44a43d4c => bitquery_logger} | 0 yarn.lock | 8 +- 27 files changed, 541 insertions(+), 269 deletions(-) create mode 100644 app/helpers/seo_helper.rb create mode 100644 app/views/shared/_seo_meta.html.erb create mode 100755 bin/dev-server create mode 100644 config/initializers/canonical_host.rb create mode 100644 lib/cached_query.rb rename vendor/{bundle/ruby/3.3.0/bundler/gems/bitquery_logger-233e44a43d4c => bitquery_logger} (100%) diff --git a/.ruby-version b/.ruby-version index 6d5369b9..61a52c91 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -ruby-3.3.4 +ruby-3.4.9 diff --git a/Dockerfile b/Dockerfile index 7355509c..54f4c819 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ruby:3.3.4-alpine AS builder +FROM ruby:3.4.9-alpine AS builder WORKDIR /app ENV RAILS_ENV="production" \ @@ -32,7 +32,7 @@ RUN SECRET_KEY_BASE_DUMMY=1 bundle exec rails shakapacker:compile && \ RUN yarn install --production && \ yarn cache clean -FROM ruby:3.3.4-alpine AS runner +FROM ruby:3.4.9-alpine AS runner WORKDIR /app diff --git a/Gemfile b/Gemfile index d123f25e..4cb5b6a9 100644 --- a/Gemfile +++ b/Gemfile @@ -1,14 +1,14 @@ source 'https://rubygems.org' git_source(:github) { |repo| "https://github.com/#{repo}.git" } -ruby '~> 3.3.4' +ruby '~> 3.4.0' # gem 'turbo-rails', '~> 2.0.6' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '~> 7.1.3.4' +gem 'rails', '~> 8.1.0' # Use Puma as the app server -gem 'puma', '~> 6.4.3' +gem 'puma', '~> 7.0' # Use SCSS for stylesheets gem 'sass-rails', '>= 6' # Transpile app-like JavaScript. Read more: https://github.com/rails/webpacker @@ -17,8 +17,7 @@ gem 'shakapacker', '~> 8.0.1' # gem 'turbolinks', '~> 5' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder gem 'jbuilder', '~> 2.12.0' -# Use Redis adapter to run Action Cable in production -# gem 'redis', '~> 4.0' +gem 'redis', '~> 5.0' # Use Active Model has_secure_password # gem 'bcrypt', '~> 3.1.7' @@ -31,14 +30,14 @@ gem 'feedjira', '~> 3.2.3' gem 'rack-attack' # Logger -gem 'bitquery_logger', git: 'https://github.com/bitquery/bitquery_logger.git', tag: 'v0.7.9' # , path: '../bitquery_logger' -gem 'exception_notification' -gem 'exception_notification-rake', '~> 0.3.1' +gem 'bitquery_logger', path: 'vendor/bitquery_logger' +gem 'exception_notification', '~> 5.0' gem 'lograge' gem 'logstash-event' # Reduces boot times through caching; required in config/boot.rb gem 'bootsnap', '~> 1.18.3', require: false +gem 'csv' group :development do # Access an interactive console on exception pages or by calling 'console' anywhere in the code. diff --git a/Gemfile.lock b/Gemfile.lock index 34536d18..1365ad0a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,11 +1,8 @@ -GIT - remote: https://github.com/bitquery/bitquery_logger.git - revision: 233e44a43d4c10881197b16dc773a513ab0ae7aa - tag: v0.7.9 +PATH + remote: vendor/bitquery_logger specs: bitquery_logger (0.7.9) - exception_notification (~> 4.5.0) - exception_notification-rake (~> 0.3.1) + exception_notification (>= 4.5.0, < 6) logstash-logger (~> 0.26.1) oj (~> 3.13.16) oj_mimic_json (~> 1.0.1) @@ -13,164 +10,170 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (7.1.3.4) - actionpack (= 7.1.3.4) - activesupport (= 7.1.3.4) + action_text-trix (2.1.19) + railties + actioncable (8.1.3) + actionpack (= 8.1.3) + activesupport (= 8.1.3) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.1.3.4) - actionpack (= 7.1.3.4) - activejob (= 7.1.3.4) - activerecord (= 7.1.3.4) - activestorage (= 7.1.3.4) - activesupport (= 7.1.3.4) - mail (>= 2.7.1) - net-imap - net-pop - net-smtp - actionmailer (7.1.3.4) - actionpack (= 7.1.3.4) - actionview (= 7.1.3.4) - activejob (= 7.1.3.4) - activesupport (= 7.1.3.4) - mail (~> 2.5, >= 2.5.4) - net-imap - net-pop - net-smtp + actionmailbox (8.1.3) + actionpack (= 8.1.3) + activejob (= 8.1.3) + activerecord (= 8.1.3) + activestorage (= 8.1.3) + activesupport (= 8.1.3) + mail (>= 2.8.0) + actionmailer (8.1.3) + actionpack (= 8.1.3) + actionview (= 8.1.3) + activejob (= 8.1.3) + activesupport (= 8.1.3) + mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.1.3.4) - actionview (= 7.1.3.4) - activesupport (= 7.1.3.4) + actionpack (8.1.3) + actionview (= 8.1.3) + activesupport (= 8.1.3) nokogiri (>= 1.8.5) - racc rack (>= 2.2.4) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actiontext (7.1.3.4) - actionpack (= 7.1.3.4) - activerecord (= 7.1.3.4) - activestorage (= 7.1.3.4) - activesupport (= 7.1.3.4) + useragent (~> 0.16) + actiontext (8.1.3) + action_text-trix (~> 2.1.15) + actionpack (= 8.1.3) + activerecord (= 8.1.3) + activestorage (= 8.1.3) + activesupport (= 8.1.3) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.1.3.4) - activesupport (= 7.1.3.4) + actionview (8.1.3) + activesupport (= 8.1.3) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.3.4) - activesupport (= 7.1.3.4) + activejob (8.1.3) + activesupport (= 8.1.3) globalid (>= 0.3.6) - activemodel (7.1.3.4) - activesupport (= 7.1.3.4) - activerecord (7.1.3.4) - activemodel (= 7.1.3.4) - activesupport (= 7.1.3.4) + activemodel (8.1.3) + activesupport (= 8.1.3) + activerecord (8.1.3) + activemodel (= 8.1.3) + activesupport (= 8.1.3) timeout (>= 0.4.0) - activestorage (7.1.3.4) - actionpack (= 7.1.3.4) - activejob (= 7.1.3.4) - activerecord (= 7.1.3.4) - activesupport (= 7.1.3.4) + activestorage (8.1.3) + actionpack (= 8.1.3) + activejob (= 8.1.3) + activerecord (= 8.1.3) + activesupport (= 8.1.3) marcel (~> 1.0) - activesupport (7.1.3.4) + activesupport (8.1.3) base64 bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + json + logger (>= 1.4.2) minitest (>= 5.1) - mutex_m - tzinfo (~> 2.0) - airbrussh (1.5.2) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) + airbrussh (1.6.1) sshkit (>= 1.6.1, != 1.7.0) - anyway_config (2.6.4) + anyway_config (2.8.0) ruby-next-core (~> 1.0) - ast (2.4.2) - base64 (0.2.0) + ast (2.4.3) + base64 (0.3.0) better_errors (2.10.1) erubi (>= 1.0.0) rack (>= 0.9.0) rouge (>= 1.0.0) - bigdecimal (3.1.8) + bigdecimal (4.1.2) bindex (0.8.1) binding_of_caller (1.0.1) debug_inspector (>= 1.2.0) - bootsnap (1.18.3) + bootsnap (1.18.6) msgpack (~> 1.2) builder (3.3.0) - capistrano (3.19.1) + capistrano (3.19.2) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) sshkit (>= 1.9.0) - capistrano-bundler (2.1.0) + capistrano-bundler (2.2.0) capistrano (~> 3.1) - capistrano-rails (1.6.3) + capistrano-rails (1.7.0) capistrano (~> 3.1) capistrano-bundler (>= 1.1, < 3) - concurrent-ruby (1.3.3) - connection_pool (2.4.1) + concurrent-ruby (1.3.6) + connection_pool (3.0.2) crass (1.0.6) - date (3.3.4) + csv (3.3.5) + date (3.5.1) debug_inspector (1.2.0) - dotenv (3.1.2) - dotenv-rails (3.1.2) - dotenv (= 3.1.2) + dotenv (3.1.8) + dotenv-rails (3.1.8) + dotenv (= 3.1.8) railties (>= 6.1) - drb (2.2.1) - dry-initializer (3.1.1) - erubi (1.13.0) + drb (2.2.3) + dry-initializer (3.2.0) + erb (6.0.4) + erubi (1.13.1) eventmachine (1.2.7) - exception_notification (4.5.0) - actionmailer (>= 5.2, < 8) - activesupport (>= 5.2, < 8) - exception_notification-rake (0.3.1) - exception_notification (~> 4.3) - rake (>= 0.9.0) + exception_notification (5.0.1) + actionmailer (>= 7.1, < 9) + activesupport (>= 7.1, < 9) faye-websocket (0.12.0) eventmachine (>= 0.12.0) websocket-driver (>= 0.8.0) - feedjira (3.2.3) + feedjira (3.2.6) + logger (>= 1.0, < 2) loofah (>= 2.3.1, < 3) sax-machine (>= 1.0, < 2) - ffi (1.17.0) - ffi (1.17.0-aarch64-linux-gnu) - ffi (1.17.0-aarch64-linux-musl) - ffi (1.17.0-arm-linux-gnu) - ffi (1.17.0-arm-linux-musl) - ffi (1.17.0-arm64-darwin) - ffi (1.17.0-x86-linux-gnu) - ffi (1.17.0-x86-linux-musl) - ffi (1.17.0-x86_64-darwin) - ffi (1.17.0-x86_64-linux-gnu) - ffi (1.17.0-x86_64-linux-musl) - globalid (1.2.1) + ffi (1.17.4) + ffi (1.17.4-aarch64-linux-gnu) + ffi (1.17.4-aarch64-linux-musl) + ffi (1.17.4-arm-linux-gnu) + ffi (1.17.4-arm-linux-musl) + ffi (1.17.4-arm64-darwin) + ffi (1.17.4-x86-linux-gnu) + ffi (1.17.4-x86-linux-musl) + ffi (1.17.4-x86_64-darwin) + ffi (1.17.4-x86_64-linux-gnu) + ffi (1.17.4-x86_64-linux-musl) + fiber-storage (1.0.1) + globalid (1.3.0) activesupport (>= 6.1) - graphql (2.3.10) + graphql (2.6.3) base64 - graphql-client (0.23.0) + fiber-storage + logger + graphql-client (0.26.0) activesupport (>= 3.0) graphql (>= 1.13.0) - i18n (1.14.5) + i18n (1.14.8) concurrent-ruby (~> 1.0) - io-console (0.7.2) - irb (1.14.0) + io-console (0.8.2) + irb (1.18.0) + pp (>= 0.6.0) + prism (>= 1.3.0) rdoc (>= 4.0.0) reline (>= 0.4.2) jbuilder (2.12.0) actionview (>= 5.0.0) activesupport (>= 5.0.0) - json (2.7.2) - language_server-protocol (3.17.0.3) + json (2.19.8) + language_server-protocol (3.17.0.5) listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) + logger (1.7.0) lograge (0.14.0) actionpack (>= 4) activesupport (>= 4) @@ -179,129 +182,175 @@ GEM logstash-event (1.2.02) logstash-logger (0.26.1) logstash-event (~> 1.2) - loofah (2.22.0) + loofah (2.25.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) - mail (2.8.1) + mail (2.9.0) + logger mini_mime (>= 0.1.1) net-imap net-pop net-smtp - marcel (1.0.4) + marcel (1.2.1) mini_mime (1.1.5) mini_portile2 (2.8.9) - minitest (5.24.1) - msgpack (1.7.2) - mutex_m (0.2.0) - net-imap (0.4.14) + minitest (6.0.6) + drb (~> 2.0) + prism (~> 1.5) + msgpack (1.8.3) + net-imap (0.6.4.1) date net-protocol net-pop (0.1.2) net-protocol net-protocol (0.2.2) timeout - net-scp (4.0.0) + net-scp (4.1.0) net-ssh (>= 2.6.5, < 8.0.0) net-sftp (4.0.0) net-ssh (>= 5.0.0, < 8.0.0) - net-smtp (0.5.0) + net-smtp (0.5.1) net-protocol - net-ssh (7.2.3) - nio4r (2.7.3) - nokogiri (1.16.6) + net-ssh (7.3.2) + nio4r (2.7.5) + nokogiri (1.19.3) mini_portile2 (~> 2.8.2) racc (~> 1.4) - nokogiri (1.16.6-aarch64-linux) + nokogiri (1.19.3-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.3-aarch64-linux-musl) racc (~> 1.4) - nokogiri (1.16.6-arm-linux) + nokogiri (1.19.3-arm-linux-gnu) racc (~> 1.4) - nokogiri (1.16.6-arm64-darwin) + nokogiri (1.19.3-arm-linux-musl) racc (~> 1.4) - nokogiri (1.16.6-x86-linux) + nokogiri (1.19.3-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.6-x86_64-darwin) + nokogiri (1.19.3-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.6-x86_64-linux) + nokogiri (1.19.3-x86_64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.3-x86_64-linux-musl) racc (~> 1.4) oj (3.13.23) oj_mimic_json (1.0.1) - package_json (0.1.0) - parallel (1.25.1) - parser (3.3.4.0) + ostruct (0.6.3) + package_json (0.2.0) + parallel (1.28.0) + parser (3.3.11.1) ast (~> 2.4.1) racc - prometheus-client-mmap (1.1.1) - rb_sys (~> 0.9.86) - prometheus-client-mmap (1.1.1-aarch64-linux) - rb_sys (~> 0.9.86) - prometheus-client-mmap (1.1.1-arm64-darwin) - rb_sys (~> 0.9.86) - prometheus-client-mmap (1.1.1-x86_64-darwin) - rb_sys (~> 0.9.86) - prometheus-client-mmap (1.1.1-x86_64-linux) - rb_sys (~> 0.9.86) - psych (5.1.2) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.9.0) + prometheus-client-mmap (1.5.0) + base64 + bigdecimal + logger + rb_sys (~> 0.9.124) + prometheus-client-mmap (1.5.0-aarch64-linux-gnu) + base64 + bigdecimal + logger + rb_sys (~> 0.9.124) + prometheus-client-mmap (1.5.0-aarch64-linux-musl) + base64 + bigdecimal + logger + rb_sys (~> 0.9.124) + prometheus-client-mmap (1.5.0-arm64-darwin) + base64 + bigdecimal + logger + rb_sys (~> 0.9.124) + prometheus-client-mmap (1.5.0-x86_64-darwin) + base64 + bigdecimal + logger + rb_sys (~> 0.9.124) + prometheus-client-mmap (1.5.0-x86_64-linux-gnu) + base64 + bigdecimal + logger + rb_sys (~> 0.9.124) + prometheus-client-mmap (1.5.0-x86_64-linux-musl) + base64 + bigdecimal + logger + rb_sys (~> 0.9.124) + psych (5.4.0) + date stringio - puma (6.4.3) + puma (7.2.1) nio4r (~> 2.0) - racc (1.8.0) - rack (2.2.9) - rack-attack (6.7.0) + racc (1.8.1) + rack (3.2.6) + rack-attack (6.8.0) rack (>= 1.0, < 4) - rack-proxy (0.7.7) + rack-proxy (0.8.2) rack - rack-session (1.0.2) - rack (< 3) - rack-test (2.1.0) + rack-session (2.1.2) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) rack (>= 1.3) - rackup (1.0.0) - rack (< 3) - webrick - rails (7.1.3.4) - actioncable (= 7.1.3.4) - actionmailbox (= 7.1.3.4) - actionmailer (= 7.1.3.4) - actionpack (= 7.1.3.4) - actiontext (= 7.1.3.4) - actionview (= 7.1.3.4) - activejob (= 7.1.3.4) - activemodel (= 7.1.3.4) - activerecord (= 7.1.3.4) - activestorage (= 7.1.3.4) - activesupport (= 7.1.3.4) + rackup (2.3.1) + rack (>= 3) + rails (8.1.3) + actioncable (= 8.1.3) + actionmailbox (= 8.1.3) + actionmailer (= 8.1.3) + actionpack (= 8.1.3) + actiontext (= 8.1.3) + actionview (= 8.1.3) + activejob (= 8.1.3) + activemodel (= 8.1.3) + activerecord (= 8.1.3) + activestorage (= 8.1.3) + activesupport (= 8.1.3) bundler (>= 1.15.0) - railties (= 7.1.3.4) - rails-dom-testing (2.2.0) + railties (= 8.1.3) + rails-dom-testing (2.3.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.0) - loofah (~> 2.21) - nokogiri (~> 1.14) - railties (7.1.3.4) - actionpack (= 7.1.3.4) - activesupport (= 7.1.3.4) - irb + rails-html-sanitizer (1.7.0) + loofah (~> 2.25) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (8.1.3) + actionpack (= 8.1.3) + activesupport (= 8.1.3) + irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) + tsort (>= 0.2) zeitwerk (~> 2.6) rainbow (3.1.1) - rake (13.2.1) + rake (13.4.2) + rake-compiler-dock (1.12.0) rb-fsevent (0.11.2) rb-inotify (0.11.1) ffi (~> 1.0) - rb_sys (0.9.99) - rdoc (6.7.0) + rb_sys (0.9.128) + rake-compiler-dock (= 1.12.0) + rdoc (7.2.0) + erb psych (>= 4.0.0) - regexp_parser (2.9.2) - reline (0.5.9) + tsort + redis (5.4.1) + redis-client (>= 0.22.0) + redis-client (0.29.0) + connection_pool + regexp_parser (2.12.0) + reline (0.6.3) io-console (~> 0.5) request_store (1.7.0) rack (>= 1.4) - rexml (3.3.6) - strscan - rouge (4.3.0) + rexml (3.4.4) + rouge (5.0.0) + strscan (~> 3.1) rubocop (1.64.1) json (~> 2.3) language_server-protocol (>= 3.17.0) @@ -313,8 +362,9 @@ GEM rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.3) - parser (>= 3.3.1.0) + rubocop-ast (1.49.1) + parser (>= 3.3.7.2) + prism (~> 1.7) rubocop-performance (1.21.1) rubocop (>= 1.48.1, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) @@ -323,7 +373,7 @@ GEM rack (>= 1.1) rubocop (>= 1.33.0, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - ruby-next-core (1.0.3) + ruby-next-core (1.2.0) ruby-progressbar (1.13.0) sass-rails (6.0.0) sassc-rails (~> 2.1, >= 2.1.1) @@ -336,8 +386,9 @@ GEM sprockets-rails tilt sax-machine (1.3.2) - semantic_range (3.0.0) - shakapacker (8.0.1) + securerandom (0.4.1) + semantic_range (3.1.1) + shakapacker (8.0.2) activesupport (>= 5.2) package_json rack-proxy (>= 0.6.1) @@ -350,63 +401,69 @@ GEM spring-watcher-listen (2.1.0) listen (>= 2.7, < 4.0) spring (>= 4) - sprockets (4.2.1) + sprockets (4.2.2) concurrent-ruby (~> 1.0) + logger rack (>= 2.2.4, < 4) - sprockets-rails (3.5.1) + sprockets-rails (3.5.2) actionpack (>= 6.1) activesupport (>= 6.1) sprockets (>= 3.0.0) - sshkit (1.23.0) + sshkit (1.25.0) base64 + logger net-scp (>= 1.1.2) net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) + ostruct sshkit-sudo (0.1.0) sshkit (~> 1.8) - stringio (3.1.1) - strscan (3.1.0) - thor (1.3.1) - tilt (2.4.0) - timeout (0.4.1) + stringio (3.2.0) + strscan (3.1.8) + thor (1.5.0) + tilt (2.7.0) + timeout (0.6.1) + tsort (0.2.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - tzinfo-data (1.2024.1) + tzinfo-data (1.2026.2) tzinfo (>= 1.0.0) - unicode-display_width (2.5.0) + unicode-display_width (2.6.0) + uri (1.1.1) + useragent (0.16.11) web-console (4.2.1) actionview (>= 6.0.0) activemodel (>= 6.0.0) bindex (>= 0.4.0) railties (>= 6.0.0) - webrick (1.8.1) - websocket-driver (0.8.0) + websocket-driver (0.8.1) base64 websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - yabeda (0.12.0) + yabeda (0.16.0) anyway_config (>= 1.0, < 3) concurrent-ruby dry-initializer - yabeda-graphql (0.2.3) - graphql (>= 1.9, < 3) + yabeda-graphql (0.3.0) + graphql (~> 2.2) yabeda (~> 0.2) - yabeda-http_requests (0.2.1) + yabeda-http_requests (0.3.0) + anyway_config (>= 1.3, < 3.0) sniffer yabeda yabeda-prometheus-mmap (0.4.0) prometheus-client-mmap yabeda (~> 0.10) - yabeda-puma-plugin (0.7.1) + yabeda-puma-plugin (0.9.0) json puma yabeda (~> 0.5) - yabeda-rails (0.9.0) + yabeda-rails (0.11.0) activesupport anyway_config (>= 1.3, < 3) railties yabeda (~> 0.8) - zeitwerk (2.6.16) + zeitwerk (2.8.2) PLATFORMS aarch64-linux @@ -429,9 +486,9 @@ DEPENDENCIES capistrano (~> 3.19.0) capistrano-bundler capistrano-rails + csv dotenv-rails (~> 3.1.2) - exception_notification - exception_notification-rake (~> 0.3.1) + exception_notification (~> 5.0) faye-websocket (~> 0.12.0) feedjira (~> 3.2.3) graphql-client @@ -439,9 +496,10 @@ DEPENDENCIES listen (~> 3.9.0) lograge logstash-event - puma (~> 6.4.3) + puma (~> 7.0) rack-attack - rails (~> 7.1.3.4) + rails (~> 8.1.0) + redis (~> 5.0) rubocop (~> 1.64.1) rubocop-performance (~> 1.21.1) rubocop-rails (~> 2.25.0) @@ -460,7 +518,7 @@ DEPENDENCIES yabeda-rails RUBY VERSION - ruby 3.3.4p94 + ruby 3.4.9 BUNDLED WITH - 2.5.11 + 4.0.8 diff --git a/Rakefile b/Rakefile index e85f9139..8c597e8a 100644 --- a/Rakefile +++ b/Rakefile @@ -2,5 +2,6 @@ # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. require_relative 'config/application' +require 'exception_notification/rake' Rails.application.load_tasks diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index add05d60..1261dcce 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,9 +1,15 @@ class ApplicationController < ActionController::Base around_action :store_request_for_logging + around_action :cache_sitemap_response, if: :sitemap_index_request? before_action :get_session_streaming_token, :set_locale, :set_theme, :set_date, :set_feed + before_action :set_sitemap_url_options, if: :sitemap_request? def default_url_options - {locale: (I18n.locale == I18n.default_locale) ? nil : I18n.locale} + opts = { locale: (I18n.locale == I18n.default_locale) ? nil : I18n.locale } + if sitemap_request? + opts.merge!(host: CANONICAL_HOST, protocol: CANONICAL_PROTOCOL, port: canonical_port) + end + opts end def innovation_in_blockchain? @@ -117,11 +123,54 @@ def set_feed def get_session_streaming_token token = StreamingTokenService.get payload = StreamingTokenService.payload - + + session["streaming_access_token"] = token @streaming_access_token = token @streaming_token_time_live = payload&.dig(:expires_at) end + def streaming_authorization_header + StreamingTokenService.get + end + + def sitemap_request? + controller_name.in?(%w[sitemap sitemaps]) + end + + def sitemap_index_request? + sitemap_request? && action_name == "index" + end + + def set_sitemap_url_options + Rails.application.routes.default_url_options.merge!( + host: CANONICAL_HOST, + protocol: CANONICAL_PROTOCOL, + port: canonical_port + ) + end + + def canonical_port + CANONICAL_PROTOCOL == "https" ? 443 : 80 + end + + def cache_sitemap_response + cache_key = ["sitemap", "v1", controller_path, params[:blockchain], params[:locale]].compact.join("/") + if (cached = Rails.cache.read(cache_key)) + render plain: cached["body"], content_type: cached["content_type"], layout: false + return + end + + yield + + return unless response.successful? && response.body.present? + + Rails.cache.write( + cache_key, + { "body" => response.body, "content_type" => response.media_type }, + expires_in: 24.hours + ) + end + # def get_streaming_access_token # url = URI("https://oauth2.bitquery.io/oauth2/token") # https = Net::HTTP.new(url.host, url.port) diff --git a/app/controllers/proxy_eap_graphql_controller.rb b/app/controllers/proxy_eap_graphql_controller.rb index cf82e8f6..18055b5b 100644 --- a/app/controllers/proxy_eap_graphql_controller.rb +++ b/app/controllers/proxy_eap_graphql_controller.rb @@ -13,7 +13,7 @@ def index { 'Content-Type' => 'application/json', 'Accept' => 'application/json', 'X-API-KEY' => api_key, - 'Authorization' => session['streaming_access_token'] }) + 'Authorization' => streaming_authorization_header }) unless res.is_a?(Net::HTTPSuccess) Rails.logger.error "EAP GraphQL failed with status #{res.code}: #{res.body}" diff --git a/app/controllers/proxy_graphql_controller.rb b/app/controllers/proxy_graphql_controller.rb index 9f6478e3..53e9f7ee 100644 --- a/app/controllers/proxy_graphql_controller.rb +++ b/app/controllers/proxy_graphql_controller.rb @@ -11,7 +11,7 @@ def index { 'Content-Type' => 'application/json', 'Accept' => 'application/json', 'X-API-KEY' => api_key, - 'Authorization' => session['streaming_access_token'] }) + 'Authorization' => streaming_authorization_header }) respond_to do |format| format.json do diff --git a/app/controllers/proxy_streaming_graphql_controller.rb b/app/controllers/proxy_streaming_graphql_controller.rb index 129e66f4..a2e06327 100644 --- a/app/controllers/proxy_streaming_graphql_controller.rb +++ b/app/controllers/proxy_streaming_graphql_controller.rb @@ -14,7 +14,7 @@ def index { query: params[:query], variables: params[:variables] }.to_json, { 'Content-Type' => 'application/json', 'Accept' => 'application/json', - 'X-API-KEY' => api_key, 'Authorization' => session['streaming_access_token'] }) + 'X-API-KEY' => api_key, 'Authorization' => streaming_authorization_header }) respond_to do |format| format.json do diff --git a/app/helpers/seo_helper.rb b/app/helpers/seo_helper.rb new file mode 100644 index 00000000..fa98cc6d --- /dev/null +++ b/app/helpers/seo_helper.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module SeoHelper + def canonical_page_url + path = request.path + "#{CANONICAL_BASE_URL}#{path}" + end + + def robots_meta_content + if @noindex_entity + "noindex,follow" + else + "index,follow" + end + end + + def page_h1_text + return @page_h1 if @page_h1.present? + return content_for(:title).presence if content_for?(:title) + + @breadcrumbs&.last&.dig(:name) + end + + def breadcrumb_json_ld + return unless @breadcrumbs&.any? + + items = @breadcrumbs.each_with_index.map do |crumb, index| + url = crumb[:url].to_s + url = "#{CANONICAL_BASE_URL}#{url}" unless url.start_with?("http") + + { + "@type" => "ListItem", + "position" => index + 1, + "name" => crumb[:name].to_s.gsub(/<[^>]*>/, ""), + "item" => url + } + end + + { + "@context" => "https://schema.org", + "@type" => "BreadcrumbList", + "itemListElement" => items + }.to_json + end + + def website_json_ld + { + "@context" => "https://schema.org", + "@type" => "WebSite", + "name" => "Bitquery Explorer", + "url" => CANONICAL_BASE_URL, + "potentialAction" => { + "@type" => "SearchAction", + "target" => { + "@type" => "EntryPoint", + "urlTemplate" => "#{CANONICAL_BASE_URL}/search/{search_term_string}" + }, + "query-input" => "required name=search_term_string" + } + }.to_json + end + + def explorer_absolute_url(url_options = {}) + url_for(url_options.merge(host: CANONICAL_HOST, protocol: CANONICAL_PROTOCOL, only_path: false)) + end +end diff --git a/app/services/streaming_token_service.rb b/app/services/streaming_token_service.rb index 4e30f137..1deadd36 100644 --- a/app/services/streaming_token_service.rb +++ b/app/services/streaming_token_service.rb @@ -8,9 +8,18 @@ def self.get return payload[:token] end + static_token = ENV["BITQUERY_ACCESS_TOKEN"].to_s + unless static_token.empty? + token = static_token.start_with?("Bearer ") ? static_token : "Bearer #{static_token}" + data = { token: token, expires_at: 1.year.from_now } + Rails.cache.write(KEY, data, expires_in: 1.year, race_condition_ttl: 10) + Rails.logger.info "[StreamingToken] STATIC token from BITQUERY_ACCESS_TOKEN" + return token + end + cid = ENV["GRAPHQL_CLIENT_ID"]; csec = ENV["GRAPHQL_CLIENT_SECRET"] if cid.to_s.empty? || csec.to_s.empty? - Rails.logger.error "[StreamingToken] MISSING ENV GRAPHQL_CLIENT_ID/SECRET" + Rails.logger.error "[StreamingToken] MISSING ENV GRAPHQL_CLIENT_ID/SECRET or BITQUERY_ACCESS_TOKEN" return payload&.dig(:token) end diff --git a/app/views/shared/_footer.html.erb b/app/views/shared/_footer.html.erb index cb837ea5..11ac6730 100644 --- a/app/views/shared/_footer.html.erb +++ b/app/views/shared/_footer.html.erb @@ -36,12 +36,6 @@ | <%= link_to "Youtube", 'https://www.youtube.com/@bitquery', target: '_blank' %>

-
- <%= link_to 'https://casinomonks.com/', target: '_blank', style: 'display: inline-flex; align-items: center; justify-content: center; text-decoration: none; color: #007bff;' do %> - - Casino Reviews - <% end %> -
diff --git a/app/views/shared/_head.html.erb b/app/views/shared/_head.html.erb index 1c6c4180..9c880b0a 100644 --- a/app/views/shared/_head.html.erb +++ b/app/views/shared/_head.html.erb @@ -11,6 +11,7 @@ <% end %> + <%= render partial: 'shared/seo_meta' %> <%= csrf_meta_tags %> <%= csp_meta_tag %> diff --git a/app/views/shared/_seo_meta.html.erb b/app/views/shared/_seo_meta.html.erb new file mode 100644 index 00000000..8e839bca --- /dev/null +++ b/app/views/shared/_seo_meta.html.erb @@ -0,0 +1,16 @@ + + + + + + + +<% if @breadcrumbs&.any? %> + +<% end %> +<% if controller_name == 'home' && action_name == 'index' %> + +<% end %> +<% if page_h1_text.present? && !content_for?(:visible_h1) %> +

<%= strip_tags(page_h1_text.to_s) %>

+<% end %> diff --git a/app/views/sitemaps/robots.text.erb b/app/views/sitemaps/robots.text.erb index e9533d46..50ba3fbd 100644 --- a/app/views/sitemaps/robots.text.erb +++ b/app/views/sitemaps/robots.text.erb @@ -1,4 +1,4 @@ -Sitemap: https://explorer.bitquery.io/sitemap.xml +Sitemap: <%= CANONICAL_BASE_URL %>/sitemap.xml User-agent: * Disallow: /bsc/token/0x7c47ef5fb27d99f1913d31071ed0bcc6cb331af1/ diff --git a/app/views/utility/errors.html.erb b/app/views/utility/errors.html.erb index daba626f..7b3a6048 100644 --- a/app/views/utility/errors.html.erb +++ b/app/views/utility/errors.html.erb @@ -1,3 +1,9 @@


-

<%= t('errors.no_methods') %>

+

Page not found

+

The requested page does not exist in this explorer.

+

+ Try the home page, + Ethereum explorer, or + Bitcoin explorer. +




\ No newline at end of file diff --git a/bin/dev-server b/bin/dev-server new file mode 100755 index 00000000..4e0abe49 --- /dev/null +++ b/bin/dev-server @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +export PATH="/opt/homebrew/opt/ruby@3.4/bin:/opt/homebrew/lib/ruby/gems/3.4.0/bin:${PATH}" +export RAILS_ENV=development + +if [[ ! -d node_modules ]]; then + echo "Installing JavaScript dependencies..." + yarn install +fi + +if [[ ! -f public/packs/manifest.json ]]; then + echo "Compiling frontend assets (first run may take a few minutes)..." + bundle exec rails shakapacker:compile +fi + +echo "Starting Bitquery Explorer at http://localhost:${PORT:-3000}" +exec bundle exec rails server -b 0.0.0.0 -p "${PORT:-3000}" diff --git a/config/application.rb b/config/application.rb index c8cda900..b8628381 100644 --- a/config/application.rb +++ b/config/application.rb @@ -22,7 +22,7 @@ module Explorer class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. - config.load_defaults 7.0 + config.load_defaults 8.0 config.eager_load_paths << Rails.root.join("lib") # Settings in config/environments/* take precedence over those specified here. # Application configuration can go into files in config/initializers diff --git a/config/environments/production.rb b/config/environments/production.rb index e38ce864..5790ca76 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -56,7 +56,11 @@ # Use a different cache store in production. # config.cache_store = :memory_store - config.cache_store = :file_store, Rails.root.join("tmp/cache") + config.cache_store = if ENV["REDIS_URL"].present? + [:redis_cache_store, { url: ENV["REDIS_URL"], reconnect_attempts: 1 }] + else + [:file_store, Rails.root.join("tmp/cache")] + end # Use a real queuing backend for Active Job (and separate queues per environment). # config.active_job.queue_adapter = :resque diff --git a/config/initializers/bitquery_logger.rb b/config/initializers/bitquery_logger.rb index cc3cb2d5..90bfa090 100644 --- a/config/initializers/bitquery_logger.rb +++ b/config/initializers/bitquery_logger.rb @@ -15,8 +15,6 @@ # config.ignore_crawlers %w{Googlebot bingbot} end -ExceptionNotifier::Rake.configure - Rails.application.config.middleware.use ExceptionNotification::Rack, bitquery: {} diff --git a/config/initializers/canonical_host.rb b/config/initializers/canonical_host.rb new file mode 100644 index 00000000..e553070d --- /dev/null +++ b/config/initializers/canonical_host.rb @@ -0,0 +1,8 @@ +Rails.application.configure do + config.canonical_host = ENV.fetch("CANONICAL_HOST", "explorer.bitquery.io") + config.canonical_protocol = ENV.fetch("CANONICAL_PROTOCOL", "https") +end + +CANONICAL_HOST = Rails.application.config.canonical_host +CANONICAL_PROTOCOL = Rails.application.config.canonical_protocol +CANONICAL_BASE_URL = "#{CANONICAL_PROTOCOL}://#{CANONICAL_HOST}" diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index c9e08627..542a2e5b 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -1,50 +1,51 @@ class Rack::Attack - BOT_USER_AGENTS = %w[ - bot - crawler - spider - slurp # Yahoo! Slurp - yandexbot # Яндекс - bingbot # Bing - baiduspider # Baidu - duckduckbot # DuckDuckGo - facebookexternalhit - facebot # Facebook - twitterbot # Twitter - applebot # Applebot (Safari News) - petalbot # Huawei Petal - semrushbot # SEMrush - ahrefsbot # Ahrefs - mj12bot # Majestic - dotbot # Moz - sogou # Sogou - exabot # Exalead - archive.org_bot # Wayback Machine - linkedinbot # LinkedIn - pinterestbot # Pinterest - telegrambot # Telegram - ].join('|').freeze - - blocklist('block known bots') do |req| - req.user_agent.to_s.downcase.match?(/#{BOT_USER_AGENTS}/) + # Aggressive SEO scrapers — block outright. + SCRAPER_USER_AGENTS = %w[ + mj12bot + dotbot + exabot + sogou + ].freeze + + blocklist("block aggressive scrapers") do |req| + ua = req.user_agent.to_s.downcase + SCRAPER_USER_AGENTS.any? { |bot| ua.include?(bot) } + end + + # Rate limits replace the old blanket bot blocklist (which matched Googlebot via "bot"). + throttle("req/ip", limit: 300, period: 5.minutes) do |req| + req.ip unless req.path.start_with?("/assets", "/packs", "/metrics") end - self.blocklisted_response = lambda do |env| - request = Rack::Request.new(env) + throttle("graphql/ip", limit: 60, period: 1.minute) do |req| + req.ip if req.post? && req.path.match?(%r{\A/(proxy_graphql|proxy_streaming_graphql|proxy_eap_graphql)\z}) + end + self.blocklisted_responder = lambda do |request| BitqueryLogger.extra_context( - ip: request.ip, + ip: request.ip, user_agent: request.user_agent, - path: request.fullpath + path: request.fullpath ) BitqueryLogger.warn <<~LOG - ===== Rack::Attack ===== Blocked bot request: + ===== Rack::Attack ===== Blocked request: + • IP: #{request.ip} + • User-Agent: #{request.user_agent.inspect} + • Path: #{request.fullpath} + LOG + + [403, { "Content-Type" => "text/plain" }, ["Forbidden"]] + end + + self.throttled_responder = lambda do |request| + BitqueryLogger.warn <<~LOG + ===== Rack::Attack ===== Throttled request: • IP: #{request.ip} • User-Agent: #{request.user_agent.inspect} • Path: #{request.fullpath} LOG - [403, { 'Content-Type' => 'text/plain' }, ['Forbidden']] + [429, { "Content-Type" => "text/plain", "Retry-After" => "60" }, ["Too Many Requests"]] end end diff --git a/config/routes.rb b/config/routes.rb index d781c1c5..102fa811 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -682,6 +682,15 @@ get "proxy_dbcode/:dashbord_url", to: "proxy_dbcode#index", defaults: { format: :json } get "platform/:action", controller: "home" + + # Home page category tabs link to /smart_contract, /bitcoin, etc. + %w[smart_contract bitcoin cosmos ripple utility testnet cardano graph offchain].each do |action| + get action, to: "home##{action}" + end + + # Common mistaken explorer URLs + get ":blockchain/network/:action", to: redirect("/%{blockchain}/%{action}") + # Disabled GraphQL UI routes # get "graphql(/:param)" => "utility#graphql" # get "graphql/reset(/:token)" => "utility#graphql" diff --git a/lib/cached_query.rb b/lib/cached_query.rb new file mode 100644 index 00000000..a09a139a --- /dev/null +++ b/lib/cached_query.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require "timeout" + +class CachedQuery + QUERY_VERSION = 1 + TIMEOUT_SECONDS = 3 + + class << self + def fetch(key_parts, expires_in:, query:, variables: {}, context: {}, use_eap: false) + cache_key = (["cached_query", QUERY_VERSION] + Array(key_parts)).join(":") + stale_key = "#{cache_key}:stale" + + Rails.cache.fetch(cache_key, expires_in:, race_condition_ttl: 30) do + result = execute(query:, variables:, context:, use_eap:) + Rails.cache.write(stale_key, result, expires_in: expires_in * 2) unless result.nil? + result + end + rescue StandardError => e + Rails.logger.warn("CachedQuery error [#{Array(key_parts).join('/') }]: #{e.message}") + Rails.cache.read(stale_key) + end + + private + + def execute(query:, variables:, context:, use_eap:) + Timeout.timeout(TIMEOUT_SECONDS) do + Graphql::V2.query_with_retry(query, variables:, context:, use_eap:) + end + end + end +end diff --git a/package.json b/package.json index d7086ca7..675aa0a4 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "moment": "^2.30.1", "popper.js": "^1.16.1", "serialize-javascript": "^6.0.2", - "shakapacker": "8.0.1", + "shakapacker": "8.0.2", "tabulator-tables": "^6.2.5", "vega-widgets": "^2.0.4", "vis": "^4.21.0-EOL" diff --git a/vendor/bundle/ruby/3.3.0/bundler/gems/bitquery_logger-233e44a43d4c b/vendor/bitquery_logger similarity index 100% rename from vendor/bundle/ruby/3.3.0/bundler/gems/bitquery_logger-233e44a43d4c rename to vendor/bitquery_logger diff --git a/yarn.lock b/yarn.lock index 2d35c94a..caa4f09f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5746,10 +5746,10 @@ setprototypeof@1.2.0: resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== -shakapacker@8.0.1: - version "8.0.1" - resolved "https://registry.yarnpkg.com/shakapacker/-/shakapacker-8.0.1.tgz#96c893ccc414d47aaf86689eb5ef732259c592c7" - integrity sha512-VjIWgJqh4680Kf2NSmupB3h6dWxlOkGvceJJuJZ0az/6wUL5d/ovmKUc2swrpWC8UgBY67YjXI92zT07EQevbQ== +shakapacker@8.0.2: + version "8.0.2" + resolved "https://registry.yarnpkg.com/shakapacker/-/shakapacker-8.0.2.tgz#064303e09958b57b7a7f98dc38d0c1150a96d3a0" + integrity sha512-nZNAwT7onNiKZ81nMz/l0ihG8qXPZUvfxs4op5586wvn7znpPVQC4mFs131YrdUubmdqUPIjcpZnV0k4ijNtXw== dependencies: js-yaml "^4.1.0" path-complete-extname "^1.0.0" From 12932914ed2f27b535f20cef348a074e1bfd81f4 Mon Sep 17 00:00:00 2001 From: Gaurav agarwal Date: Thu, 11 Jun 2026 18:26:02 +0530 Subject: [PATCH 2/4] Allow search engine crawlers while blocking scrapers and scripted clients. Safelist legitimate search and social preview bots, throttle normal traffic, and return 403 for SEO harvesters, generic bots, and curl/python/scrapy clients. Co-authored-by: Cursor --- config/initializers/rack_attack.rb | 136 ++++++++++++++++++++++++++--- 1 file changed, 125 insertions(+), 11 deletions(-) diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 542a2e5b..8e7c1431 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -1,24 +1,138 @@ class Rack::Attack - # Aggressive SEO scrapers — block outright. - SCRAPER_USER_AGENTS = %w[ - mj12bot - dotbot - exabot - sogou + # Search engines + social link-preview crawlers: safelisted (skip throttles & bot blocks). + SEARCH_ENGINE_PATTERNS = [ + /googlebot/i, + /bingbot/i, + /duckduckbot/i, + /applebot/i, + /yandexbot/i, + /baiduspider/i, + /sogou/i, + /petalbot/i, + /ia_archiver/i # Internet Archive / Wayback Machine ].freeze - blocklist("block aggressive scrapers") do |req| - ua = req.user_agent.to_s.downcase - SCRAPER_USER_AGENTS.any? { |bot| ua.include?(bot) } + SOCIAL_CRAWLER_PATTERNS = [ + /facebookexternalhit/i, + /facebot/i, + /twitterbot/i, + /linkedinbot/i, + /pinterestbot/i, + /telegrambot/i, + /slackbot/i, + /discordbot/i, + /whatsapp/i, + /embedly/i + ].freeze + + # SEO tools, AI scrapers, and aggressive harvesters — always blocked. + BLOCKED_SCRAPER_PATTERNS = [ + /mj12bot/i, + /dotbot/i, + /semrushbot/i, + /ahrefsbot/i, + /rogerbot/i, + /screaming[\s_-]?frog/i, + /bytespider/i, + /gptbot/i, + /claudebot/i, + /anthropic-ai/i, + /ccbot/i, + /dataforseo/i, + /serpstatbot/i, + /seokicks/i, + /megaindex/i, + /exabot/i, + /blexbot/i, + /petalsearch/i # Huawei Petal SEO spider (distinct from petalbot search crawler) + ].freeze + + # Generic bot/scraper signatures when not safelisted above. + GENERIC_BOT_PATTERN = / + crawler | + spider | + scraper | + httpclient | + libwww | + headlesschrome | + phantomjs | + selenium + /ix + + SCRIPTED_CLIENT_PATTERN = %r{ + \bcurl/ | + \bwget/ | + python- | + \bscrapy | + go-http-client | + java/ | + \bokhttp | + axios/ | + node-fetch | + httpx/ | + ruby/ | + perl/ | + php/ + }ix + + class << self + def allowed_crawler?(req) + ua = req.user_agent.to_s + return false if ua.strip.empty? + + SEARCH_ENGINE_PATTERNS.any? { |pattern| ua.match?(pattern) } || + SOCIAL_CRAWLER_PATTERNS.any? { |pattern| ua.match?(pattern) } + end + end + + # --- Safelist: legitimate search & social crawlers --------------------------------- + + safelist("allow search engines and social crawlers") do |req| + allowed_crawler?(req) + end + + # --- Blocklist: scrapers, SEO bots, scripted clients ------------------------------ + + blocklist("block known scrapers") do |req| + ua = req.user_agent.to_s + BLOCKED_SCRAPER_PATTERNS.any? { |pattern| ua.match?(pattern) } end - # Rate limits replace the old blanket bot blocklist (which matched Googlebot via "bot"). + blocklist("block unlisted bots") do |req| + next false if allowed_crawler?(req) + + ua = req.user_agent.to_s + ua.match?(GENERIC_BOT_PATTERN) || ua.match?(/bot/i) + end + + blocklist("block scripted clients") do |req| + next false if allowed_crawler?(req) + next false if req.path.start_with?("/metrics") + + ua = req.user_agent.to_s + ua.strip.empty? || ua.match?(SCRIPTED_CLIENT_PATTERN) + end + + # --- Throttles: humans & scrapers posing as browsers -------------------------------- + throttle("req/ip", limit: 300, period: 5.minutes) do |req| + next if allowed_crawler?(req) + req.ip unless req.path.start_with?("/assets", "/packs", "/metrics") end + throttle("html burst/ip", limit: 45, period: 1.minute) do |req| + next if allowed_crawler?(req) + next if req.path.start_with?("/assets", "/packs", "/metrics", "/proxy_") + + req.ip + end + throttle("graphql/ip", limit: 60, period: 1.minute) do |req| - req.ip if req.post? && req.path.match?(%r{\A/(proxy_graphql|proxy_streaming_graphql|proxy_eap_graphql)\z}) + next unless req.post? && req.path.match?(%r{\A/(proxy_graphql|proxy_streaming_graphql|proxy_eap_graphql)\z}) + next if allowed_crawler?(req) + + req.ip end self.blocklisted_responder = lambda do |request| From 73601eb6feace1480ea4c52094f2c8456e39e6c3 Mon Sep 17 00:00:00 2001 From: Gaurav agarwal Date: Thu, 11 Jun 2026 20:15:36 +0530 Subject: [PATCH 3/4] Remove Wave 0 caching infrastructure. Drop CachedQuery helper, sitemap response cache, and Redis cache store config; production falls back to file_store only. Co-authored-by: Cursor --- app/controllers/application_controller.rb | 23 ---------------- config/environments/production.rb | 6 +---- lib/cached_query.rb | 32 ----------------------- 3 files changed, 1 insertion(+), 60 deletions(-) delete mode 100644 lib/cached_query.rb diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 1261dcce..ea93dc9f 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,6 +1,5 @@ class ApplicationController < ActionController::Base around_action :store_request_for_logging - around_action :cache_sitemap_response, if: :sitemap_index_request? before_action :get_session_streaming_token, :set_locale, :set_theme, :set_date, :set_feed before_action :set_sitemap_url_options, if: :sitemap_request? @@ -137,10 +136,6 @@ def sitemap_request? controller_name.in?(%w[sitemap sitemaps]) end - def sitemap_index_request? - sitemap_request? && action_name == "index" - end - def set_sitemap_url_options Rails.application.routes.default_url_options.merge!( host: CANONICAL_HOST, @@ -153,24 +148,6 @@ def canonical_port CANONICAL_PROTOCOL == "https" ? 443 : 80 end - def cache_sitemap_response - cache_key = ["sitemap", "v1", controller_path, params[:blockchain], params[:locale]].compact.join("/") - if (cached = Rails.cache.read(cache_key)) - render plain: cached["body"], content_type: cached["content_type"], layout: false - return - end - - yield - - return unless response.successful? && response.body.present? - - Rails.cache.write( - cache_key, - { "body" => response.body, "content_type" => response.media_type }, - expires_in: 24.hours - ) - end - # def get_streaming_access_token # url = URI("https://oauth2.bitquery.io/oauth2/token") # https = Net::HTTP.new(url.host, url.port) diff --git a/config/environments/production.rb b/config/environments/production.rb index 5790ca76..e38ce864 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -56,11 +56,7 @@ # Use a different cache store in production. # config.cache_store = :memory_store - config.cache_store = if ENV["REDIS_URL"].present? - [:redis_cache_store, { url: ENV["REDIS_URL"], reconnect_attempts: 1 }] - else - [:file_store, Rails.root.join("tmp/cache")] - end + config.cache_store = :file_store, Rails.root.join("tmp/cache") # Use a real queuing backend for Active Job (and separate queues per environment). # config.active_job.queue_adapter = :resque diff --git a/lib/cached_query.rb b/lib/cached_query.rb deleted file mode 100644 index a09a139a..00000000 --- a/lib/cached_query.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -require "timeout" - -class CachedQuery - QUERY_VERSION = 1 - TIMEOUT_SECONDS = 3 - - class << self - def fetch(key_parts, expires_in:, query:, variables: {}, context: {}, use_eap: false) - cache_key = (["cached_query", QUERY_VERSION] + Array(key_parts)).join(":") - stale_key = "#{cache_key}:stale" - - Rails.cache.fetch(cache_key, expires_in:, race_condition_ttl: 30) do - result = execute(query:, variables:, context:, use_eap:) - Rails.cache.write(stale_key, result, expires_in: expires_in * 2) unless result.nil? - result - end - rescue StandardError => e - Rails.logger.warn("CachedQuery error [#{Array(key_parts).join('/') }]: #{e.message}") - Rails.cache.read(stale_key) - end - - private - - def execute(query:, variables:, context:, use_eap:) - Timeout.timeout(TIMEOUT_SECONDS) do - Graphql::V2.query_with_retry(query, variables:, context:, use_eap:) - end - end - end -end From de7e6e9727996f4ef445fadbe0915caa97bd27f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 09:04:11 +0000 Subject: [PATCH 4/4] Bump http-proxy-middleware from 2.0.6 to 2.0.10 Bumps [http-proxy-middleware](https://github.com/chimurai/http-proxy-middleware) from 2.0.6 to 2.0.10. - [Release notes](https://github.com/chimurai/http-proxy-middleware/releases) - [Changelog](https://github.com/chimurai/http-proxy-middleware/blob/v2.0.10/CHANGELOG.md) - [Commits](https://github.com/chimurai/http-proxy-middleware/compare/v2.0.6...v2.0.10) --- updated-dependencies: - dependency-name: http-proxy-middleware dependency-version: 2.0.10 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index caa4f09f..64e05414 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3928,9 +3928,9 @@ http-parser-js@>=0.5.1: integrity sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q== http-proxy-middleware@^2.0.3: - version "2.0.6" - resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz#e1a4dd6979572c7ab5a4e4b55095d1f32a74963f" - integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw== + version "2.0.10" + resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.10.tgz#b2df7b705203d7a8c269ac8450cf96b00c532f94" + integrity sha512-RKzRWNPxUZqbuk3BC5mGVJbBnWgr+diEnjJexIOytFbBzDy88Fbh/YvBr3DsNrl1jYAfjWfpATEv0NO35FDuPQ== dependencies: "@types/http-proxy" "^1.17.8" http-proxy "^1.18.1"