Merge eb744032dc into 1959365c2f

.devcontainer/docker-compose-redis-sentinel.yml

@@ -0,0 +1,115 @@
+version: '3'
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ../..:/workspaces:cached
+    environment:
+      RAILS_ENV: development
+      NODE_ENV: development
+      BIND: 0.0.0.0
+      REDIS_SENTINEL: redis-sentinel
+      REDIS_PASSWORD: str0ng_passw0rd
+      DB_HOST: db
+      DB_USER: postgres
+      DB_PASS: postgres
+      DB_PORT: '5432'
+      ES_ENABLED: 'true'
+      ES_HOST: es
+      ES_PORT: '9200'
+      LIBRE_TRANSLATE_ENDPOINT: http://libretranslate:5000
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+    ports:
+      - '127.0.0.1:3000:3000'
+      - '127.0.0.1:3035:3035'
+      - '127.0.0.1:4000:4000'
+    networks:
+      - external_network
+      - internal_network
+
+  db:
+    image: postgres:14-alpine
+    restart: unless-stopped
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_HOST_AUTH_METHOD: trust
+    networks:
+      - internal_network
+
+  # Redis Sentinel with one Replica for testing
+  redis:
+    image: 'bitnami/redis:latest'
+    environment:
+      - REDIS_REPLICATION_MODE=master
+      - REDIS_PASSWORD=str0ng_passw0rd
+    networks:
+      - internal_network
+    ports:
+      - '6379:6379'
+  redis-replica:
+    image: 'bitnami/redis:latest'
+    environment:
+      - REDIS_REPLICATION_MODE=slave
+      - REDIS_MASTER_HOST=redis
+      - REDIS_MASTER_PASSWORD=str0ng_passw0rd
+      - REDIS_PASSWORD=str0ng_passw0rd
+    networks:
+      - internal_network
+    ports:
+      - '6379:6379'
+  redis-sentinel:
+    image: 'bitnami/redis-sentinel:latest'
+    environment:
+      - REDIS_MASTER_PASSWORD=str0ng_passw0rd
+    ports:
+      - '26379:26379'
+    networks:
+      - internal_network
+    depends_on:
+      - redis
+      - redis-replica
+
+  es:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
+    restart: unless-stopped
+    environment:
+      ES_JAVA_OPTS: -Xms512m -Xmx512m
+      cluster.name: es-mastodon
+      discovery.type: single-node
+      bootstrap.memory_lock: 'true'
+    volumes:
+      - es-data:/usr/share/elasticsearch/data
+    networks:
+      - internal_network
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+
+  libretranslate:
+    image: libretranslate/libretranslate:v1.3.11
+    restart: unless-stopped
+    volumes:
+      - lt-data:/home/libretranslate/.local
+    networks:
+      - external_network
+      - internal_network
+
+volumes:
+  postgres-data:
+  redis-data:
+  es-data:
+  lt-data:
+
+networks:
+  external_network:
+  internal_network:
+    internal: true

app/lib/redis_configuration.rb

@@ -39,12 +39,47 @@ class RedisConfiguration
   end
 
   def url
-    ENV['REDIS_URL']
+    if ENV['REDIS_SENTINEL']
+      m = ENV.fetch('REDIS_SENTINEL_MASTER', 'mymaster')
+      p = ENV.fetch('REDIS_PASSWORD', '')
+      "redis://:#{p}@#{m}"
+    else
+      ENV['REDIS_URL']
+    end
+  end
+
+  def sentinels
+    return unless ENV['REDIS_SENTINEL']
+
+    unless ENV['REDIS_SENTINEL'].include? ','
+      ips = Resolv.getaddresses(ENV['REDIS_SENTINEL'])
+      port = ENV.fetch('REDIS_SENTINEL_PORT', '26379')
+
+      ENV['REDIS_SENTINEL'] = ips.map do |ip|
+        "#{ip}:#{port}"
+      end.join(',')
+    end
+    ENV['REDIS_SENTINEL'].split(',').map do |server|
+      host, port = server.split(':')
+      { host: host, port: port.to_i }
+    end
+  end
+
+  def master_name
+    ENV.fetch('REDIS_SENTINEL_MASTER', 'mymaster')
+  end
+
+  def sentinel_mode?
+    ENV.include? 'REDIS_SENTINEL'
   end
 
   private
 
   def raw_connection
-    Redis.new(url: url, driver: :hiredis)
+    if sentinel_mode?
+      Redis.new(url: url, driver: :hiredis, sentinels: sentinels, master_name: master_name)
+    else
+      Redis.new(url: url, driver: :hiredis)
+    end
   end
 end

lib/mastodon/redis_config.rb

@@ -20,6 +20,24 @@ def setup_redis_env_url(prefix = nil, defaults = true)
       end.normalize.to_str
     end
   end
+
+  return unless ENV["#{prefix}REDIS_SENTINEL"] || ENV['REDIS_SENTINEL']
+
+  ENV["#{prefix}REDIS_SENTINEL"] = ENV["#{prefix}REDIS_SENTINEL"] || ENV['REDIS_SENTINEL']
+
+  sentinel_master = ENV.fetch("#{prefix}REDIS_SENTINEL_MASTER", 'mymaster')
+  password = ENV["#{prefix}REDIS_PASSWORD"] || ENV.fetch('REDIS_PASSWORD')
+
+  unless ENV["#{prefix}REDIS_SENTINEL"].include? ','
+    ips = Resolv.getaddresses(ENV["#{prefix}REDIS_SENTINEL"])
+    port = ENV.fetch("#{prefix}REDIS_SENTINEL_PORT", '26379')
+
+    ENV["#{prefix}REDIS_SENTINEL"] = ips.map do |ip|
+      "#{ip}:#{port}"
+    end.join(',').concat(',')
+  end
+
+  ENV["#{prefix}REDIS_URL"] = "redis://:#{password}@#{sentinel_master}"
 end
 
 setup_redis_env_url
@@ -40,12 +58,28 @@ REDIS_CACHE_PARAMS = {
     size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
     timeout: 5,
   },
+
+  master_name: (ENV.fetch('CACHE_REDIS_SENTINEL_MASTER', 'mymaster') if ENV['CACHE_REDIS_SENTINEL']),
+  sentinels: (if ENV['CACHE_REDIS_SENTINEL']
+                ENV['CACHE_REDIS_SENTINEL'].split(',').map do |server|
+                  host, port = server.split(':')
+                  { host: host, port: port.to_i }
+                end
+              end),
 }.freeze
 
 REDIS_SIDEKIQ_PARAMS = {
   driver: :hiredis,
   url: ENV['SIDEKIQ_REDIS_URL'],
   namespace: sidekiq_namespace,
+
+  master_name: (ENV.fetch('SIDEKIQ_REDIS_SENTINEL_MASTER', 'mymaster') if ENV['SIDEKIQ_REDIS_SENTINEL']),
+  sentinels: (if ENV['SIDEKIQ_REDIS_SENTINEL']
+                ENV['SIDEKIQ_REDIS_SENTINEL'].split(',').map do |server|
+                  host, port = server.split(':')
+                  { host: host, port: port.to_i }
+                end
+              end),
 }.freeze
 
 ENV['REDIS_NAMESPACE'] = "mastodon_test#{ENV['TEST_ENV_NUMBER']}" if Rails.env.test?

streaming/index.js

@@ -245,13 +245,26 @@ const redisConfigFromEnv = (env) => {
   let redisDatabase = parseIntFromEnv(env.REDIS_DB, 0, 'REDIS_DB');
 
   /** @type {import('ioredis').RedisOptions} */
-  const redisParams = {
+  let redisParams = {
     host: env.REDIS_HOST || '127.0.0.1',
     port: redisPort,
     db: redisDatabase,
     password: env.REDIS_PASSWORD || undefined,
   };
 
+  if (env.REDIS_SENTINEL) {
+    redisParams = {
+      password: env.REDIS_PASSWORD || undefined,
+      sentinels: env.REDIS_SENTINEL.split(',').map(h => {
+        const s = h.split(':');
+        const host = s[0] || '127.0.0.1';
+        const port =  s[1] || '26379';
+        return { "host": host, "port": port };
+      }),
+      name: env.REDIS_SENTINEL_MASTER || 'mymaster',
+    };
+  }
+
   // redisParams.path takes precedence over host and port.
   if (env.REDIS_URL && env.REDIS_URL.startsWith('unix://')) {
     redisParams.path = env.REDIS_URL.slice(7);