zelo72
/
mastodon
mirror of https://github.com/mastodon/mastodon
2
2
Fork 1
Code Issues 9 Releases Wiki Activity

Merge 814c868342 into 1959365c2f

This commit is contained in:
Matt Jankowski 2024-05-10 20:06:51 +00:00 committed by GitHub
parent 1959365c2f 814c868342
commit dce4cda471
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 41 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Gemfile
View File

@ -53,6 +53,7 @@ gem 'ed25519', '~> 1.3'
gem 'fast_blank', '~> 1.0'
gem 'fastimage'
gem 'hiredis', '~> 0.6'
gem 'hiredis-client'
gem 'htmlentities', '~> 4.3'
gem 'http', '~> 5.2.0'
gem 'http_accept_language', '~> 2.1'
@ -77,15 +78,14 @@ gem 'rack-cors', '~> 2.0', require: 'rack/cors'
gem 'rails-i18n', '~> 7.0'
gem 'redcarpet', '~> 3.6'
gem 'redis', '~> 4.5', require: ['redis', 'redis/connection/hiredis']
gem 'redis-namespace', '~> 1.10'
gem 'rqrcode', '~> 2.2'
gem 'ruby-progressbar', '~> 1.13'
gem 'sanitize', '~> 6.0'
gem 'scenic', '~> 1.7'
gem 'sidekiq', '~> 6.5'
gem 'sidekiq', '< 8'
gem 'sidekiq-bulk', '~> 0.2.0'
gem 'sidekiq-scheduler', '~> 5.0'
gem 'sidekiq-unique-jobs', '~> 7.1'
gem 'sidekiq-unique-jobs', '> 8'
gem 'simple_form', '~> 5.2'
gem 'simple-navigation', '~> 4.4'
gem 'stoplight', '~> 4.1'

32
Gemfile.lock
View File

@ -140,9 +140,6 @@ GEM
brakeman (6.1.2)
racc
browser (5.3.1)
brpoplpush-redis_script (0.1.3)
concurrent-ruby (~> 1.0, >= 1.0.5)
redis (>= 1.0, < 6)
builder (3.2.4)
bundler-audit (0.9.1)
bundler (>= 1.2.0, < 3)
@ -308,6 +305,8 @@ GEM
json
highline (3.0.1)
hiredis (0.6.3)
hiredis-client (0.22.1)
redis-client (= 0.22.1)
hkdf (0.3.0)
htmlentities (4.3.4)
http (5.2.0)
@ -681,8 +680,8 @@ GEM
psych (>= 4.0.0)
redcarpet (3.6.0)
redis (4.8.1)
redis-namespace (1.11.0)
redis (>= 4)
redis-client (0.22.1)
connection_pool
redlock (1.3.2)
redis (>= 3.0.0, < 6.0)
regexp_parser (2.9.0)
@ -780,22 +779,21 @@ GEM
rubyzip (>= 1.2.2, < 3.0)
websocket (~> 1.0)
semantic_range (3.0.0)
sidekiq (6.5.12)
connection_pool (>= 2.2.5, < 3)
rack (~> 2.0)
redis (>= 4.5.0, < 5)
sidekiq (7.2.4)
concurrent-ruby (< 2)
connection_pool (>= 2.3.0)
rack (>= 2.2.4)
redis-client (>= 0.19.0)
sidekiq-bulk (0.2.0)
sidekiq
sidekiq-scheduler (5.0.3)
rufus-scheduler (~> 3.2)
sidekiq (>= 6, < 8)
tilt (>= 1.4.0)
sidekiq-unique-jobs (7.1.33)
brpoplpush-redis_script (> 0.1.1, <= 2.0.0)
sidekiq-unique-jobs (8.0.10)
concurrent-ruby (~> 1.0, >= 1.0.5)
redis (< 5.0)
sidekiq (>= 5.0, < 7.0)
thor (>= 0.20, < 3.0)
sidekiq (>= 7.0.0, < 8.0.0)
thor (>= 1.0, < 3.0)
simple-navigation (4.4.0)
activesupport (>= 2.3.2)
simple_form (5.3.0)
@ -940,6 +938,7 @@ DEPENDENCIES
haml_lint
hcaptcha (~> 7.1)
hiredis (~> 0.6)
hiredis-client
htmlentities (~> 4.3)
http (~> 5.2.0)
http_accept_language (~> 2.1)
@ -1008,7 +1007,6 @@ DEPENDENCIES
rdf-normalize (~> 0.5)
redcarpet (~> 3.6)
redis (~> 4.5)
redis-namespace (~> 1.10)
rqrcode (~> 2.2)
rspec-github (~> 2.4)
rspec-rails (~> 6.0)
@ -1024,10 +1022,10 @@ DEPENDENCIES
sanitize (~> 6.0)
scenic (~> 1.7)
selenium-webdriver
sidekiq (~> 6.5)
sidekiq (< 8)
sidekiq-bulk (~> 0.2.0)
sidekiq-scheduler (~> 5.0)
sidekiq-unique-jobs (~> 7.1)
sidekiq-unique-jobs (> 8)
simple-navigation (~> 4.4)
simple_form (~> 5.2)
simplecov (~> 0.22)

6
app/lib/admin/metrics/dimension/software_versions_dimension.rb
View File

@ -75,10 +75,6 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim
end
def redis_info
@redis_info ||= if redis.is_a?(Redis::Namespace)
redis.redis.info
else
redis.info
end
@redis_info ||= redis.info
end
end

6
app/lib/admin/metrics/dimension/space_usage_dimension.rb
View File

@ -59,11 +59,7 @@ class Admin::Metrics::Dimension::SpaceUsageDimension < Admin::Metrics::Dimension
end
def redis_info
@redis_info ||= if redis.is_a?(Redis::Namespace)
redis.redis.info
else
redis.info
end
@redis_info ||= redis.info
end
def search_size

20
app/lib/redis_configuration.rb
View File

@ -23,28 +23,10 @@ class RedisConfiguration
end
def connection
if namespace?
Redis::Namespace.new(namespace, redis: raw_connection)
else
raw_connection
end
end
def namespace?
namespace.present?
end
def namespace
ENV.fetch('REDIS_NAMESPACE', nil)
Redis.new(url: url, driver: :hiredis)
end
def url
ENV['REDIS_URL']
end
private
def raw_connection
Redis.new(url: url, driver: :hiredis)
end
end

2
app/workers/scheduler/self_destruct_scheduler.rb
View File

@ -20,7 +20,7 @@ class Scheduler::SelfDestructScheduler
private
def sidekiq_overwhelmed?
redis_mem_info = Sidekiq.redis_info
redis_mem_info = Sidekiq.default_configuration.redis_info
Sidekiq::Stats.new.enqueued > MAX_ENQUEUED || redis_mem_info['used_memory'].to_f > redis_mem_info['total_system_memory'].to_f * MAX_REDIS_MEM_USAGE
end

1
config/application.rb
View File

@ -24,7 +24,6 @@ Bundler.require(*Rails.groups)
require_relative '../lib/exceptions'
require_relative '../lib/sanitize_ext/sanitize_config'
require_relative '../lib/redis/namespace_extensions'
require_relative '../lib/paperclip/url_generator_extensions'
require_relative '../lib/paperclip/attachment_extensions'
require_relative '../lib/paperclip/lazy_thumbnail'

2
config/initializers/chewy.rb
View File

@ -5,7 +5,7 @@ host = ENV.fetch('ES_HOST') { 'localhost' }
port = ENV.fetch('ES_PORT') { 9200 }
user = ENV.fetch('ES_USER', nil).presence
password = ENV.fetch('ES_PASS', nil).presence
fallback_prefix = ENV.fetch('REDIS_NAMESPACE', nil).presence
fallback_prefix = ENV.fetch('TODO_PLACEHOLDER', nil).presence
prefix = ENV.fetch('ES_PREFIX') { fallback_prefix }
ca_file = ENV.fetch('ES_CA_FILE', nil).presence

2
config/initializers/sidekiq.rb
View File

@ -41,7 +41,7 @@ Sidekiq.configure_client do |config|
end
end
Sidekiq.logger.level = ::Logger.const_get(ENV.fetch('RAILS_LOG_LEVEL', 'info').upcase.to_s)
Sidekiq.default_configuration.logger.level = ::Logger.const_get(ENV.fetch('RAILS_LOG_LEVEL', 'warn').upcase.to_s)
SidekiqUniqueJobs.configure do |config|
config.enabled = !Rails.env.test?

10
lib/mastodon/redis_config.rb
View File

@ -26,15 +26,11 @@ setup_redis_env_url
setup_redis_env_url(:cache, false)
setup_redis_env_url(:sidekiq, false)
namespace = ENV.fetch('REDIS_NAMESPACE', nil)
cache_namespace = namespace ? "#{namespace}_cache" : 'cache'
sidekiq_namespace = namespace
REDIS_CACHE_PARAMS = {
driver: :hiredis,
url: ENV['CACHE_REDIS_URL'],
expires_in: 10.minutes,
namespace: "#{cache_namespace}:7.1",
db: 1,
connect_timeout: 5,
pool: {
size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
@ -44,8 +40,6 @@ REDIS_CACHE_PARAMS = {
REDIS_SIDEKIQ_PARAMS = {
driver: :hiredis,
db: 0,
url: ENV['SIDEKIQ_REDIS_URL'],
namespace: sidekiq_namespace,
}.freeze
ENV['REDIS_NAMESPACE'] = "mastodon_test#{ENV['TEST_ENV_NUMBER']}" if Rails.env.test?

12
lib/redis/namespace_extensions.rb
View File

@ -1,12 +0,0 @@
# frozen_string_literal: true
class Redis
module NamespaceExtensions
def exists?(...)
call_with_namespace('exists?', ...)
end
end
end
Redis::Namespace::COMMANDS['exists?'] = [:first]
Redis::Namespace.prepend(Redis::NamespaceExtensions)

2
spec/models/user_spec.rb
View File

@ -469,7 +469,7 @@ RSpec.describe User do
let!(:access_token) { Fabricate(:access_token, resource_owner_id: user.id) }
let!(:web_push_subscription) { Fabricate(:web_push_subscription, access_token: access_token) }
let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) }
let(:redis_pipeline_stub) { instance_double(Redis::PipelinedConnection, publish: nil) }
before do
allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub)

4
spec/rails_helper.rb
View File

@ -26,7 +26,9 @@ WebMock.disable_net_connect!(
allow_localhost: true,
allow: Chewy.settings[:host]
)
Sidekiq.logger = nil
Sidekiq.configure_server do |config|
config.logger = nil
end
DatabaseCleaner.strategy = [:deletion]

1
spec/support/streaming_server_manager.rb
View File

@ -17,7 +17,6 @@ class StreamingServerManager
@running_thread = Thread.new do
Open3.popen2e(
{
'REDIS_NAMESPACE' => ENV.fetch('REDIS_NAMESPACE'),
'DB_NAME' => "#{ENV.fetch('DB_NAME', 'mastodon')}_test#{ENV.fetch('TEST_ENV_NUMBER', '')}",
'RAILS_ENV' => ENV.fetch('RAILS_ENV', 'test'),
'NODE_ENV' => ENV.fetch('STREAMING_NODE_ENV', 'development'),

29
streaming/index.js
View File

@ -228,7 +228,6 @@ const pgConfigFromEnv = (env) => {
/**
* @typedef RedisConfiguration
* @property {import('ioredis').RedisOptions} redisParams
* @property {string} redisPrefix
* @property {string|undefined} redisUrl
*/
@ -237,10 +236,6 @@ const pgConfigFromEnv = (env) => {
* @returns {RedisConfiguration} configuration for the Redis connection
*/
const redisConfigFromEnv = (env) => {
// ioredis *can* transparently add prefixes for us, but it doesn't *in some cases*,
// which means we can't use it. But this is something that should be looked into.
const redisPrefix = env.REDIS_NAMESPACE ? `${env.REDIS_NAMESPACE}:` : '';
let redisPort = parseIntFromEnv(env.REDIS_PORT, 6379, 'REDIS_PORT');
let redisDatabase = parseIntFromEnv(env.REDIS_DB, 0, 'REDIS_DB');
@ -259,7 +254,6 @@ const redisConfigFromEnv = (env) => {
return {
redisParams,
redisPrefix,
redisUrl: typeof env.REDIS_URL === 'string' ? env.REDIS_URL : undefined,
};
};
@ -382,7 +376,6 @@ const startServer = async () => {
const redisConfig = redisConfigFromEnv(process.env);
const redisSubscribeClient = await createRedisClient(redisConfig);
const redisClient = await createRedisClient(redisConfig);
const { redisPrefix } = redisConfig;
const metrics = setupMetrics(CHANNEL_NAMES, pgPool);
// TODO: migrate all metrics to metrics.X.method() instead of just X.method()
@ -422,7 +415,7 @@ const startServer = async () => {
const interval = 6 * 60;
const tellSubscribed = () => {
channels.forEach(channel => redisClient.set(`${redisPrefix}subscribed:${channel}`, '1', 'EX', interval * 3));
channels.forEach(channel => redisClient.set(`subscribed:${channel}`, '1', 'EX', interval * 3));
};
tellSubscribed();
@ -443,7 +436,7 @@ const startServer = async () => {
const callbacks = subs[channel];
logger.debug(`New message on channel ${redisPrefix}${channel}`);
logger.debug(`New message on channel ${channel}`);
if (!callbacks) {
return;
@ -697,14 +690,14 @@ const startServer = async () => {
});
res.on('close', () => {
unsubscribe(`${redisPrefix}${accessTokenChannelId}`, listener);
unsubscribe(`${redisPrefix}${systemChannelId}`, listener);
unsubscribe(`${accessTokenChannelId}`, listener);
unsubscribe(`${systemChannelId}`, listener);
connectedChannels.labels({ type: 'eventsource', channel: 'system' }).dec(2);
});
subscribe(`${redisPrefix}${accessTokenChannelId}`, listener);
subscribe(`${redisPrefix}${systemChannelId}`, listener);
subscribe(`${accessTokenChannelId}`, listener);
subscribe(`${systemChannelId}`, listener);
connectedChannels.labels({ type: 'eventsource', channel: 'system' }).inc(2);
};
@ -1034,11 +1027,11 @@ const startServer = async () => {
};
channelIds.forEach(id => {
subscribe(`${redisPrefix}${id}`, listener);
subscribe(`${id}`, listener);
});
if (typeof attachCloseHandler === 'function') {
attachCloseHandler(channelIds.map(id => `${redisPrefix}${id}`), listener);
attachCloseHandler(channelIds.map(id => `${id}`), listener);
}
return listener;
@ -1385,7 +1378,7 @@ const startServer = async () => {
}
channelIds.forEach(channelId => {
unsubscribe(`${redisPrefix}${channelId}`, subscription.listener);
unsubscribe(`${channelId}`, subscription.listener);
});
connectedChannels.labels({ type: 'websocket', channel: subscription.channelName }).dec();
@ -1429,8 +1422,8 @@ const startServer = async () => {
},
});
subscribe(`${redisPrefix}${accessTokenChannelId}`, listener);
subscribe(`${redisPrefix}${systemChannelId}`, listener);
subscribe(`${accessTokenChannelId}`, listener);
subscribe(`${systemChannelId}`, listener);
subscriptions[accessTokenChannelId] = {
channelName: 'system',