Change automatic post deletion thresholds and load detection (#24614)

This commit is contained in:
Claire 2023-04-21 18:14:19 +02:00 committed by GitHub
parent fbb4de3dbc
commit 501d6197c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 24 additions and 21 deletions

View File

@ -7,28 +7,30 @@ class Scheduler::AccountsStatusesCleanupScheduler
# This limit is mostly to be nice to the fediverse at large and not # This limit is mostly to be nice to the fediverse at large and not
# generate too much traffic. # generate too much traffic.
# This also helps limiting the running time of the scheduler itself. # This also helps limiting the running time of the scheduler itself.
MAX_BUDGET = 150 MAX_BUDGET = 300
# This is an attempt to spread the load across instances, as various # This is an attempt to spread the load across remote servers, as
# accounts are likely to have various followers. # spreading deletions across diverse accounts is likely to spread
# the deletion across diverse followers. It also helps each individual
# user see some effect sooner.
PER_ACCOUNT_BUDGET = 5 PER_ACCOUNT_BUDGET = 5
# This is an attempt to limit the workload generated by status removal # This is an attempt to limit the workload generated by status removal
# jobs to something the particular instance can handle. # jobs to something the particular server can handle.
PER_THREAD_BUDGET = 6 PER_THREAD_BUDGET = 5
# Those avoid loading an instance that is already under load # These are latency limits on various queues above which a server is
MAX_DEFAULT_SIZE = 200 # considered to be under load, causing the auto-deletion to be entirely
MAX_DEFAULT_LATENCY = 5 # skipped for that run.
MAX_PUSH_SIZE = 500 LOAD_LATENCY_THRESHOLDS = {
MAX_PUSH_LATENCY = 10 default: 5,
push: 10,
# 'pull' queue has lower priority jobs, and it's unlikely that pushing # The `pull` queue has lower priority jobs, and it's unlikely that
# deletes would cause much issues with this queue if it didn't cause issues # pushing deletes would cause much issues with this queue if it didn't
# with default and push. Yet, do not enqueue deletes if the instance is # cause issues with `default` and `push`. Yet, do not enqueue deletes
# lagging behind too much. # if the instance is lagging behind too much.
MAX_PULL_SIZE = 10_000 pull: 5.minutes.to_i,
MAX_PULL_LATENCY = 5.minutes.to_i }.freeze
sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
@ -62,19 +64,20 @@ class Scheduler::AccountsStatusesCleanupScheduler
end end
def compute_budget def compute_budget
# Each post deletion is a `RemovalWorker` job (on `default` queue), each
# potentially spawning many `ActivityPub::DeliveryWorker` jobs (on the `push` queue).
threads = Sidekiq::ProcessSet.new.select { |x| x['queues'].include?('push') }.pluck('concurrency').sum threads = Sidekiq::ProcessSet.new.select { |x| x['queues'].include?('push') }.pluck('concurrency').sum
[PER_THREAD_BUDGET * threads, MAX_BUDGET].min [PER_THREAD_BUDGET * threads, MAX_BUDGET].min
end end
def under_load? def under_load?
queue_under_load?('default', MAX_DEFAULT_SIZE, MAX_DEFAULT_LATENCY) || queue_under_load?('push', MAX_PUSH_SIZE, MAX_PUSH_LATENCY) || queue_under_load?('pull', MAX_PULL_SIZE, MAX_PULL_LATENCY) LOAD_LATENCY_THRESHOLDS.any? { |queue, max_latency| queue_under_load?(queue, max_latency) }
end end
private private
def queue_under_load?(name, max_size, max_latency) def queue_under_load?(name, max_latency)
queue = Sidekiq::Queue.new(name) Sidekiq::Queue.new(name).latency > max_latency
queue.size > max_size || queue.latency > max_latency
end end
def last_processed_id def last_processed_id