More duplicates in cli maintenance spec, misc bug fixes (#28449)

This commit is contained in:
Matt Jankowski 2023-12-21 03:51:03 -05:00 committed by GitHub
parent 01f0a6ca4f
commit 2463b53363
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 443 additions and 23 deletions

View File

@ -40,6 +40,10 @@ module Mastodon::CLI
class BulkImport < ApplicationRecord; end class BulkImport < ApplicationRecord; end
class SoftwareUpdate < ApplicationRecord; end class SoftwareUpdate < ApplicationRecord; end
class DomainBlock < ApplicationRecord
scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), domain')) }
end
class PreviewCard < ApplicationRecord class PreviewCard < ApplicationRecord
self.inheritance_column = false self.inheritance_column = false
end end
@ -249,19 +253,7 @@ module Mastodon::CLI
say 'Deduplicating user records…' say 'Deduplicating user records…'
# Deduplicating email deduplicate_users_process_email
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
ref_user = users.shift
say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
users.each_with_index do |user, index|
user.update!(email: "#{index} " + user.email)
end
end
deduplicate_users_process_confirmation_token deduplicate_users_process_confirmation_token
deduplicate_users_process_remember_token deduplicate_users_process_remember_token
deduplicate_users_process_password_token deduplicate_users_process_password_token
@ -280,6 +272,20 @@ module Mastodon::CLI
ActiveRecord::Base.connection.execute('REINDEX INDEX index_users_on_unconfirmed_email;') if migrator_version >= 2023_07_02_151753 ActiveRecord::Base.connection.execute('REINDEX INDEX index_users_on_unconfirmed_email;') if migrator_version >= 2023_07_02_151753
end end
def deduplicate_users_process_email
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
ref_user = users.shift
say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
users.each_with_index do |user, index|
user.update!(email: "#{index} " + user.email)
end
end
end
def deduplicate_users_process_confirmation_token def deduplicate_users_process_confirmation_token
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row| ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1) users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1)
@ -571,7 +577,7 @@ module Mastodon::CLI
say 'Deduplicating webhooks…' say 'Deduplicating webhooks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1").each do |row| ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1").each do |row|
Webhooks.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) Webhook.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end end
say 'Restoring webhooks indexes…' say 'Restoring webhooks indexes…'
@ -604,11 +610,7 @@ module Mastodon::CLI
say 'Please chose the one to keep unchanged, other ones will be automatically renamed.' say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'
ref_id = ask('Account to keep unchanged:') do |q| ref_id = ask('Account to keep unchanged:', required: true, default: 0).to_i
q.required true
q.default 0
q.convert :int
end
accounts.delete_at(ref_id) accounts.delete_at(ref_id)

View File

@ -62,6 +62,7 @@ describe Mastodon::CLI::Maintenance do
context 'with duplicate accounts' do context 'with duplicate accounts' do
before do before do
prepare_duplicate_data prepare_duplicate_data
choose_local_account_to_keep
end end
let(:duplicate_account_username) { 'username' } let(:duplicate_account_username) { 'username' }
@ -71,21 +72,37 @@ describe Mastodon::CLI::Maintenance do
expect { subject } expect { subject }
.to output_results( .to output_results(
'Deduplicating accounts', 'Deduplicating accounts',
'Multiple local accounts were found for',
'Restoring index_accounts_on_username_and_domain_lower', 'Restoring index_accounts_on_username_and_domain_lower',
'Reindexing textual indexes on accounts…', 'Reindexing textual indexes on accounts…',
'Finished!' 'Finished!'
) )
.and change(duplicate_accounts, :count).from(2).to(1) .and change(duplicate_remote_accounts, :count).from(2).to(1)
.and change(duplicate_local_accounts, :count).from(2).to(1)
end end
def duplicate_accounts def duplicate_remote_accounts
Account.where(username: duplicate_account_username, domain: duplicate_account_domain) Account.where(username: duplicate_account_username, domain: duplicate_account_domain)
end end
def duplicate_local_accounts
Account.where(username: duplicate_account_username, domain: nil)
end
def prepare_duplicate_data def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :accounts, name: :index_accounts_on_username_and_domain_lower ActiveRecord::Base.connection.remove_index :accounts, name: :index_accounts_on_username_and_domain_lower
Fabricate(:account, username: duplicate_account_username, domain: duplicate_account_domain) _remote_account = Fabricate(:account, username: duplicate_account_username, domain: duplicate_account_domain)
Fabricate.build(:account, username: duplicate_account_username, domain: duplicate_account_domain).save(validate: false) _remote_account_dupe = Fabricate.build(:account, username: duplicate_account_username, domain: duplicate_account_domain).save(validate: false)
_local_account = Fabricate(:account, username: duplicate_account_username, domain: nil)
_local_account_dupe = Fabricate.build(:account, username: duplicate_account_username, domain: nil).save(validate: false)
end
def choose_local_account_to_keep
allow(cli.shell)
.to receive(:ask)
.with(/Account to keep unchanged/, anything)
.and_return('0')
.once
end end
end end
@ -175,6 +192,407 @@ describe Mastodon::CLI::Maintenance do
end end
end end
context 'with duplicate account_domain_blocks' do
before do
prepare_duplicate_data
end
let(:duplicate_domain) { 'example.host' }
let(:account) { Fabricate(:account) }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Removing duplicate account domain blocks',
'Restoring account domain blocks indexes',
'Finished!'
)
.and change(duplicate_account_domain_blocks, :count).from(2).to(1)
end
def duplicate_account_domain_blocks
AccountDomainBlock.where(account: account, domain: duplicate_domain)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :account_domain_blocks, [:account_id, :domain]
Fabricate(:account_domain_block, account: account, domain: duplicate_domain)
Fabricate.build(:account_domain_block, account: account, domain: duplicate_domain).save(validate: false)
end
end
context 'with duplicate announcement_reactions' do
before do
prepare_duplicate_data
end
let(:account) { Fabricate(:account) }
let(:announcement) { Fabricate(:announcement) }
let(:name) { Fabricate(:custom_emoji).shortcode }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Removing duplicate announcement reactions',
'Restoring announcement_reactions indexes',
'Finished!'
)
.and change(duplicate_announcement_reactions, :count).from(2).to(1)
end
def duplicate_announcement_reactions
AnnouncementReaction.where(account: account, announcement: announcement, name: name)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :announcement_reactions, [:account_id, :announcement_id, :name]
Fabricate(:announcement_reaction, account: account, announcement: announcement, name: name)
Fabricate.build(:announcement_reaction, account: account, announcement: announcement, name: name).save(validate: false)
end
end
context 'with duplicate conversations' do
before do
prepare_duplicate_data
end
let(:uri) { 'https://example.host/path' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating conversations',
'Restoring conversations indexes',
'Finished!'
)
.and change(duplicate_conversations, :count).from(2).to(1)
end
def duplicate_conversations
Conversation.where(uri: uri)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :conversations, :uri
Fabricate(:conversation, uri: uri)
Fabricate.build(:conversation, uri: uri).save(validate: false)
end
end
context 'with duplicate custom_emojis' do
before do
prepare_duplicate_data
end
let(:duplicate_shortcode) { 'wowzers' }
let(:duplicate_domain) { 'example.host' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating custom_emojis',
'Restoring custom_emojis indexes',
'Finished!'
)
.and change(duplicate_custom_emojis, :count).from(2).to(1)
end
def duplicate_custom_emojis
CustomEmoji.where(shortcode: duplicate_shortcode, domain: duplicate_domain)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :custom_emojis, [:shortcode, :domain]
Fabricate(:custom_emoji, shortcode: duplicate_shortcode, domain: duplicate_domain)
Fabricate.build(:custom_emoji, shortcode: duplicate_shortcode, domain: duplicate_domain).save(validate: false)
end
end
context 'with duplicate custom_emoji_categories' do
before do
prepare_duplicate_data
end
let(:duplicate_name) { 'name_value' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating custom_emoji_categories',
'Restoring custom_emoji_categories indexes',
'Finished!'
)
.and change(duplicate_custom_emoji_categories, :count).from(2).to(1)
end
def duplicate_custom_emoji_categories
CustomEmojiCategory.where(name: duplicate_name)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :custom_emoji_categories, :name
Fabricate(:custom_emoji_category, name: duplicate_name)
Fabricate.build(:custom_emoji_category, name: duplicate_name).save(validate: false)
end
end
context 'with duplicate domain_allows' do
before do
prepare_duplicate_data
end
let(:domain) { 'example.host' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating domain_allows',
'Restoring domain_allows indexes',
'Finished!'
)
.and change(duplicate_domain_allows, :count).from(2).to(1)
end
def duplicate_domain_allows
DomainAllow.where(domain: domain)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :domain_allows, :domain
Fabricate(:domain_allow, domain: domain)
Fabricate.build(:domain_allow, domain: domain).save(validate: false)
end
end
context 'with duplicate domain_blocks' do
before do
prepare_duplicate_data
end
let(:domain) { 'example.host' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating domain_blocks',
'Restoring domain_blocks indexes',
'Finished!'
)
.and change(duplicate_domain_blocks, :count).from(2).to(1)
end
def duplicate_domain_blocks
DomainBlock.where(domain: domain)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :domain_blocks, :domain
Fabricate(:domain_block, domain: domain)
Fabricate.build(:domain_block, domain: domain).save(validate: false)
end
end
context 'with duplicate email_domain_blocks' do
before do
prepare_duplicate_data
end
let(:domain) { 'example.host' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating email_domain_blocks',
'Restoring email_domain_blocks indexes',
'Finished!'
)
.and change(duplicate_email_domain_blocks, :count).from(2).to(1)
end
def duplicate_email_domain_blocks
EmailDomainBlock.where(domain: domain)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :email_domain_blocks, :domain
Fabricate(:email_domain_block, domain: domain)
Fabricate.build(:email_domain_block, domain: domain).save(validate: false)
end
end
context 'with duplicate media_attachments' do
before do
prepare_duplicate_data
end
let(:shortcode) { 'codenam' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating media_attachments',
'Restoring media_attachments indexes',
'Finished!'
)
.and change(duplicate_media_attachments, :count).from(2).to(1)
end
def duplicate_media_attachments
MediaAttachment.where(shortcode: shortcode)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :media_attachments, :shortcode
Fabricate(:media_attachment, shortcode: shortcode)
Fabricate.build(:media_attachment, shortcode: shortcode).save(validate: false)
end
end
context 'with duplicate preview_cards' do
before do
prepare_duplicate_data
end
let(:url) { 'https://example.host/path' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating preview_cards',
'Restoring preview_cards indexes',
'Finished!'
)
.and change(duplicate_preview_cards, :count).from(2).to(1)
end
def duplicate_preview_cards
PreviewCard.where(url: url)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :preview_cards, :url
Fabricate(:preview_card, url: url)
Fabricate.build(:preview_card, url: url).save(validate: false)
end
end
context 'with duplicate statuses' do
before do
prepare_duplicate_data
end
let(:uri) { 'https://example.host/path' }
let(:account) { Fabricate(:account) }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating statuses',
'Restoring statuses indexes',
'Finished!'
)
.and change(duplicate_statuses, :count).from(2).to(1)
end
def duplicate_statuses
Status.where(uri: uri)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :statuses, :uri
Fabricate(:status, account: account, uri: uri)
duplicate = Fabricate.build(:status, account: account, uri: uri)
duplicate.save(validate: false)
Fabricate(:status_pin, account: account, status: duplicate)
Fabricate(:status, in_reply_to_id: duplicate.id)
Fabricate(:status, reblog_of_id: duplicate.id)
end
end
context 'with duplicate tags' do
before do
prepare_duplicate_data
end
let(:name) { 'tagname' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating tags',
'Restoring tags indexes',
'Finished!'
)
.and change(duplicate_tags, :count).from(2).to(1)
end
def duplicate_tags
Tag.where(name: name)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :tags, name: 'index_tags_on_name_lower_btree'
Fabricate(:tag, name: name)
Fabricate.build(:tag, name: name).save(validate: false)
end
end
context 'with duplicate webauthn_credentials' do
before do
prepare_duplicate_data
end
let(:external_id) { '123_123_123' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating webauthn_credentials',
'Restoring webauthn_credentials indexes',
'Finished!'
)
.and change(duplicate_webauthn_credentials, :count).from(2).to(1)
end
def duplicate_webauthn_credentials
WebauthnCredential.where(external_id: external_id)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :webauthn_credentials, :external_id
Fabricate(:webauthn_credential, external_id: external_id)
Fabricate.build(:webauthn_credential, external_id: external_id).save(validate: false)
end
end
context 'with duplicate webhooks' do
before do
prepare_duplicate_data
end
let(:url) { 'https://example.host/path' }
it 'runs the deduplication process' do
expect { subject }
.to output_results(
'Deduplicating webhooks',
'Restoring webhooks indexes',
'Finished!'
)
.and change(duplicate_webhooks, :count).from(2).to(1)
end
def duplicate_webhooks
Webhook.where(url: url)
end
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :webhooks, :url
Fabricate(:webhook, url: url)
Fabricate.build(:webhook, url: url).save(validate: false)
end
end
def agree_to_backup_warning def agree_to_backup_warning
allow(cli.shell) allow(cli.shell)
.to receive(:yes?) .to receive(:yes?)