Change account search to match by text when opted-in (#25599)

Co-authored-by: Eugen Rochko <eugen@zeonfederated.com>
This commit is contained in:
jsgoldstein 2023-06-29 07:05:21 -04:00 committed by GitHub
parent 285a691936
commit 4581a528f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 90 additions and 29 deletions

View File

@ -2,8 +2,37 @@
class AccountsIndex < Chewy::Index class AccountsIndex < Chewy::Index
settings index: { refresh_interval: '30s' }, analysis: { settings index: { refresh_interval: '30s' }, analysis: {
filter: {
english_stop: {
type: 'stop',
stopwords: '_english_',
},
english_stemmer: {
type: 'stemmer',
language: 'english',
},
english_possessive_stemmer: {
type: 'stemmer',
language: 'possessive_english',
},
},
analyzer: { analyzer: {
content: { natural: {
tokenizer: 'uax_url_email',
filter: %w(
english_possessive_stemmer
lowercase
asciifolding
cjk_width
english_stop
english_stemmer
),
},
verbatim: {
tokenizer: 'whitespace', tokenizer: 'whitespace',
filter: %w(lowercase asciifolding cjk_width), filter: %w(lowercase asciifolding cjk_width),
}, },
@ -26,18 +55,13 @@ class AccountsIndex < Chewy::Index
index_scope ::Account.searchable.includes(:account_stat) index_scope ::Account.searchable.includes(:account_stat)
root date_detection: false do root date_detection: false do
field :id, type: 'long' field(:id, type: 'long')
field(:following_count, type: 'long')
field :display_name, type: 'text', analyzer: 'content' do field(:followers_count, type: 'long')
field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content' field(:properties, type: 'keyword', value: ->(account) { account.searchable_properties })
end field(:last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at })
field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
field :acct, type: 'text', analyzer: 'content', value: ->(account) { [account.username, account.domain].compact.join('@') } do field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content' field(:text, type: 'text', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
end
field :following_count, type: 'long', value: ->(account) { account.following_count }
field :followers_count, type: 'long', value: ->(account) { account.followers_count }
field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
end end
end end

View File

@ -106,6 +106,17 @@ module AccountSearch
LIMIT :limit OFFSET :offset LIMIT :limit OFFSET :offset
SQL SQL
def searchable_text
PlainTextFormatter.new(note, local?).to_s if discoverable?
end
def searchable_properties
[].tap do |properties|
properties << 'bot' if bot?
properties << 'verified' if fields.any?(&:verified?)
end
end
class_methods do class_methods do
def search_for(terms, limit: 10, offset: 0) def search_for(terms, limit: 10, offset: 0)
tsquery = generate_query_for_search(terms) tsquery = generate_query_for_search(terms)

View File

@ -9,12 +9,11 @@ class AccountSearchService < BaseService
MIN_QUERY_LENGTH = 5 MIN_QUERY_LENGTH = 5
def call(query, account = nil, options = {}) def call(query, account = nil, options = {})
@acct_hint = query&.start_with?('@') @query = query&.strip&.gsub(/\A@/, '')
@query = query&.strip&.gsub(/\A@/, '') @limit = options[:limit].to_i
@limit = options[:limit].to_i @offset = options[:offset].to_i
@offset = options[:offset].to_i @options = options
@options = options @account = account
@account = account
search_service_results.compact.uniq search_service_results.compact.uniq
end end
@ -72,8 +71,8 @@ class AccountSearchService < BaseService
end end
def from_elasticsearch def from_elasticsearch
must_clauses = [{ multi_match: { query: terms_for_query, fields: likely_acct? ? %w(acct.edge_ngram acct) : %w(acct.edge_ngram acct display_name.edge_ngram display_name), type: 'most_fields', operator: 'and' } }] must_clauses = must_clause
should_clauses = [] should_clauses = should_clause
if account if account
return [] if options[:following] && following_ids.empty? return [] if options[:following] && following_ids.empty?
@ -88,7 +87,7 @@ class AccountSearchService < BaseService
query = { bool: { must: must_clauses, should: should_clauses } } query = { bool: { must: must_clauses, should: should_clauses } }
functions = [reputation_score_function, followers_score_function, time_distance_function] functions = [reputation_score_function, followers_score_function, time_distance_function]
records = AccountsIndex.query(function_score: { query: query, functions: functions, boost_mode: 'multiply', score_mode: 'avg' }) records = AccountsIndex.query(function_score: { query: query, functions: functions })
.limit(limit_for_non_exact_results) .limit(limit_for_non_exact_results)
.offset(offset) .offset(offset)
.objects .objects
@ -133,6 +132,36 @@ class AccountSearchService < BaseService
} }
end end
def must_clause
fields = %w(username username.* display_name display_name.*)
fields << 'text' << 'text.*' if options[:use_searchable_text]
[
{
multi_match: {
query: terms_for_query,
fields: fields,
type: 'best_fields',
operator: 'or',
},
},
]
end
def should_clause
[
{
multi_match: {
query: terms_for_query,
fields: %w(username username.* display_name display_name.*),
type: 'best_fields',
operator: 'and',
boost: 10,
},
},
]
end
def following_ids def following_ids
@following_ids ||= account.active_relationships.pluck(:target_account_id) + [account.id] @following_ids ||= account.active_relationships.pluck(:target_account_id) + [account.id]
end end
@ -182,8 +211,4 @@ class AccountSearchService < BaseService
def username_complete? def username_complete?
query.include?('@') && "@#{query}".match?(MENTION_ONLY_RE) query.include?('@') && "@#{query}".match?(MENTION_ONLY_RE)
end end
def likely_acct?
@acct_hint || username_complete?
end
end end

View File

@ -30,7 +30,8 @@ class SearchService < BaseService
@account, @account,
limit: @limit, limit: @limit,
resolve: @resolve, resolve: @resolve,
offset: @offset offset: @offset,
use_searchable_text: true
) )
end end

View File

@ -68,7 +68,7 @@ describe SearchService, type: :service do
allow(AccountSearchService).to receive(:new).and_return(service) allow(AccountSearchService).to receive(:new).and_return(service)
results = subject.call(query, nil, 10) results = subject.call(query, nil, 10)
expect(service).to have_received(:call).with(query, nil, limit: 10, offset: 0, resolve: false) expect(service).to have_received(:call).with(query, nil, limit: 10, offset: 0, resolve: false, use_searchable_text: true)
expect(results).to eq empty_results.merge(accounts: [account]) expect(results).to eq empty_results.merge(accounts: [account])
end end
end end