From 9bd23dc4e51ba47283a8e3a66cd94b4e624a5235 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 27 May 2018 21:45:30 +0200 Subject: Track trending tags (#7638) * Track trending tags - Half-life of 1 day - Historical usage in daily buckets (last 7 days stored) - GET /api/v1/trends Fix #271 * Add trends to web UI * Don't render compose form on search route, adjust search results header * Disqualify tag from trends if it's in disallowed hashtags setting * Count distinct accounts using tag, ignore silenced accounts --- app/models/tag.rb | 16 ++++++++++++ app/models/trending_tags.rb | 61 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 app/models/trending_tags.rb (limited to 'app/models') diff --git a/app/models/tag.rb b/app/models/tag.rb index 8b1b02412..4f31f796e 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -21,6 +21,22 @@ class Tag < ApplicationRecord name end + def history + days = [] + + 7.times do |i| + day = i.days.ago.beginning_of_day.to_i + + days << { + day: day.to_s, + uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0', + accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s, + } + end + + days + end + class << self def search_for(term, limit = 5) pattern = sanitize_sql_like(term.strip) + '%' diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb new file mode 100644 index 000000000..eedd92644 --- /dev/null +++ b/app/models/trending_tags.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +class TrendingTags + KEY = 'trending_tags' + HALF_LIFE = 1.day.to_i + MAX_ITEMS = 500 + EXPIRE_HISTORY_AFTER = 7.days.seconds + + class << self + def record_use!(tag, account, at_time = Time.now.utc) + return if disallowed_hashtags.include?(tag.name) || account.silenced? + + increment_vote!(tag.id, at_time) + increment_historical_use!(tag.id, at_time) + increment_unique_use!(tag.id, account.id, at_time) + end + + def get(limit) + tag_ids = redis.zrevrange(KEY, 0, limit).map(&:to_i) + tags = Tag.where(id: tag_ids).to_a.map { |tag| [tag.id, tag] }.to_h + tag_ids.map { |tag_id| tags[tag_id] }.compact + end + + private + + def increment_vote!(tag_id, at_time) + redis.zincrby(KEY, (2**((at_time.to_i - epoch) / HALF_LIFE)).to_f, tag_id.to_s) + redis.zremrangebyrank(KEY, 0, -MAX_ITEMS) if rand < (2.to_f / MAX_ITEMS) + end + + def increment_historical_use!(tag_id, at_time) + key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}" + redis.incrby(key, 1) + redis.expire(key, EXPIRE_HISTORY_AFTER) + end + + def increment_unique_use!(tag_id, account_id, at_time) + key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts" + redis.pfadd(key, account_id) + redis.expire(key, EXPIRE_HISTORY_AFTER) + end + + # The epoch needs to be 2.5 years in the future if the half-life is one day + # While dynamic, it will always be the same within one year + def epoch + @epoch ||= Date.new(Date.current.year + 2.5, 10, 1).to_datetime.to_i + end + + def disallowed_hashtags + return @disallowed_hashtags if defined?(@disallowed_hashtags) + + @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags + @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String + @disallowed_hashtags = @disallowed_hashtags.map(&:downcase) + end + + def redis + Redis.current + end + end +end -- cgit From b87a1229c74aebec02cd08091f25613f6dff4428 Mon Sep 17 00:00:00 2001 From: tateisu Date: Mon, 28 May 2018 18:04:06 +0900 Subject: optimize direct timeline (#7614) * optimize direct timeline * fix typo in class name * change filter condition for direct timeline * fix codestyle issue * revoke index_accounts_not_silenced because direct timeline does not use it. * revoke index_accounts_not_silenced because direct timeline does not use it. * fix rspec test condition. * fix rspec test condition. * fix rspec test condition. * revoke adding column and partial index * (direct timeline) move merging logic to model * fix pagination parameter * add method arguments that switches return array of status or cache_ids * fix order by * returns ActiveRecord.Relation in default behavor * fix codestyle issue --- .../api/v1/timelines/direct_controller.rb | 15 +++++--- app/models/status.rb | 43 +++++++++++++++++++--- spec/models/status_spec.rb | 23 ++++++------ 3 files changed, 59 insertions(+), 22 deletions(-) (limited to 'app/models') diff --git a/app/controllers/api/v1/timelines/direct_controller.rb b/app/controllers/api/v1/timelines/direct_controller.rb index d455227eb..ef64078be 100644 --- a/app/controllers/api/v1/timelines/direct_controller.rb +++ b/app/controllers/api/v1/timelines/direct_controller.rb @@ -23,15 +23,18 @@ class Api::V1::Timelines::DirectController < Api::BaseController end def direct_statuses - direct_timeline_statuses.paginate_by_max_id( - limit_param(DEFAULT_STATUSES_LIMIT), - params[:max_id], - params[:since_id] - ) + direct_timeline_statuses end def direct_timeline_statuses - Status.as_direct_timeline(current_account) + # this query requires built in pagination. + Status.as_direct_timeline( + current_account, + limit_param(DEFAULT_STATUSES_LIMIT), + params[:max_id], + params[:since_id], + true # returns array of cache_ids object + ) end def insert_pagination_headers diff --git a/app/models/status.rb b/app/models/status.rb index 853e75b43..54f3f68f5 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -188,12 +188,45 @@ class Status < ApplicationRecord where(account: [account] + account.following).where(visibility: [:public, :unlisted, :private]) end - def as_direct_timeline(account) - query = joins("LEFT OUTER JOIN mentions ON statuses.id = mentions.status_id AND mentions.account_id = #{account.id}") - .where("mentions.account_id = #{account.id} OR statuses.account_id = #{account.id}") - .where(visibility: [:direct]) + def as_direct_timeline(account, limit = 20, max_id = nil, since_id = nil, cache_ids = false) + # direct timeline is mix of direct message from_me and to_me. + # 2 querys are executed with pagination. + # constant expression using arel_table is required for partial index + + # _from_me part does not require any timeline filters + query_from_me = where(account_id: account.id) + .where(Status.arel_table[:visibility].eq(3)) + .limit(limit) + .order('statuses.id DESC') + + # _to_me part requires mute and block filter. + # FIXME: may we check mutes.hide_notifications? + query_to_me = Status + .joins(:mentions) + .merge(Mention.where(account_id: account.id)) + .where(Status.arel_table[:visibility].eq(3)) + .limit(limit) + .order('mentions.status_id DESC') + .not_excluded_by_account(account) + + if max_id.present? + query_from_me = query_from_me.where('statuses.id < ?', max_id) + query_to_me = query_to_me.where('mentions.status_id < ?', max_id) + end + + if since_id.present? + query_from_me = query_from_me.where('statuses.id > ?', since_id) + query_to_me = query_to_me.where('mentions.status_id > ?', since_id) + end - apply_timeline_filters(query, account, false) + if cache_ids + # returns array of cache_ids object that have id and updated_at + (query_from_me.cache_ids.to_a + query_to_me.cache_ids.to_a).uniq(&:id).sort_by(&:id).reverse.take(limit) + else + # returns ActiveRecord.Relation + items = (query_from_me.select(:id).to_a + query_to_me.select(:id).to_a).uniq(&:id).sort_by(&:id).reverse.take(limit) + Status.where(id: items.map(&:id)) + end end def as_public_timeline(account = nil, local_only = false) diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb index c6701018e..aee4f49b4 100644 --- a/spec/models/status_spec.rb +++ b/spec/models/status_spec.rb @@ -154,7 +154,7 @@ RSpec.describe Status, type: :model do describe '#target' do it 'returns nil if the status is self-contained' do - expect(subject.target).to be_nil + expect(subject.target).to be_nil end it 'returns nil if the status is a reply' do @@ -333,24 +333,25 @@ RSpec.describe Status, type: :model do expect(@results).to_not include(@followed_public_status) end - it 'includes direct statuses mentioning recipient from followed' do - Fabricate(:mention, account: account, status: @followed_direct_status) - expect(@results).to include(@followed_direct_status) - end - it 'does not include direct statuses not mentioning recipient from followed' do expect(@results).to_not include(@followed_direct_status) end - it 'includes direct statuses mentioning recipient from non-followed' do - Fabricate(:mention, account: account, status: @not_followed_direct_status) - expect(@results).to include(@not_followed_direct_status) - end - it 'does not include direct statuses not mentioning recipient from non-followed' do expect(@results).to_not include(@not_followed_direct_status) end + it 'includes direct statuses mentioning recipient from followed' do + Fabricate(:mention, account: account, status: @followed_direct_status) + results2 = Status.as_direct_timeline(account) + expect(results2).to include(@followed_direct_status) + end + + it 'includes direct statuses mentioning recipient from non-followed' do + Fabricate(:mention, account: account, status: @not_followed_direct_status) + results2 = Status.as_direct_timeline(account) + expect(results2).to include(@not_followed_direct_status) + end end describe '.as_public_timeline' do -- cgit From e599d7caf2642c7143616e8402b7d730d32c349d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 29 May 2018 01:39:02 +0200 Subject: Rescue Mastodon::DimensionsValidationError in Remoteable (#7662) Fix #7660 --- app/models/concerns/remotable.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb index 20ddbca53..c17f04776 100644 --- a/app/models/concerns/remotable.rb +++ b/app/models/concerns/remotable.rb @@ -41,7 +41,7 @@ module Remotable rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" nil - rescue Paperclip::Error => e + rescue Paperclip::Error, Mastodon::DimensionsValidationError => e Rails.logger.debug "Error processing remote #{attachment_name}: #{e}" nil end -- cgit