From 465ee7792ff48905088efdf3df6f718081b5e244 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 7 Apr 2022 18:06:15 +0200
Subject: Fix pagination header on empty trends responses in REST API (#17986)

---
 app/models/trends/query.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/trends/query.rb b/app/models/trends/query.rb
index 231b65228..f19df162f 100644
--- a/app/models/trends/query.rb
+++ b/app/models/trends/query.rb
@@ -59,7 +59,7 @@ class Trends::Query
     @records
   end
 
-  delegate :each, :empty?, :first, :last, to: :records
+  delegate :each, :empty?, :first, :last, :size, to: :records
 
   def to_ary
     records.dup
-- 
cgit 


From fd9a9b07c2cd19ef08d15e138fd3fc59fc5318b6 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 8 Apr 2022 17:10:53 +0200
Subject: Fix trends returning less results per page when filtered in REST API
 (#17996)

- Change filtering and pagination to occur in SQL instead of Redis
- Change rank/score displayed on trends in admin UI to be locale-specific
---
 app/models/trends/base.rb                          |  8 ++++----
 app/models/trends/query.rb                         | 11 ++++++----
 app/models/trends/statuses.rb                      | 24 +++++-----------------
 .../admin/trends/links/_preview_card.html.haml     |  4 ++--
 app/views/admin/trends/statuses/_status.html.haml  |  4 ++--
 5 files changed, 20 insertions(+), 31 deletions(-)

(limited to 'app/models')

diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb
index 7ed13228d..38a49246b 100644
--- a/app/models/trends/base.rb
+++ b/app/models/trends/base.rb
@@ -37,12 +37,12 @@ class Trends::Base
     Trends::Query.new(key_prefix, klass)
   end
 
-  def score(id)
-    redis.zscore("#{key_prefix}:all", id) || 0
+  def score(id, locale: nil)
+    redis.zscore([key_prefix, 'all', locale].compact.join(':'), id) || 0
   end
 
-  def rank(id)
-    redis.zrevrank("#{key_prefix}:allowed", id)
+  def rank(id, locale: nil)
+    redis.zrevrank([key_prefix, 'allowed', locale].compact.join(':'), id)
   end
 
   def currently_trending_ids(allowed, limit)
diff --git a/app/models/trends/query.rb b/app/models/trends/query.rb
index f19df162f..cd5571bc6 100644
--- a/app/models/trends/query.rb
+++ b/app/models/trends/query.rb
@@ -14,8 +14,8 @@ class Trends::Query
     @records = []
     @loaded  = false
     @allowed = false
-    @limit   = -1
-    @offset  = 0
+    @limit   = nil
+    @offset  = nil
   end
 
   def allowed!
@@ -73,7 +73,10 @@ class Trends::Query
     if tmp_ids.empty?
       klass.none
     else
-      klass.joins("join unnest(array[#{tmp_ids.join(',')}]) with ordinality as x (id, ordering) on #{klass.table_name}.id = x.id").reorder('x.ordering')
+      scope = klass.joins("join unnest(array[#{tmp_ids.join(',')}]) with ordinality as x (id, ordering) on #{klass.table_name}.id = x.id").reorder('x.ordering')
+      scope = scope.offset(@offset) if @offset.present?
+      scope = scope.limit(@limit) if @limit.present?
+      scope
     end
   end
 
@@ -93,7 +96,7 @@ class Trends::Query
   end
 
   def ids
-    redis.zrevrange(key, @offset, @limit.positive? ? @limit - 1 : @limit).map(&:to_i)
+    redis.zrevrange(key, 0, -1).map(&:to_i)
   end
 
   def perform_queries
diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb
index e785413ec..dc5309554 100644
--- a/app/models/trends/statuses.rb
+++ b/app/models/trends/statuses.rb
@@ -22,25 +22,11 @@ class Trends::Statuses < Trends::Base
     private
 
     def apply_scopes(scope)
-      scope.includes(:account)
-    end
-
-    def perform_queries
-      return super if @account.nil?
-
-      statuses        = super
-      account_ids     = statuses.map(&:account_id)
-      account_domains = statuses.map(&:account_domain)
-
-      preloaded_relations = {
-        blocking: Account.blocking_map(account_ids, @account.id),
-        blocked_by: Account.blocked_by_map(account_ids, @account.id),
-        muting: Account.muting_map(account_ids, @account.id),
-        following: Account.following_map(account_ids, @account.id),
-        domain_blocking_by_domain: Account.domain_blocking_map_by_domain(account_domains, @account.id),
-      }
-
-      statuses.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? }
+      if @account.nil?
+        scope
+      else
+        scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account)
+      end
     end
   end
 
diff --git a/app/views/admin/trends/links/_preview_card.html.haml b/app/views/admin/trends/links/_preview_card.html.haml
index 2e6a0c62f..7d4897c7e 100644
--- a/app/views/admin/trends/links/_preview_card.html.haml
+++ b/app/views/admin/trends/links/_preview_card.html.haml
@@ -18,9 +18,9 @@
 
       = t('admin.trends.links.shared_by_over_week', count: preview_card.history.reduce(0) { |sum, day| sum + day.accounts })
 
-      - if preview_card.trendable? && (rank = Trends.links.rank(preview_card.id))
+      - if preview_card.trendable? && (rank = Trends.links.rank(preview_card.id, locale: params[:locale].presence))
         •
-        %abbr{ title: t('admin.trends.tags.current_score', score: Trends.links.score(preview_card.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
+        %abbr{ title: t('admin.trends.tags.current_score', score: Trends.links.score(preview_card.id, locale: params[:locale].presence)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
 
         - if preview_card.decaying?
           •
diff --git a/app/views/admin/trends/statuses/_status.html.haml b/app/views/admin/trends/statuses/_status.html.haml
index 0c463c6b1..e4d75bbb9 100644
--- a/app/views/admin/trends/statuses/_status.html.haml
+++ b/app/views/admin/trends/statuses/_status.html.haml
@@ -25,9 +25,9 @@
     - if status.trendable? && !status.account.discoverable?
       •
       = t('admin.trends.statuses.not_discoverable')
-    - if status.trendable? && (rank = Trends.statuses.rank(status.id))
+    - if status.trendable? && (rank = Trends.statuses.rank(status.id, locale: params[:locale].presence))
       •
-      %abbr{ title: t('admin.trends.tags.current_score', score: Trends.statuses.score(status.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
+      %abbr{ title: t('admin.trends.tags.current_score', score: Trends.statuses.score(status.id, locale: params[:locale].presence)) }= t('admin.trends.tags.trending_rank', rank: rank + 1)
     - elsif status.requires_review?
       •
       = t('admin.trends.pending_review')
-- 
cgit 


From 8e20e16cf030fef48850df4764bbf13a317f6b0c Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 8 Apr 2022 18:03:31 +0200
Subject: Change e-mail notifications to only be sent when recipient is offline
 (#17984)

* Change e-mail notifications to only be sent when recipient is offline

Change the default for follow and mention notifications back on

* Add preference to always send e-mail notifications

* Change wording
---
 app/controllers/settings/preferences_controller.rb |  3 +-
 app/lib/user_settings_decorator.rb                 |  5 ++
 app/models/user.rb                                 |  2 +-
 app/services/notify_service.rb                     | 63 ++++++++++++++--------
 .../preferences/notifications/show.html.haml       |  3 ++
 config/locales/simple_form.en.yml                  |  2 +
 config/settings.yml                                |  5 +-
 7 files changed, 58 insertions(+), 25 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb
index c7492700c..bfe651bc6 100644
--- a/app/controllers/settings/preferences_controller.rb
+++ b/app/controllers/settings/preferences_controller.rb
@@ -54,7 +54,8 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_use_pending_items,
       :setting_trends,
       :setting_crop_images,
-      notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag),
+      :setting_always_send_emails,
+      notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag appeal),
       interactions: %i(must_be_follower must_be_following must_be_following_dm)
     )
   end
diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb
index de054e403..5fb7655a9 100644
--- a/app/lib/user_settings_decorator.rb
+++ b/app/lib/user_settings_decorator.rb
@@ -38,6 +38,7 @@ class UserSettingsDecorator
     user.settings['use_pending_items']   = use_pending_items_preference if change?('setting_use_pending_items')
     user.settings['trends']              = trends_preference if change?('setting_trends')
     user.settings['crop_images']         = crop_images_preference if change?('setting_crop_images')
+    user.settings['always_send_emails']  = always_send_emails_preference if change?('setting_always_send_emails')
   end
 
   def merged_notification_emails
@@ -132,6 +133,10 @@ class UserSettingsDecorator
     boolean_cast_setting 'setting_crop_images'
   end
 
+  def always_send_emails_preference
+    boolean_cast_setting 'setting_always_send_emails'
+  end
+
   def boolean_cast_setting(key)
     ActiveModel::Type::Boolean.new.cast(settings[key])
   end
diff --git a/app/models/user.rb b/app/models/user.rb
index d19fe2c92..9da7b2639 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -130,7 +130,7 @@ class User < ApplicationRecord
            :reduce_motion, :system_font_ui, :noindex, :theme, :display_media,
            :expand_spoilers, :default_language, :aggregate_reblogs, :show_application,
            :advanced_layout, :use_blurhash, :use_pending_items, :trends, :crop_images,
-           :disable_swiping,
+           :disable_swiping, :always_send_emails,
            to: :settings, prefix: :setting, allow_nil: false
 
   attr_reader :invite_code
diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb
index a90f17cfd..d30b33876 100644
--- a/app/services/notify_service.rb
+++ b/app/services/notify_service.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class NotifyService < BaseService
+  include Redisable
+
   def call(recipient, type, activity)
     @recipient    = recipient
     @activity     = activity
@@ -8,10 +10,15 @@ class NotifyService < BaseService
 
     return if recipient.user.nil? || blocked?
 
-    create_notification!
+    @notification.save!
+
+    # It's possible the underlying activity has been deleted
+    # between the save call and now
+    return if @notification.activity.nil?
+
     push_notification!
     push_to_conversation! if direct_message?
-    send_email! if email_enabled?
+    send_email! if email_needed?
   rescue ActiveRecord::RecordInvalid
     nil
   end
@@ -92,8 +99,8 @@ class NotifyService < BaseService
   end
 
   def blocked?
-    blocked   = @recipient.suspended?                            # Skip if the recipient account is suspended anyway
-    blocked ||= from_self? && @notification.type != :poll        # Skip for interactions with self
+    blocked   = @recipient.suspended?
+    blocked ||= from_self? && @notification.type != :poll
 
     return blocked if message? && from_staff?
 
@@ -117,38 +124,52 @@ class NotifyService < BaseService
     end
   end
 
-  def create_notification!
-    @notification.save!
+  def push_notification!
+    push_to_streaming_api! if subscribed_to_streaming_api?
+    push_to_web_push_subscriptions!
   end
 
-  def push_notification!
-    return if @notification.activity.nil?
+  def push_to_streaming_api!
+    redis.publish("timeline:#{@recipient.id}:notifications", Oj.dump(event: :notification, payload: InlineRenderer.render(@notification, @recipient, :notification)))
+  end
 
-    Redis.current.publish("timeline:#{@recipient.id}:notifications", Oj.dump(event: :notification, payload: InlineRenderer.render(@notification, @recipient, :notification)))
-    send_push_notifications!
+  def subscribed_to_streaming_api?
+    redis.exists?("subscribed:timeline:#{@recipient.id}") || redis.exists?("subscribed:timeline:#{@recipient.id}:notifications")
   end
 
   def push_to_conversation!
-    return if @notification.activity.nil?
     AccountConversation.add_status(@recipient, @notification.target_status)
   end
 
-  def send_push_notifications!
-    subscriptions_ids = ::Web::PushSubscription.where(user_id: @recipient.user.id)
-                                               .select { |subscription| subscription.pushable?(@notification) }
-                                               .map(&:id)
+  def push_to_web_push_subscriptions!
+    ::Web::PushNotificationWorker.push_bulk(web_push_subscriptions.select { |subscription| subscription.pushable?(@notification) }) { |subscription| [subscription.id, @notification.id] }
+  end
 
-    ::Web::PushNotificationWorker.push_bulk(subscriptions_ids) do |subscription_id|
-      [subscription_id, @notification.id]
-    end
+  def web_push_subscriptions
+    @web_push_subscriptions ||= ::Web::PushSubscription.where(user_id: @recipient.user.id).to_a
+  end
+
+  def subscribed_to_web_push?
+    web_push_subscriptions.any?
   end
 
   def send_email!
-    return if @notification.activity.nil?
-    NotificationMailer.public_send(@notification.type, @recipient, @notification).deliver_later(wait: 2.minutes)
+    NotificationMailer.public_send(@notification.type, @recipient, @notification).deliver_later(wait: 2.minutes) if NotificationMailer.respond_to?(@notification.type)
+  end
+
+  def email_needed?
+    (!recipient_online? || always_send_emails?) && send_email_for_notification_type?
+  end
+
+  def recipient_online?
+    subscribed_to_streaming_api? || subscribed_to_web_push?
+  end
+
+  def always_send_emails?
+    @recipient.user.settings.always_send_emails
   end
 
-  def email_enabled?
+  def send_email_for_notification_type?
     @recipient.user.settings.notification_emails[@notification.type.to_s]
   end
 end
diff --git a/app/views/settings/preferences/notifications/show.html.haml b/app/views/settings/preferences/notifications/show.html.haml
index 223e5d740..42754a852 100644
--- a/app/views/settings/preferences/notifications/show.html.haml
+++ b/app/views/settings/preferences/notifications/show.html.haml
@@ -25,6 +25,9 @@
         = ff.input :pending_account, as: :boolean, wrapper: :with_label
         = ff.input :trending_tag, as: :boolean, wrapper: :with_label
 
+  .fields-group
+    = f.input :setting_always_send_emails, as: :boolean, wrapper: :with_label
+
   .fields-group
     = f.simple_fields_for :notification_emails, hash_to_object(current_user.settings.notification_emails) do |ff|
       = ff.input :digest, as: :boolean, wrapper: :with_label
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index b19b7891f..b784b1da7 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -49,6 +49,7 @@ en:
         phrase: Will be matched regardless of casing in text or content warning of a post
         scopes: Which APIs the application will be allowed to access. If you select a top-level scope, you don't need to select individual ones.
         setting_aggregate_reblogs: Do not show new boosts for posts that have been recently boosted (only affects newly-received boosts)
+        setting_always_send_emails: Normally e-mail notifications won't be sent when you are actively using Mastodon
         setting_default_sensitive: Sensitive media is hidden by default and can be revealed with a click
         setting_display_media_default: Hide media marked as sensitive
         setting_display_media_hide_all: Always hide media
@@ -151,6 +152,7 @@ en:
         phrase: Keyword or phrase
         setting_advanced_layout: Enable advanced web interface
         setting_aggregate_reblogs: Group boosts in timelines
+        setting_always_send_emails: Always send e-mail notifications
         setting_auto_play_gif: Auto-play animated GIFs
         setting_boost_modal: Show confirmation dialog before boosting
         setting_crop_images: Crop images in non-expanded posts to 16x9
diff --git a/config/settings.yml b/config/settings.yml
index 06dd2b3f3..eaa05071e 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -38,16 +38,17 @@ defaults: &defaults
   trendable_by_default: false
   crop_images: true
   notification_emails:
-    follow: false
+    follow: true
     reblog: false
     favourite: false
-    mention: false
+    mention: true
     follow_request: true
     digest: true
     report: true
     pending_account: true
     trending_tag: true
     appeal: true
+  always_send_emails: false
   interactions:
     must_be_follower: false
     must_be_following: false
-- 
cgit 


From 68273a7c6d6c630b6c88764579580682e12eebce Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 8 Apr 2022 19:35:31 +0200
Subject: Fix dangling language-specific trends (#17997)

- Change score half-life for trending statuses from 2 to 6 hours
- Change score threshold for trimming old items from 1 to 0.3
---
 app/models/trends/base.rb     | 4 ++--
 app/models/trends/links.rb    | 5 +++--
 app/models/trends/statuses.rb | 7 ++++---
 3 files changed, 9 insertions(+), 7 deletions(-)

(limited to 'app/models')

diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb
index 38a49246b..200f8d635 100644
--- a/app/models/trends/base.rb
+++ b/app/models/trends/base.rb
@@ -65,8 +65,8 @@ class Trends::Base
   end
 
   def trim_older_items
-    redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1')
-    redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1')
+    redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(0.3')
+    redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(0.3')
   end
 
   def score_at_rank(rank)
diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb
index 62308e706..5f046643a 100644
--- a/app/models/trends/links.rb
+++ b/app/models/trends/links.rb
@@ -30,7 +30,6 @@ class Trends::Links < Trends::Base
   def refresh(at_time = Time.now.utc)
     preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
     calculate_scores(preview_cards, at_time)
-    trim_older_items
   end
 
   def request_review
@@ -101,6 +100,8 @@ class Trends::Links < Trends::Base
       })
     end
 
+    trim_older_items
+
     # Clean up localized sets by calculating the intersection with the main
     # set. We do this instead of just deleting the localized sets to avoid
     # having moments where the API returns empty results
@@ -108,7 +109,7 @@ class Trends::Links < Trends::Base
     redis.pipelined do
       Trends.available_locales.each do |locale|
         redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
-        redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
+        redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:allowed"], aggregate: 'max')
       end
     end
   end
diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb
index dc5309554..d9b07446e 100644
--- a/app/models/trends/statuses.rb
+++ b/app/models/trends/statuses.rb
@@ -6,7 +6,7 @@ class Trends::Statuses < Trends::Base
   self.default_options = {
     threshold: 5,
     review_threshold: 3,
-    score_halflife: 2.hours.freeze,
+    score_halflife: 6.hours.freeze,
   }
 
   class Query < Trends::Query
@@ -48,7 +48,6 @@ class Trends::Statuses < Trends::Base
   def refresh(at_time = Time.now.utc)
     statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments)
     calculate_scores(statuses, at_time)
-    trim_older_items
   end
 
   def request_review
@@ -111,13 +110,15 @@ class Trends::Statuses < Trends::Base
         })
       end
 
+      trim_older_items
+
       # Clean up localized sets by calculating the intersection with the main
       # set. We do this instead of just deleting the localized sets to avoid
       # having moments where the API returns empty results
 
       Trends.available_locales.each do |locale|
         redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
-        redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max')
+        redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:allowed"], aggregate: 'max')
       end
     end
   end
-- 
cgit 


From dc503472477000597f2ff1b6a5c37e8ac4dd2d8c Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Sat, 9 Apr 2022 20:11:06 +0200
Subject: Fix crash in alias settings page (#18004)

---
 app/models/account_alias.rb | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'app/models')

diff --git a/app/models/account_alias.rb b/app/models/account_alias.rb
index 3d659142a..b421c66e2 100644
--- a/app/models/account_alias.rb
+++ b/app/models/account_alias.rb
@@ -28,6 +28,11 @@ class AccountAlias < ApplicationRecord
     super(val.start_with?('@') ? val[1..-1] : val)
   end
 
+  def pretty_acct
+    username, domain = acct.split('@')
+    domain.nil? ? username : "#{username}@#{Addressable::IDNA.to_unicode(domain)}"
+  end
+
   private
 
   def set_uri
-- 
cgit 


From d353bb5ee395bbf65da608b2c5427e655786fb97 Mon Sep 17 00:00:00 2001
From: Jeong Arm <kjwonmail@gmail.com>
Date: Wed, 13 Apr 2022 00:52:14 +0900
Subject: Implement infinity home timeline (#1610)

* Implement infinity home timeline

* Fix test for infinite home timeline

* Fix infinity home timeline with min_id

* Fix infinite home timeline duplicated statuses

* Codeclimate for infinite home timeline

* Refactor code as reviewed

* Fix redis sufficient check

* Fix typo on variable name
---
 app/models/home_feed.rb       | 37 +++++++++++++++++++++++++++++++++++++
 spec/models/home_feed_spec.rb | 18 ++++++++++++++----
 2 files changed, 51 insertions(+), 4 deletions(-)

(limited to 'app/models')

diff --git a/app/models/home_feed.rb b/app/models/home_feed.rb
index d6ebb5fa6..08f421c9c 100644
--- a/app/models/home_feed.rb
+++ b/app/models/home_feed.rb
@@ -9,4 +9,41 @@ class HomeFeed < Feed
   def regenerating?
     redis.exists?("account:#{@account.id}:regeneration")
   end
+
+  def get(limit, max_id = nil, since_id = nil, min_id = nil)
+    limit    = limit.to_i
+    max_id   = max_id.to_i if max_id.present?
+    since_id = since_id.to_i if since_id.present?
+    min_id   = min_id.to_i if min_id.present?
+
+    statuses = from_redis(limit, max_id, since_id, min_id)
+
+    return statuses if statuses.size >= limit
+
+    redis_min_id = from_redis(1, nil, nil, 0).first&.id if min_id.present? || since_id.present?
+    redis_sufficient = redis_min_id && (
+      (min_id.present? && min_id >= redis_min_id) ||
+      (since_id.present? && since_id >= redis_min_id)
+    )
+
+    unless redis_sufficient
+      remaining_limit = limit - statuses.size
+      max_id = statuses.last.id unless statuses.empty?
+      statuses += from_database(remaining_limit, max_id, since_id, min_id)
+    end
+
+    statuses
+  end
+
+  protected
+
+  def from_database(limit, max_id, since_id, min_id)
+    # Note that this query will not contains direct messages
+    Status
+      .where(account: [@account] + @account.following)
+      .where(visibility: [:public, :unlisted, :private])
+      .to_a_paginated_by_id(limit, min_id: min_id, max_id: max_id, since_id: since_id)
+      .reject { |status| FeedManager.instance.filter?(:home, status, @account) }
+      .sort_by { |status| -status.id }
+  end
 end
diff --git a/spec/models/home_feed_spec.rb b/spec/models/home_feed_spec.rb
index ee7a83960..179459e53 100644
--- a/spec/models/home_feed_spec.rb
+++ b/spec/models/home_feed_spec.rb
@@ -21,12 +21,17 @@ RSpec.describe HomeFeed, type: :model do
         )
       end
 
-      it 'gets statuses with ids in the range from redis' do
+      it 'gets statuses with ids in the range from redis with database' do
         results = subject.get(3)
 
-        expect(results.map(&:id)).to eq [3, 2]
+        expect(results.map(&:id)).to eq [3, 2, 1]
         expect(results.first.attributes.keys).to eq %w(id updated_at)
       end
+
+      it 'with min_id present' do
+        results = subject.get(3, nil, nil, 0)
+        expect(results.map(&:id)).to eq [3, 2, 1]
+      end
     end
 
     context 'when feed is being generated' do
@@ -34,10 +39,15 @@ RSpec.describe HomeFeed, type: :model do
         Redis.current.set("account:#{account.id}:regeneration", true)
       end
 
-      it 'returns nothing' do
+      it 'returns from database' do
         results = subject.get(3)
 
-        expect(results.map(&:id)).to eq []
+        expect(results.map(&:id)).to eq [10, 3, 2]
+      end
+
+      it 'with min_id present' do
+        results = subject.get(3, nil, nil, 0)
+        expect(results.map(&:id)).to eq [3, 2, 1]
       end
     end
   end
-- 
cgit 


From 5db3a14388cf780364b213c63aaf97b6f444ca17 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 14 Apr 2022 17:44:21 +0200
Subject: Fix loading Home TL from database not respecting `min_id` and not
 including DMs (#1744)

* Rework tests

* Add tests

* Fix HomeFeed#get with min_id fetching from database

* Minor code cleanup and optimizations

* Add tests

* Take DMs into account when fetching home TL from database

* Fix not listing own DMs in Home timeline

* Add tests

* Please CodeClimate
---
 app/models/home_feed.rb       | 43 ++++++++++++++++---------
 spec/models/home_feed_spec.rb | 73 +++++++++++++++++++++++++++++++++++++------
 2 files changed, 92 insertions(+), 24 deletions(-)

(limited to 'app/models')

diff --git a/app/models/home_feed.rb b/app/models/home_feed.rb
index 08f421c9c..447445092 100644
--- a/app/models/home_feed.rb
+++ b/app/models/home_feed.rb
@@ -16,34 +16,47 @@ class HomeFeed < Feed
     since_id = since_id.to_i if since_id.present?
     min_id   = min_id.to_i if min_id.present?
 
-    statuses = from_redis(limit, max_id, since_id, min_id)
+    if min_id.present?
+      redis_min_id = fetch_min_redis_id
+      return from_redis(limit, max_id, since_id, min_id) if redis_min_id && min_id >= redis_min_id
 
-    return statuses if statuses.size >= limit
+      statuses = from_database(limit, redis_min_id, since_id, min_id)
+      return statuses if statuses.size >= limit
 
-    redis_min_id = from_redis(1, nil, nil, 0).first&.id if min_id.present? || since_id.present?
-    redis_sufficient = redis_min_id && (
-      (min_id.present? && min_id >= redis_min_id) ||
-      (since_id.present? && since_id >= redis_min_id)
-    )
+      remaining_limit = limit - statuses.size
+      min_id = statuses.first.id unless statuses.empty?
+      from_redis(remaining_limit, max_id, since_id, min_id) + statuses
+    else
+      statuses = from_redis(limit, max_id, since_id, min_id)
+      return statuses if statuses.size >= limit
+
+      if since_id.present?
+        redis_min_id = fetch_min_redis_id
+        return statuses if redis_min_id.present? && since_id >= redis_min_id
+      end
 
-    unless redis_sufficient
       remaining_limit = limit - statuses.size
       max_id = statuses.last.id unless statuses.empty?
-      statuses += from_database(remaining_limit, max_id, since_id, min_id)
+      statuses + from_database(remaining_limit, max_id, since_id, min_id)
     end
-
-    statuses
   end
 
   protected
 
   def from_database(limit, max_id, since_id, min_id)
-    # Note that this query will not contains direct messages
-    Status
-      .where(account: [@account] + @account.following)
-      .where(visibility: [:public, :unlisted, :private])
+    scope = Status.where(account: @account.following)
+    scope = scope.left_outer_joins(:mentions)
+    scope = scope.where(visibility: %i(public unlisted private)).or(scope.where(mentions: { account_id: @account.id })).group(Status.arel_table[:id])
+    scope = scope.or(Status.where(account: @account))
+    scope
       .to_a_paginated_by_id(limit, min_id: min_id, max_id: max_id, since_id: since_id)
       .reject { |status| FeedManager.instance.filter?(:home, status, @account) }
       .sort_by { |status| -status.id }
   end
+
+  private
+
+  def fetch_min_redis_id
+    redis.zrangebyscore(key, '(0', '(+inf', limit: [0, 1], with_scores: true).first&.first&.to_i
+  end
 end
diff --git a/spec/models/home_feed_spec.rb b/spec/models/home_feed_spec.rb
index 179459e53..565357d7b 100644
--- a/spec/models/home_feed_spec.rb
+++ b/spec/models/home_feed_spec.rb
@@ -1,33 +1,83 @@
 require 'rails_helper'
 
 RSpec.describe HomeFeed, type: :model do
-  let(:account) { Fabricate(:account) }
+  let(:account)  { Fabricate(:account) }
+  let(:followed) { Fabricate(:account) }
+  let(:other)    { Fabricate(:account) }
 
   subject { described_class.new(account) }
 
   describe '#get' do
     before do
-      Fabricate(:status, account: account, id: 1)
-      Fabricate(:status, account: account, id: 2)
-      Fabricate(:status, account: account, id: 3)
-      Fabricate(:status, account: account, id: 10)
+      account.follow!(followed)
+
+      Fabricate(:status, account: account,  id: 1)
+      Fabricate(:status, account: account,  id: 2)
+      status = Fabricate(:status, account: followed, id: 3)
+      Fabricate(:mention, account: account, status: status)
+      Fabricate(:status, account: account,  id: 10)
+      Fabricate(:status, account: other,    id: 11)
+      Fabricate(:status, account: followed, id: 12, visibility: :private)
+      Fabricate(:status, account: followed, id: 13, visibility: :direct)
+      Fabricate(:status, account: account,  id: 14, visibility: :direct)
+      dm = Fabricate(:status, account: followed, id: 15, visibility: :direct)
+      Fabricate(:mention, account: account, status: dm)
     end
 
     context 'when feed is generated' do
       before do
+        FeedManager.instance.populate_home(account)
+
+        # Add direct messages because populate_home does not do that
         Redis.current.zadd(
           FeedManager.instance.key(:home, account.id),
-          [[4, 4], [3, 3], [2, 2], [1, 1]]
+          [[14, 14], [15, 15]]
         )
       end
 
       it 'gets statuses with ids in the range from redis with database' do
-        results = subject.get(3)
+        results = subject.get(5)
+
+        expect(results.map(&:id)).to eq [15, 14, 12, 10, 3]
+        expect(results.first.attributes.keys).to eq %w(id updated_at)
+      end
+
+      it 'with since_id present' do
+        results = subject.get(5, nil, 3, nil)
+        expect(results.map(&:id)).to eq [15, 14, 12, 10]
+      end
 
+      it 'with min_id present' do
+        results = subject.get(3, nil, nil, 0)
         expect(results.map(&:id)).to eq [3, 2, 1]
+      end
+    end
+
+    context 'when feed is only partial' do
+      before do
+        FeedManager.instance.populate_home(account)
+
+        # Add direct messages because populate_home does not do that
+        Redis.current.zadd(
+          FeedManager.instance.key(:home, account.id),
+          [[14, 14], [15, 15]]
+        )
+
+        Redis.current.zremrangebyrank(FeedManager.instance.key(:home, account.id), 0, -2)
+      end
+
+      it 'gets statuses with ids in the range from redis with database' do
+        results = subject.get(5)
+
+        expect(results.map(&:id)).to eq [15, 14, 12, 10, 3]
         expect(results.first.attributes.keys).to eq %w(id updated_at)
       end
 
+      it 'with since_id present' do
+        results = subject.get(5, nil, 3, nil)
+        expect(results.map(&:id)).to eq [15, 14, 12, 10]
+      end
+
       it 'with min_id present' do
         results = subject.get(3, nil, nil, 0)
         expect(results.map(&:id)).to eq [3, 2, 1]
@@ -40,9 +90,14 @@ RSpec.describe HomeFeed, type: :model do
       end
 
       it 'returns from database' do
-        results = subject.get(3)
+        results = subject.get(5)
+
+        expect(results.map(&:id)).to eq [15, 14, 12, 10, 3]
+      end
 
-        expect(results.map(&:id)).to eq [10, 3, 2]
+      it 'with since_id present' do
+        results = subject.get(5, nil, 3, nil)
+        expect(results.map(&:id)).to eq [15, 14, 12, 10]
       end
 
       it 'with min_id present' do
-- 
cgit 


From ec4a8d81b141c46a6fa967f8416724d0162cd6c7 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Sat, 16 Apr 2022 21:20:07 +0200
Subject: Revert DM support in HomeFeed#from_database

Fixes #1746

Queries could get prohibitively expensive.
---
 app/models/home_feed.rb       |  6 ++----
 spec/models/home_feed_spec.rb | 39 ++++++++++++---------------------------
 2 files changed, 14 insertions(+), 31 deletions(-)

(limited to 'app/models')

diff --git a/app/models/home_feed.rb b/app/models/home_feed.rb
index 447445092..53550b7db 100644
--- a/app/models/home_feed.rb
+++ b/app/models/home_feed.rb
@@ -44,10 +44,8 @@ class HomeFeed < Feed
   protected
 
   def from_database(limit, max_id, since_id, min_id)
-    scope = Status.where(account: @account.following)
-    scope = scope.left_outer_joins(:mentions)
-    scope = scope.where(visibility: %i(public unlisted private)).or(scope.where(mentions: { account_id: @account.id })).group(Status.arel_table[:id])
-    scope = scope.or(Status.where(account: @account))
+    scope = Status.where(account: @account).or(Status.where(account: @account.following))
+    scope = scope.where(visibility: %i(public unlisted private))
     scope
       .to_a_paginated_by_id(limit, min_id: min_id, max_id: max_id, since_id: since_id)
       .reject { |status| FeedManager.instance.filter?(:home, status, @account) }
diff --git a/spec/models/home_feed_spec.rb b/spec/models/home_feed_spec.rb
index 565357d7b..20756633c 100644
--- a/spec/models/home_feed_spec.rb
+++ b/spec/models/home_feed_spec.rb
@@ -19,32 +19,23 @@ RSpec.describe HomeFeed, type: :model do
       Fabricate(:status, account: other,    id: 11)
       Fabricate(:status, account: followed, id: 12, visibility: :private)
       Fabricate(:status, account: followed, id: 13, visibility: :direct)
-      Fabricate(:status, account: account,  id: 14, visibility: :direct)
-      dm = Fabricate(:status, account: followed, id: 15, visibility: :direct)
-      Fabricate(:mention, account: account, status: dm)
     end
 
     context 'when feed is generated' do
       before do
         FeedManager.instance.populate_home(account)
-
-        # Add direct messages because populate_home does not do that
-        Redis.current.zadd(
-          FeedManager.instance.key(:home, account.id),
-          [[14, 14], [15, 15]]
-        )
       end
 
       it 'gets statuses with ids in the range from redis with database' do
-        results = subject.get(5)
+        results = subject.get(3)
 
-        expect(results.map(&:id)).to eq [15, 14, 12, 10, 3]
+        expect(results.map(&:id)).to eq [12, 10, 3]
         expect(results.first.attributes.keys).to eq %w(id updated_at)
       end
 
       it 'with since_id present' do
-        results = subject.get(5, nil, 3, nil)
-        expect(results.map(&:id)).to eq [15, 14, 12, 10]
+        results = subject.get(3, nil, 3, nil)
+        expect(results.map(&:id)).to eq [12, 10]
       end
 
       it 'with min_id present' do
@@ -57,25 +48,19 @@ RSpec.describe HomeFeed, type: :model do
       before do
         FeedManager.instance.populate_home(account)
 
-        # Add direct messages because populate_home does not do that
-        Redis.current.zadd(
-          FeedManager.instance.key(:home, account.id),
-          [[14, 14], [15, 15]]
-        )
-
         Redis.current.zremrangebyrank(FeedManager.instance.key(:home, account.id), 0, -2)
       end
 
       it 'gets statuses with ids in the range from redis with database' do
-        results = subject.get(5)
+        results = subject.get(3)
 
-        expect(results.map(&:id)).to eq [15, 14, 12, 10, 3]
+        expect(results.map(&:id)).to eq [12, 10, 3]
         expect(results.first.attributes.keys).to eq %w(id updated_at)
       end
 
       it 'with since_id present' do
-        results = subject.get(5, nil, 3, nil)
-        expect(results.map(&:id)).to eq [15, 14, 12, 10]
+        results = subject.get(3, nil, 3, nil)
+        expect(results.map(&:id)).to eq [12, 10]
       end
 
       it 'with min_id present' do
@@ -90,14 +75,14 @@ RSpec.describe HomeFeed, type: :model do
       end
 
       it 'returns from database' do
-        results = subject.get(5)
+        results = subject.get(3)
 
-        expect(results.map(&:id)).to eq [15, 14, 12, 10, 3]
+        expect(results.map(&:id)).to eq [12, 10, 3]
       end
 
       it 'with since_id present' do
-        results = subject.get(5, nil, 3, nil)
-        expect(results.map(&:id)).to eq [15, 14, 12, 10]
+        results = subject.get(3, nil, 3, nil)
+        expect(results.map(&:id)).to eq [12, 10]
       end
 
       it 'with min_id present' do
-- 
cgit 


From 5c808ee0decac075b105e1f3a36933ce38d76255 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Tue, 19 Apr 2022 14:54:43 +0200
Subject: Revert support from loading Home timeline from database

Unfortunately, the database query could turn out very inefficient and I did not
manage to find a way to improve that. Furthermore, there were still behavior
inconsistencies between fetching the timeline from Redis and fetching it from
Postgres.
---
 app/models/home_feed.rb       | 48 ---------------------------
 spec/models/home_feed_spec.rb | 76 ++++++++-----------------------------------
 2 files changed, 13 insertions(+), 111 deletions(-)

(limited to 'app/models')

diff --git a/app/models/home_feed.rb b/app/models/home_feed.rb
index 53550b7db..d6ebb5fa6 100644
--- a/app/models/home_feed.rb
+++ b/app/models/home_feed.rb
@@ -9,52 +9,4 @@ class HomeFeed < Feed
   def regenerating?
     redis.exists?("account:#{@account.id}:regeneration")
   end
-
-  def get(limit, max_id = nil, since_id = nil, min_id = nil)
-    limit    = limit.to_i
-    max_id   = max_id.to_i if max_id.present?
-    since_id = since_id.to_i if since_id.present?
-    min_id   = min_id.to_i if min_id.present?
-
-    if min_id.present?
-      redis_min_id = fetch_min_redis_id
-      return from_redis(limit, max_id, since_id, min_id) if redis_min_id && min_id >= redis_min_id
-
-      statuses = from_database(limit, redis_min_id, since_id, min_id)
-      return statuses if statuses.size >= limit
-
-      remaining_limit = limit - statuses.size
-      min_id = statuses.first.id unless statuses.empty?
-      from_redis(remaining_limit, max_id, since_id, min_id) + statuses
-    else
-      statuses = from_redis(limit, max_id, since_id, min_id)
-      return statuses if statuses.size >= limit
-
-      if since_id.present?
-        redis_min_id = fetch_min_redis_id
-        return statuses if redis_min_id.present? && since_id >= redis_min_id
-      end
-
-      remaining_limit = limit - statuses.size
-      max_id = statuses.last.id unless statuses.empty?
-      statuses + from_database(remaining_limit, max_id, since_id, min_id)
-    end
-  end
-
-  protected
-
-  def from_database(limit, max_id, since_id, min_id)
-    scope = Status.where(account: @account).or(Status.where(account: @account.following))
-    scope = scope.where(visibility: %i(public unlisted private))
-    scope
-      .to_a_paginated_by_id(limit, min_id: min_id, max_id: max_id, since_id: since_id)
-      .reject { |status| FeedManager.instance.filter?(:home, status, @account) }
-      .sort_by { |status| -status.id }
-  end
-
-  private
-
-  def fetch_min_redis_id
-    redis.zrangebyscore(key, '(0', '(+inf', limit: [0, 1], with_scores: true).first&.first&.to_i
-  end
 end
diff --git a/spec/models/home_feed_spec.rb b/spec/models/home_feed_spec.rb
index 20756633c..ee7a83960 100644
--- a/spec/models/home_feed_spec.rb
+++ b/spec/models/home_feed_spec.rb
@@ -1,72 +1,32 @@
 require 'rails_helper'
 
 RSpec.describe HomeFeed, type: :model do
-  let(:account)  { Fabricate(:account) }
-  let(:followed) { Fabricate(:account) }
-  let(:other)    { Fabricate(:account) }
+  let(:account) { Fabricate(:account) }
 
   subject { described_class.new(account) }
 
   describe '#get' do
     before do
-      account.follow!(followed)
-
-      Fabricate(:status, account: account,  id: 1)
-      Fabricate(:status, account: account,  id: 2)
-      status = Fabricate(:status, account: followed, id: 3)
-      Fabricate(:mention, account: account, status: status)
-      Fabricate(:status, account: account,  id: 10)
-      Fabricate(:status, account: other,    id: 11)
-      Fabricate(:status, account: followed, id: 12, visibility: :private)
-      Fabricate(:status, account: followed, id: 13, visibility: :direct)
+      Fabricate(:status, account: account, id: 1)
+      Fabricate(:status, account: account, id: 2)
+      Fabricate(:status, account: account, id: 3)
+      Fabricate(:status, account: account, id: 10)
     end
 
     context 'when feed is generated' do
       before do
-        FeedManager.instance.populate_home(account)
+        Redis.current.zadd(
+          FeedManager.instance.key(:home, account.id),
+          [[4, 4], [3, 3], [2, 2], [1, 1]]
+        )
       end
 
-      it 'gets statuses with ids in the range from redis with database' do
+      it 'gets statuses with ids in the range from redis' do
         results = subject.get(3)
 
-        expect(results.map(&:id)).to eq [12, 10, 3]
+        expect(results.map(&:id)).to eq [3, 2]
         expect(results.first.attributes.keys).to eq %w(id updated_at)
       end
-
-      it 'with since_id present' do
-        results = subject.get(3, nil, 3, nil)
-        expect(results.map(&:id)).to eq [12, 10]
-      end
-
-      it 'with min_id present' do
-        results = subject.get(3, nil, nil, 0)
-        expect(results.map(&:id)).to eq [3, 2, 1]
-      end
-    end
-
-    context 'when feed is only partial' do
-      before do
-        FeedManager.instance.populate_home(account)
-
-        Redis.current.zremrangebyrank(FeedManager.instance.key(:home, account.id), 0, -2)
-      end
-
-      it 'gets statuses with ids in the range from redis with database' do
-        results = subject.get(3)
-
-        expect(results.map(&:id)).to eq [12, 10, 3]
-        expect(results.first.attributes.keys).to eq %w(id updated_at)
-      end
-
-      it 'with since_id present' do
-        results = subject.get(3, nil, 3, nil)
-        expect(results.map(&:id)).to eq [12, 10]
-      end
-
-      it 'with min_id present' do
-        results = subject.get(3, nil, nil, 0)
-        expect(results.map(&:id)).to eq [3, 2, 1]
-      end
     end
 
     context 'when feed is being generated' do
@@ -74,20 +34,10 @@ RSpec.describe HomeFeed, type: :model do
         Redis.current.set("account:#{account.id}:regeneration", true)
       end
 
-      it 'returns from database' do
+      it 'returns nothing' do
         results = subject.get(3)
 
-        expect(results.map(&:id)).to eq [12, 10, 3]
-      end
-
-      it 'with since_id present' do
-        results = subject.get(3, nil, 3, nil)
-        expect(results.map(&:id)).to eq [12, 10]
-      end
-
-      it 'with min_id present' do
-        results = subject.get(3, nil, nil, 0)
-        expect(results.map(&:id)).to eq [3, 2, 1]
+        expect(results.map(&:id)).to eq []
       end
     end
   end
-- 
cgit 


From ea383278160bcee81477e86a95107d4a2dc315ef Mon Sep 17 00:00:00 2001
From: Jeong Arm <kjwonmail@gmail.com>
Date: Sun, 24 Apr 2022 04:47:27 +0900
Subject: Let votes statuses are also searchable (#18070)

---
 app/chewy/statuses_index.rb | 5 +++++
 app/models/status.rb        | 2 ++
 2 files changed, 7 insertions(+)

(limited to 'app/models')

diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb
index 1381a96ed..1304aeedb 100644
--- a/app/chewy/statuses_index.rb
+++ b/app/chewy/statuses_index.rb
@@ -55,6 +55,11 @@ class StatusesIndex < Chewy::Index
     data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
   end
 
+  crutch :votes do |collection|
+    data = ::PollVote.joins(:poll).where(poll: { status_id: collection.map(&:id) }).where(account: Account.local).pluck(:status_id, :account_id)
+    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+  end
+
   root date_detection: false do
     field :id, type: 'long'
     field :account_id, type: 'long'
diff --git a/app/models/status.rb b/app/models/status.rb
index a71451f50..288d374fd 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -145,11 +145,13 @@ class Status < ApplicationRecord
       ids += favourites.where(account: Account.local).pluck(:account_id)
       ids += reblogs.where(account: Account.local).pluck(:account_id)
       ids += bookmarks.where(account: Account.local).pluck(:account_id)
+      ids += poll.votes.where(account: Account.local).pluck(:account_id) if poll.present?
     else
       ids += preloaded.mentions[id] || []
       ids += preloaded.favourites[id] || []
       ids += preloaded.reblogs[id] || []
       ids += preloaded.bookmarks[id] || []
+      ids += preloaded.votes[id] || []
     end
 
     ids.uniq
-- 
cgit