about summary refs log tree commit diff
diff options
context:
space:
mode:
-rw-r--r--Gemfile3
-rw-r--r--Gemfile.lock4
-rw-r--r--ISSUE_TEMPLATE.md5
-rw-r--r--app/lib/feed_manager.rb83
-rw-r--r--app/lib/inline_rabl_scope.rb17
-rw-r--r--app/services/precompute_feed_service.rb8
-rw-r--r--app/workers/processing_worker.rb2
-rw-r--r--app/workers/regeneration_worker.rb2
-rw-r--r--app/workers/salmon_worker.rb2
-rw-r--r--docs/Running-Mastodon/Heroku-guide.md2
-rw-r--r--docs/Using-Mastodon/List-of-Mastodon-instances.md17
-rw-r--r--docs/Using-Mastodon/User-guide.md6
12 files changed, 97 insertions, 54 deletions
diff --git a/Gemfile b/Gemfile
index 46baed307..4c6314763 100644
--- a/Gemfile
+++ b/Gemfile
@@ -38,7 +38,7 @@ gem 'rqrcode'
 gem 'twitter-text'
 gem 'oj'
 gem 'hiredis'
-gem 'redis', '~>3.2'
+gem 'redis', '~>3.2', require: ['redis', 'redis/connection/hiredis']
 gem 'fast_blank'
 gem 'htmlentities'
 gem 'simple_form'
@@ -46,6 +46,7 @@ gem 'will_paginate'
 gem 'rack-attack'
 gem 'rack-cors', require: 'rack/cors'
 gem 'sidekiq'
+gem 'sidekiq-unique-jobs'
 gem 'rails-settings-cached'
 gem 'simple-navigation'
 gem 'statsd-instrument'
diff --git a/Gemfile.lock b/Gemfile.lock
index 6e3115249..26c7b9962 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -387,6 +387,9 @@ GEM
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
       redis (~> 3.2, >= 3.2.1)
+    sidekiq-unique-jobs (4.0.18)
+      sidekiq (>= 2.6)
+      thor
     simple-navigation (4.0.3)
       activesupport (>= 2.3.2)
     simple_form (3.2.1)
@@ -510,6 +513,7 @@ DEPENDENCIES
   sass-rails (~> 5.0)
   sdoc (~> 0.4.0)
   sidekiq
+  sidekiq-unique-jobs
   simple-navigation
   simple_form
   simplecov
diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md
new file mode 100644
index 000000000..8394b2424
--- /dev/null
+++ b/ISSUE_TEMPLATE.md
@@ -0,0 +1,5 @@
+[Issue text goes here].
+
+* * * *
+
+- [ ] I searched or browsed the repo’s other issues to ensure this is not a duplicate.
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index cd6ca1291..a2efcce10 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -5,7 +5,7 @@ require 'singleton'
 class FeedManager
   include Singleton
 
-  MAX_ITEMS = 800
+  MAX_ITEMS = 400
 
   def key(type, id)
     "feed:#{type}:#{id}"
@@ -50,10 +50,18 @@ class FeedManager
 
   def merge_into_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
+    query        = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
 
-    from_account.statuses.limit(MAX_ITEMS).each do |status|
-      next if status.direct_visibility? || filter?(:home, status, into_account)
-      redis.zadd(timeline_key, status.id, status.id)
+    if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
+      oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
+      query = query.where('id > ?', oldest_home_score)
+    end
+
+    redis.pipelined do
+      query.each do |status|
+        next if status.direct_visibility? || filter?(:home, status, into_account)
+        redis.zadd(timeline_key, status.id, status.id)
+      end
     end
 
     trim(:home, into_account.id)
@@ -61,31 +69,20 @@ class FeedManager
 
   def unmerge_from_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
-
-    from_account.statuses.select('id').find_each do |status|
-      redis.zrem(timeline_key, status.id)
-      redis.zremrangebyscore(timeline_key, status.id, status.id)
+    oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
+
+    from_account.statuses.select('id').where('id > ?', oldest_home_score).find_in_batches do |statuses|
+      redis.pipelined do
+        statuses.each do |status|
+          redis.zrem(timeline_key, status.id)
+          redis.zremrangebyscore(timeline_key, status.id, status.id)
+        end
+      end
     end
   end
 
   def inline_render(target_account, template, object)
-    rabl_scope = Class.new do
-      include RoutingHelper
-
-      def initialize(account)
-        @account = account
-      end
-
-      def current_user
-        @account.try(:user)
-      end
-
-      def current_account
-        @account
-      end
-    end
-
-    Rabl::Renderer.new(template, object, view_path: 'app/views', format: :json, scope: rabl_scope.new(target_account)).render
+    Rabl::Renderer.new(template, object, view_path: 'app/views', format: :json, scope: InlineRablScope.new(target_account)).render
   end
 
   private
@@ -95,37 +92,39 @@ class FeedManager
   end
 
   def filter_from_home?(status, receiver)
-    return true if receiver.muting?(status.account)
+    return true if status.reply? && status.in_reply_to_id.nil?
+
+    check_for_mutes = [status.account_id]
+    check_for_mutes.concat([status.reblog.account_id]) if status.reblog?
+
+    return true if receiver.muting?(check_for_mutes)
+
+    check_for_blocks = status.mentions.map(&:account_id)
+    check_for_blocks.concat([status.reblog.account_id]) if status.reblog?
 
-    should_filter = false
+    return true if receiver.blocking?(check_for_blocks)
 
-    if status.reply? && status.in_reply_to_id.nil?
-      should_filter = true
-    elsif status.reply? && !status.in_reply_to_account_id.nil?                # Filter out if it's a reply
+    if status.reply? && !status.in_reply_to_account_id.nil?                   # Filter out if it's a reply
       should_filter   = !receiver.following?(status.in_reply_to_account)      # and I'm not following the person it's a reply to
       should_filter &&= !(receiver.id == status.in_reply_to_account_id)       # and it's not a reply to me
       should_filter &&= !(status.account_id == status.in_reply_to_account_id) # and it's not a self-reply
+      return should_filter
     elsif status.reblog?                                                      # Filter out a reblog
-      should_filter = receiver.blocking?(status.reblog.account)               # if I'm blocking the reblogged person
-      should_filter ||= receiver.muting?(status.reblog.account)               # or muting that person
-      should_filter ||= status.reblog.account.blocking?(receiver)             # or if the author of the reblogged status is blocking me
+      return status.reblog.account.blocking?(receiver)                        # or if the author of the reblogged status is blocking me
     end
 
-    should_filter ||= receiver.blocking?(status.mentions.map(&:account_id))   # or if it mentions someone I blocked
-
-    should_filter
+    false
   end
 
   def filter_from_mentions?(status, receiver)
+    check_for_blocks = [status.account_id]
+    check_for_blocks.concat(status.mentions.select('account_id').map(&:account_id))
+    check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil?
+
     should_filter   = receiver.id == status.account_id                                      # Filter if I'm mentioning myself
-    should_filter ||= receiver.blocking?(status.account)                                    # or it's from someone I blocked
-    should_filter ||= receiver.blocking?(status.mentions.includes(:account).map(&:account)) # or if it mentions someone I blocked
+    should_filter ||= receiver.blocking?(check_for_blocks)                                  # or it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked
     should_filter ||= (status.account.silenced? && !receiver.following?(status.account))    # of if the account is silenced and I'm not following them
 
-    if status.reply? && !status.in_reply_to_account_id.nil?                                 # or it's a reply
-      should_filter ||= receiver.blocking?(status.in_reply_to_account)                      # to a user I blocked
-    end
-
     should_filter
   end
 end
diff --git a/app/lib/inline_rabl_scope.rb b/app/lib/inline_rabl_scope.rb
new file mode 100644
index 000000000..26adcb03a
--- /dev/null
+++ b/app/lib/inline_rabl_scope.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class InlineRablScope
+  include RoutingHelper
+
+  def initialize(account)
+    @account = account
+  end
+
+  def current_user
+    @account.try(:user)
+  end
+
+  def current_account
+    @account
+  end
+end
diff --git a/app/services/precompute_feed_service.rb b/app/services/precompute_feed_service.rb
index e1ec56e8d..a57c401d0 100644
--- a/app/services/precompute_feed_service.rb
+++ b/app/services/precompute_feed_service.rb
@@ -5,9 +5,11 @@ class PrecomputeFeedService < BaseService
   # @param [Symbol] type :home or :mentions
   # @param [Account] account
   def call(_, account)
-    Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS).each do |status|
-      next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
-      redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
+    redis.pipelined do
+      Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS / 4).each do |status|
+        next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
+        redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
+      end
     end
   end
 
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 4a467d924..5df404bcc 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -3,7 +3,7 @@
 class ProcessingWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull', backtrace: true
+  sidekiq_options backtrace: true
 
   def perform(account_id, body)
     ProcessFeedService.new.call(body, Account.find(account_id))
diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb
index 82665b581..da8b845f6 100644
--- a/app/workers/regeneration_worker.rb
+++ b/app/workers/regeneration_worker.rb
@@ -3,7 +3,7 @@
 class RegenerationWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull', backtrace: true
+  sidekiq_options queue: 'pull', backtrace: true, unique: :until_executed
 
   def perform(account_id, _ = :home)
     PrecomputeFeedService.new.call(:home, Account.find(account_id))
diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb
index 2888b574b..fc95ce47f 100644
--- a/app/workers/salmon_worker.rb
+++ b/app/workers/salmon_worker.rb
@@ -3,7 +3,7 @@
 class SalmonWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull', backtrace: true
+  sidekiq_options backtrace: true
 
   def perform(account_id, body)
     ProcessInteractionService.new.call(body, Account.find(account_id))
diff --git a/docs/Running-Mastodon/Heroku-guide.md b/docs/Running-Mastodon/Heroku-guide.md
index 799b8a64c..b66e56200 100644
--- a/docs/Running-Mastodon/Heroku-guide.md
+++ b/docs/Running-Mastodon/Heroku-guide.md
@@ -8,6 +8,6 @@ Mastodon can theoretically run indefinitely on a free [Heroku](https://heroku.co
 1. Click the above button.
 2. Fill in the options requested.
   * You can use a .herokuapp.com domain, which will be simple to set up, or you can use a custom domain. If you want a custom domain and HTTPS, you will need to upgrade to a paid plan (to use Heroku's SSL features), or set up [CloudFlare](https://cloudflare.com) who offer free "Flexible SSL" (note: CloudFlare have some undefined limits on WebSockets. So far, no one has reported hitting concurrent connection limits).
-  * You will want Amazon S3 for file storage. The only exception is for development purposes, where you may not care if files are not saaved. Follow a guide online for creating a free Amazon S3 bucket and Access Key, then enter the details.
+  * You will want Amazon S3 for file storage. The only exception is for development purposes, where you may not care if files are not saved. Follow a guide online for creating a free Amazon S3 bucket and Access Key, then enter the details.
   * If you want your Mastodon to be able to send emails, configure SMTP settings here (or later). Consider using [Mailgun](https://mailgun.com) or similar, who offer free plans that should suit your interests.
 3. Deploy! The app should be set up, with a working web interface and database. You can change settings and manage versions from the Heroku dashboard.
diff --git a/docs/Using-Mastodon/List-of-Mastodon-instances.md b/docs/Using-Mastodon/List-of-Mastodon-instances.md
index 780977bd4..677ec7e56 100644
--- a/docs/Using-Mastodon/List-of-Mastodon-instances.md
+++ b/docs/Using-Mastodon/List-of-Mastodon-instances.md
@@ -11,16 +11,31 @@ There is also a list at [instances.mastodon.xyz](https://instances.mastodon.xyz)
 | [animalliberation.social](https://animalliberation.social) |Animal Rights|Yes|No|
 | [socially.constructed.space](https://socially.constructed.space) |Single user|No|No|
 | [epiktistes.com](https://epiktistes.com) |N/A|Yes|No|
+| [fern.surgeplay.com](https://fern.surgeplay.com) |Federates everywhere, Minecraft-focused|Yes|No
 | [gay.crime.team](https://gay.crime.team) |the place for doin' gay crime online (please don't actually do crime here)|Yes|No|
 | [icosahedron.website](https://icosahedron.website/) |Icosahedron-themed (well, visually), open registration.|Yes|No|
 | [memetastic.space](https://memetastic.space) |Memes|Yes|No|
 | [social.diskseven.com](https://social.diskseven.com) |Single user|No|No (DNS entry but no response)|
 | [social.gestaltzerfall.net](https://social.gestaltzerfall.net) |Single user|No|No|
 | [mastodon.xyz](https://mastodon.xyz) |N/A|Yes|Yes|
-| [social.targaryen.house](https://social.targaryen.house) |N/A|Yes|No|
+| [social.targaryen.house](https://social.targaryen.house) |Federates everywhere, quick updates.|Yes|Yes|
 | [social.mashek.net](https://social.mashek.net) |Themed and customised for Mashekstein Labs community. Selectively federates.|Yes|No|
 | [masto.themimitoof.fr](https://masto.themimitoof.fr) |N/A|Yes|Yes|
 | [social.imirhil.fr](https://social.imirhil.fr) |N/A|No|Yes|
 | [social.wxcafe.net](https://social.wxcafe.net) |Open registrations, federates everywhere, no moderation yet|Yes|Yes|
+| [octodon.social](https://octodon.social) |Open registrations, federates everywhere, cutest instance yet|Yes|Yes|
+| [hostux.social](https://hostux.social) |N/A|Yes|Yes|
+| [social.alex73630.xyz](https://social.alex73630.xyz) |Francophones|Yes|Yes|
+| [maly.io](https://maly.io) |N/A|Yes|No|
+| [social.lou.lt](https://social.lou.lt) |N/A|Yes|No|
+| [mastodon.ninetailed.uk](https://mastodon.ninetailed.uk) |N/A|Yes|No|
+| [soc.louiz.org](https://soc.louiz.org) |"Coucou"|Yes|No|
+| [7nw.eu](https://7nw.eu) |N/A|Yes|No|
+| [mastodon.gougere.fr](https://mastodon.gougere.fr)|N/A|Yes|No|
+| [aleph.land](https://aleph.land)|N/A|Yes|No|
+| [share.elouworld.org](https://share.elouworld.org)|N/A|No|No|
+| [social.lkw.tf](https://social.lkw.tf)|N/A|No|No|
+| [manowar.social](https://manowar.social)|N/A|No|No|
+| [social.ballpointcarrot.net](https://social.ballpointcarrot.net)|Down at time of entry|No|No|
 
 Let me know if you start running one so I can add it to the list! (Alternatively, add it yourself as a pull request).
diff --git a/docs/Using-Mastodon/User-guide.md b/docs/Using-Mastodon/User-guide.md
index f78921c6f..f8018909a 100644
--- a/docs/Using-Mastodon/User-guide.md
+++ b/docs/Using-Mastodon/User-guide.md
@@ -26,17 +26,17 @@ Mastodon User's Guide
 
 ## Intro
 
-Mastodon is a social network application based on the GNU Social protocol. It behaves a lot like other social networks, especially Twitter, with one key difference - it is open-source and anyone can start their own server (also called an "instance"), and users of any instance can interact freely with those of other instances (called "federation"). Thus, it is possible for small communities to set up their own servers to use amongst themselves while also allowing interaction with other communities.
+Mastodon is a social network application based on the GNU Social protocol. It behaves a lot like other social networks, especially Twitter, with one key difference - it is open-source and anyone can start their own server (also called an "*instance*"), and users of any instance can interact freely with those of other instances (called "*federation*"). Thus, it is possible for small communities to set up their own servers to use amongst themselves while also allowing interaction with other communities.
 
 #### Decentralization and Federation
 
-Mastodon is a system decentralized through a concept called "federation" - rather than depending on a single person or organization to run its infrastructure, anyone can download and run the software and run their own server. Federation means different Mastodon servers can interact with each other seamlessly, similar to e.g. e-mail.
+Mastodon is a system decentralized through a concept called "*federation*" - rather than depending on a single person or organization to run its infrastructure, anyone can download and run the software and run their own server. Federation means different Mastodon servers can interact with each other seamlessly, similar to e.g. e-mail.
 
 As such, anyone can download Mastodon and e.g. run it for a small community of people, but any user registered on that instance can follow and send and read posts from other Mastodon instances (as well as servers running other GNU Social-compatible services). This means that not only is users' data not inherently owned by a company with an interest in selling it to advertisers, but also that if any given server shuts down its users can set up a new one or migrate to another instance, rather than the entire service being lost.
 
 Within each Mastodon instance, usernames just appear as `@username`, similar to other services such as Twitter. Users from other instances appear, and can be searched for and followed, as `@user@servername.ext` - so e.g. `@gargron` on the `mastodon.social` instance can be followed from other instances as `@gargron@mastodon.social`).
 
-Posts from users on external instances are "federated" into the local one, i.e. if `user1@mastodon1` follows `user2@gnusocial2`, any posts `user2@gnusocial2` makes appear in both `user1@mastodon`'s Home feed and the public timeline on the `mastodon1` server. Mastodon server administrators have some control over this and can exclude users' posts from appearing on the public timeline; post privacy settings from users on Mastodon instances also affect this, see below in the [Toot Privacy](User-guide.md#toot-privacy) section.
+Posts from users on external instances are "*federated*" into the local one, i.e. if `user1@mastodon1` follows `user2@gnusocial2`, any posts `user2@gnusocial2` makes appear in both `user1@mastodon`'s Home feed and the public timeline on the `mastodon1` server. Mastodon server administrators have some control over this and can exclude users' posts from appearing on the public timeline; post privacy settings from users on Mastodon instances also affect this, see below in the [Toot Privacy](User-guide.md#toot-privacy) section.
 
 ## Getting Started