From 8a45a97e2e1ed74983ce25bcf0e03f51ed2eb8fe Mon Sep 17 00:00:00 2001 From: shel Date: Mon, 3 Apr 2017 12:28:36 -0400 Subject: Add instances from instances.mastodon.xyz Updated list with lots of instances that have been added to instances.mastodon.xyz but not this list --- docs/Using-Mastodon/List-of-Mastodon-instances.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/Using-Mastodon/List-of-Mastodon-instances.md b/docs/Using-Mastodon/List-of-Mastodon-instances.md index 780977bd4..c424cab64 100644 --- a/docs/Using-Mastodon/List-of-Mastodon-instances.md +++ b/docs/Using-Mastodon/List-of-Mastodon-instances.md @@ -22,5 +22,18 @@ There is also a list at [instances.mastodon.xyz](https://instances.mastodon.xyz) | [masto.themimitoof.fr](https://masto.themimitoof.fr) |N/A|Yes|Yes| | [social.imirhil.fr](https://social.imirhil.fr) |N/A|No|Yes| | [social.wxcafe.net](https://social.wxcafe.net) |Open registrations, federates everywhere, no moderation yet|Yes|Yes| +| [hostux.social](https://hostux.social) |N/A|Yes|Yes| +| [social.alex73630.xyz](https://social.alex73630.xyz) |Francophones|Yes|Yes| +| [maly.io](https://maly.io) |N/A|Yes|No| +| [social.lou.lt](https://social.lou.lt) |N/A|Yes|No| +| [mastodon.ninetailed.uk](https://mastodon.ninetailed.uk) |N/A|Yes|No| +| [soc.louiz.org](https://soc.louiz.org) |"Coucou"|Yes|No| +| [7nw.eu](https://7nw.eu) |N/A|Yes|No| +| [mastodon.gougere.fr](https://mastodon.gougere.fr)|N/A|Yes|No| +| [aleph.land](https://aleph.land)|N/A|Yes|No| +| [share.elouworld.org](https://share.elouworld.org)|N/A|No|No| +| [social.lkw.tf](https://social.lkw.tf)|N/A|No|No| +| [manowar.social](https://manowar.social)|N/A|No|No| +| [social.ballpointcarrot.net](https://social.ballpointcarrot.net)|Down at time of entry|No|No| Let me know if you start running one so I can add it to the list! (Alternatively, add it yourself as a pull request). -- cgit From cc451e1fcbb7682c2f440e6cf624d0086fae0e11 Mon Sep 17 00:00:00 2001 From: Wonderfall Date: Mon, 3 Apr 2017 21:36:28 +0200 Subject: update social.targaryen.house info --- docs/Using-Mastodon/List-of-Mastodon-instances.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Using-Mastodon/List-of-Mastodon-instances.md b/docs/Using-Mastodon/List-of-Mastodon-instances.md index 780977bd4..d9d0aa2e1 100644 --- a/docs/Using-Mastodon/List-of-Mastodon-instances.md +++ b/docs/Using-Mastodon/List-of-Mastodon-instances.md @@ -17,7 +17,7 @@ There is also a list at [instances.mastodon.xyz](https://instances.mastodon.xyz) | [social.diskseven.com](https://social.diskseven.com) |Single user|No|No (DNS entry but no response)| | [social.gestaltzerfall.net](https://social.gestaltzerfall.net) |Single user|No|No| | [mastodon.xyz](https://mastodon.xyz) |N/A|Yes|Yes| -| [social.targaryen.house](https://social.targaryen.house) |N/A|Yes|No| +| [social.targaryen.house](https://social.targaryen.house) |Federates everywhere, quick updates.|Yes|Yes| | [social.mashek.net](https://social.mashek.net) |Themed and customised for Mashekstein Labs community. Selectively federates.|Yes|No| | [masto.themimitoof.fr](https://masto.themimitoof.fr) |N/A|Yes|Yes| | [social.imirhil.fr](https://social.imirhil.fr) |N/A|No|Yes| -- cgit From 1ed37fae9b9d4807c01d056d6dc81c57b18c59a9 Mon Sep 17 00:00:00 2001 From: Aesen Date: Mon, 3 Apr 2017 15:52:33 -0400 Subject: Add fern.surgeplay.com to the instances list It's hard to tell if this is supposed to be alphabetically sorted or not. I put it after epiktistes since it starts with F - let me know if it should go elsewhere. --- docs/Using-Mastodon/List-of-Mastodon-instances.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Using-Mastodon/List-of-Mastodon-instances.md b/docs/Using-Mastodon/List-of-Mastodon-instances.md index 780977bd4..be9d3723a 100644 --- a/docs/Using-Mastodon/List-of-Mastodon-instances.md +++ b/docs/Using-Mastodon/List-of-Mastodon-instances.md @@ -11,6 +11,7 @@ There is also a list at [instances.mastodon.xyz](https://instances.mastodon.xyz) | [animalliberation.social](https://animalliberation.social) |Animal Rights|Yes|No| | [socially.constructed.space](https://socially.constructed.space) |Single user|No|No| | [epiktistes.com](https://epiktistes.com) |N/A|Yes|No| +| [fern.surgeplay.com](https://fern.surgeplay.com) |Federates everywhere, Minecraft-focused|Yes|No | [gay.crime.team](https://gay.crime.team) |the place for doin' gay crime online (please don't actually do crime here)|Yes|No| | [icosahedron.website](https://icosahedron.website/) |Icosahedron-themed (well, visually), open registration.|Yes|No| | [memetastic.space](https://memetastic.space) |Memes|Yes|No| -- cgit From f722bd2387df9163760014e9555928ec487ae95f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 00:53:20 +0200 Subject: Separate background jobs into different queues. ATTENTION: new queue "pull" must be added to the Sidekiq invokation in your systemd file The pull queue will handle link crawling, thread resolving, and OStatus processing. Such tasks are more likely to hang for a longer time (due to network requests) so it is more sensible to not make the "in-house" tasks wait for them. --- app/workers/after_remote_follow_request_worker.rb | 2 +- app/workers/after_remote_follow_worker.rb | 2 +- app/workers/import_worker.rb | 2 +- app/workers/link_crawl_worker.rb | 2 +- app/workers/merge_worker.rb | 2 ++ app/workers/notification_worker.rb | 2 +- app/workers/processing_worker.rb | 2 +- app/workers/regeneration_worker.rb | 2 ++ app/workers/salmon_worker.rb | 2 +- app/workers/thread_resolve_worker.rb | 2 +- app/workers/unmerge_worker.rb | 2 ++ docker-compose.yml | 2 +- docs/Running-Mastodon/Production-guide.md | 2 +- 13 files changed, 16 insertions(+), 10 deletions(-) diff --git a/app/workers/after_remote_follow_request_worker.rb b/app/workers/after_remote_follow_request_worker.rb index f1d6869cc..1f2db3061 100644 --- a/app/workers/after_remote_follow_request_worker.rb +++ b/app/workers/after_remote_follow_request_worker.rb @@ -3,7 +3,7 @@ class AfterRemoteFollowRequestWorker include Sidekiq::Worker - sidekiq_options retry: 5 + sidekiq_options queue: 'pull', retry: 5 def perform(follow_request_id) follow_request = FollowRequest.find(follow_request_id) diff --git a/app/workers/after_remote_follow_worker.rb b/app/workers/after_remote_follow_worker.rb index 0d04456a9..bdd2c2a91 100644 --- a/app/workers/after_remote_follow_worker.rb +++ b/app/workers/after_remote_follow_worker.rb @@ -3,7 +3,7 @@ class AfterRemoteFollowWorker include Sidekiq::Worker - sidekiq_options retry: 5 + sidekiq_options queue: 'pull', retry: 5 def perform(follow_id) follow = Follow.find(follow_id) diff --git a/app/workers/import_worker.rb b/app/workers/import_worker.rb index a3ae2a85a..7cf29fb53 100644 --- a/app/workers/import_worker.rb +++ b/app/workers/import_worker.rb @@ -5,7 +5,7 @@ require 'csv' class ImportWorker include Sidekiq::Worker - sidekiq_options retry: false + sidekiq_options queue: 'pull', retry: false def perform(import_id) import = Import.find(import_id) diff --git a/app/workers/link_crawl_worker.rb b/app/workers/link_crawl_worker.rb index af3394b8b..834b0088b 100644 --- a/app/workers/link_crawl_worker.rb +++ b/app/workers/link_crawl_worker.rb @@ -3,7 +3,7 @@ class LinkCrawlWorker include Sidekiq::Worker - sidekiq_options retry: false + sidekiq_options queue: 'pull', retry: false def perform(status_id) FetchLinkCardService.new.call(Status.find(status_id)) diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb index 0f288f43f..d745cb99c 100644 --- a/app/workers/merge_worker.rb +++ b/app/workers/merge_worker.rb @@ -3,6 +3,8 @@ class MergeWorker include Sidekiq::Worker + sidekiq_options queue: 'pull' + def perform(from_account_id, into_account_id) FeedManager.instance.merge_into_timeline(Account.find(from_account_id), Account.find(into_account_id)) end diff --git a/app/workers/notification_worker.rb b/app/workers/notification_worker.rb index 1a2faefd8..da1d6ab45 100644 --- a/app/workers/notification_worker.rb +++ b/app/workers/notification_worker.rb @@ -3,7 +3,7 @@ class NotificationWorker include Sidekiq::Worker - sidekiq_options retry: 5 + sidekiq_options queue: 'push', retry: 5 def perform(xml, source_account_id, target_account_id) SendInteractionService.new.call(xml, Account.find(source_account_id), Account.find(target_account_id)) diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb index 5df404bcc..4a467d924 100644 --- a/app/workers/processing_worker.rb +++ b/app/workers/processing_worker.rb @@ -3,7 +3,7 @@ class ProcessingWorker include Sidekiq::Worker - sidekiq_options backtrace: true + sidekiq_options queue: 'pull', backtrace: true def perform(account_id, body) ProcessFeedService.new.call(body, Account.find(account_id)) diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb index 3aece0ba2..289b63d84 100644 --- a/app/workers/regeneration_worker.rb +++ b/app/workers/regeneration_worker.rb @@ -3,6 +3,8 @@ class RegenerationWorker include Sidekiq::Worker + sidekiq_options queue: 'pull', backtrace: true + def perform(account_id, timeline_type) PrecomputeFeedService.new.call(timeline_type, Account.find(account_id)) end diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb index fc95ce47f..2888b574b 100644 --- a/app/workers/salmon_worker.rb +++ b/app/workers/salmon_worker.rb @@ -3,7 +3,7 @@ class SalmonWorker include Sidekiq::Worker - sidekiq_options backtrace: true + sidekiq_options queue: 'pull', backtrace: true def perform(account_id, body) ProcessInteractionService.new.call(body, Account.find(account_id)) diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb index 593edd032..38287e8e6 100644 --- a/app/workers/thread_resolve_worker.rb +++ b/app/workers/thread_resolve_worker.rb @@ -3,7 +3,7 @@ class ThreadResolveWorker include Sidekiq::Worker - sidekiq_options retry: false + sidekiq_options queue: 'pull', retry: false def perform(child_status_id, parent_url) child_status = Status.find(child_status_id) diff --git a/app/workers/unmerge_worker.rb b/app/workers/unmerge_worker.rb index dbf7243de..ea6aacebf 100644 --- a/app/workers/unmerge_worker.rb +++ b/app/workers/unmerge_worker.rb @@ -3,6 +3,8 @@ class UnmergeWorker include Sidekiq::Worker + sidekiq_options queue: 'pull' + def perform(from_account_id, into_account_id) FeedManager.instance.unmerge_from_timeline(Account.find(from_account_id), Account.find(into_account_id)) end diff --git a/docker-compose.yml b/docker-compose.yml index 68c8ef960..d6ba66dde 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: restart: always build: . env_file: .env.production - command: bundle exec sidekiq -q default -q mailers -q push + command: bundle exec sidekiq -q default -q mailers -q pull -q push depends_on: - db - redis diff --git a/docs/Running-Mastodon/Production-guide.md b/docs/Running-Mastodon/Production-guide.md index f0dd7bd2b..469fefa94 100644 --- a/docs/Running-Mastodon/Production-guide.md +++ b/docs/Running-Mastodon/Production-guide.md @@ -180,7 +180,7 @@ User=mastodon WorkingDirectory=/home/mastodon/live Environment="RAILS_ENV=production" Environment="DB_POOL=5" -ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q push +ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q pull -q push TimeoutSec=15 Restart=always -- cgit From 4c53af64f0b10bc11473df5e3fd1cd7a11b755f6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 01:33:34 +0200 Subject: Fix ActionController::Parameters in API issue --- app/controllers/api/v1/apps_controller.rb | 8 +++++++- app/controllers/api/v1/follows_controller.rb | 8 ++++++-- app/controllers/api/v1/media_controller.rb | 8 +++++++- app/controllers/api/v1/reports_controller.rb | 12 +++++++++--- app/controllers/api/v1/statuses_controller.rb | 14 +++++++++----- app/models/status.rb | 2 +- 6 files changed, 39 insertions(+), 13 deletions(-) diff --git a/app/controllers/api/v1/apps_controller.rb b/app/controllers/api/v1/apps_controller.rb index ca9dd0b7e..2ec7280af 100644 --- a/app/controllers/api/v1/apps_controller.rb +++ b/app/controllers/api/v1/apps_controller.rb @@ -4,6 +4,12 @@ class Api::V1::AppsController < ApiController respond_to :json def create - @app = Doorkeeper::Application.create!(name: params[:client_name], redirect_uri: params[:redirect_uris], scopes: (params[:scopes] || Doorkeeper.configuration.default_scopes), website: params[:website]) + @app = Doorkeeper::Application.create!(name: app_params[:client_name], redirect_uri: app_params[:redirect_uris], scopes: (app_params[:scopes] || Doorkeeper.configuration.default_scopes), website: app_params[:website]) + end + + private + + def app_params + params.permit(:client_name, :redirect_uris, :scopes, :website) end end diff --git a/app/controllers/api/v1/follows_controller.rb b/app/controllers/api/v1/follows_controller.rb index c22dacbaa..7c0f44f03 100644 --- a/app/controllers/api/v1/follows_controller.rb +++ b/app/controllers/api/v1/follows_controller.rb @@ -7,7 +7,7 @@ class Api::V1::FollowsController < ApiController respond_to :json def create - raise ActiveRecord::RecordNotFound if params[:uri].blank? + raise ActiveRecord::RecordNotFound if follow_params[:uri].blank? @account = FollowService.new.call(current_user.account, target_uri).try(:target_account) render action: :show @@ -16,6 +16,10 @@ class Api::V1::FollowsController < ApiController private def target_uri - params[:uri].strip.gsub(/\A@/, '') + follow_params[:uri].strip.gsub(/\A@/, '') + end + + def follow_params + params.permit(:uri) end end diff --git a/app/controllers/api/v1/media_controller.rb b/app/controllers/api/v1/media_controller.rb index f8139ade7..aed3578d7 100644 --- a/app/controllers/api/v1/media_controller.rb +++ b/app/controllers/api/v1/media_controller.rb @@ -10,10 +10,16 @@ class Api::V1::MediaController < ApiController respond_to :json def create - @media = MediaAttachment.create!(account: current_user.account, file: params[:file]) + @media = MediaAttachment.create!(account: current_user.account, file: media_params[:file]) rescue Paperclip::Errors::NotIdentifiedByImageMagickError render json: { error: 'File type of uploaded media could not be verified' }, status: 422 rescue Paperclip::Error render json: { error: 'Error processing thumbnail for uploaded media' }, status: 500 end + + private + + def media_params + params.permit(:file) + end end diff --git a/app/controllers/api/v1/reports_controller.rb b/app/controllers/api/v1/reports_controller.rb index 46bdddbc1..f83c573cb 100644 --- a/app/controllers/api/v1/reports_controller.rb +++ b/app/controllers/api/v1/reports_controller.rb @@ -12,13 +12,19 @@ class Api::V1::ReportsController < ApiController end def create - status_ids = params[:status_ids].is_a?(Enumerable) ? params[:status_ids] : [params[:status_ids]] + status_ids = report_params[:status_ids].is_a?(Enumerable) ? report_params[:status_ids] : [report_params[:status_ids]] @report = Report.create!(account: current_account, - target_account: Account.find(params[:account_id]), + target_account: Account.find(report_params[:account_id]), status_ids: Status.find(status_ids).pluck(:id), - comment: params[:comment]) + comment: report_params[:comment]) render :show end + + private + + def report_params + params.permit(:account_id, :comment, status_ids: []) + end end diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb index 024258c0e..4ece7e702 100644 --- a/app/controllers/api/v1/statuses_controller.rb +++ b/app/controllers/api/v1/statuses_controller.rb @@ -62,11 +62,11 @@ class Api::V1::StatusesController < ApiController end def create - @status = PostStatusService.new.call(current_user.account, params[:status], params[:in_reply_to_id].blank? ? nil : Status.find(params[:in_reply_to_id]), media_ids: params[:media_ids], - sensitive: params[:sensitive], - spoiler_text: params[:spoiler_text], - visibility: params[:visibility], - application: doorkeeper_token.application) + @status = PostStatusService.new.call(current_user.account, status_params[:status], status_params[:in_reply_to_id].blank? ? nil : Status.find(status_params[:in_reply_to_id]), media_ids: status_params[:media_ids], + sensitive: status_params[:sensitive], + spoiler_text: status_params[:spoiler_text], + visibility: status_params[:visibility], + application: doorkeeper_token.application) render action: :show end @@ -111,4 +111,8 @@ class Api::V1::StatusesController < ApiController @status = Status.find(params[:id]) raise ActiveRecord::RecordNotFound unless @status.permitted?(current_account) end + + def status_params + params.permit(:status, :in_reply_to_id, :sensitive, :spoiler_text, :visibility, media_ids: []) + end end diff --git a/app/models/status.rb b/app/models/status.rb index 81b26fd14..daf128572 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -188,7 +188,7 @@ class Status < ApplicationRecord end before_validation do - text.strip! + text&.strip! spoiler_text&.strip! self.reply = !(in_reply_to_id.nil? && thread.nil?) unless reply -- cgit From b510a56c0c3d8c1a48bb295a85b688af94466723 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 02:00:10 +0200 Subject: Only call regeneration worker after first login after a 14 day break --- app/controllers/application_controller.rb | 9 ++++++++- app/controllers/oauth/authorizations_controller.rb | 7 +++++++ app/models/feed.rb | 12 ++---------- app/workers/regeneration_worker.rb | 4 ++-- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index ef9364897..c06142fd4 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -39,7 +39,14 @@ class ApplicationController < ActionController::Base end def set_user_activity - current_user.touch(:current_sign_in_at) if !current_user.nil? && (current_user.current_sign_in_at.nil? || current_user.current_sign_in_at < 24.hours.ago) + return unless !current_user.nil? && (current_user.current_sign_in_at.nil? || current_user.current_sign_in_at < 24.hours.ago) + + # Mark user as signed-in today + current_user.update_tracked_fields(request) + + # If the sign in is after a two week break, we need to regenerate their feed + RegenerationWorker.perform_async(current_user.account_id) if current_user.last_sign_in_at < 14.days.ago + return end def check_suspension diff --git a/app/controllers/oauth/authorizations_controller.rb b/app/controllers/oauth/authorizations_controller.rb index feaad04f6..7c25266d8 100644 --- a/app/controllers/oauth/authorizations_controller.rb +++ b/app/controllers/oauth/authorizations_controller.rb @@ -3,6 +3,7 @@ class Oauth::AuthorizationsController < Doorkeeper::AuthorizationsController skip_before_action :authenticate_resource_owner! + before_action :set_locale before_action :store_current_location before_action :authenticate_resource_owner! @@ -11,4 +12,10 @@ class Oauth::AuthorizationsController < Doorkeeper::AuthorizationsController def store_current_location store_location_for(:user, request.url) end + + def set_locale + I18n.locale = current_user.try(:locale) || I18n.default_locale + rescue I18n::InvalidLocale + I18n.locale = I18n.default_locale + end end diff --git a/app/models/feed.rb b/app/models/feed.rb index 5e1905e15..3cbc160a0 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -10,17 +10,9 @@ class Feed max_id = '+inf' if max_id.blank? since_id = '-inf' if since_id.blank? unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:last).map(&:to_i) + status_map = Status.where(id: unhydrated).cache_ids.map { |s| [s.id, s] }.to_h - # If we're after most recent items and none are there, we need to precompute the feed - if unhydrated.empty? && max_id == '+inf' && since_id == '-inf' - RegenerationWorker.perform_async(@account.id, @type) - @statuses = Status.send("as_#{@type}_timeline", @account).cache_ids.paginate_by_max_id(limit, nil, nil) - else - status_map = Status.where(id: unhydrated).cache_ids.map { |s| [s.id, s] }.to_h - @statuses = unhydrated.map { |id| status_map[id] }.compact - end - - @statuses + unhydrated.map { |id| status_map[id] }.compact end private diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb index 289b63d84..82665b581 100644 --- a/app/workers/regeneration_worker.rb +++ b/app/workers/regeneration_worker.rb @@ -5,7 +5,7 @@ class RegenerationWorker sidekiq_options queue: 'pull', backtrace: true - def perform(account_id, timeline_type) - PrecomputeFeedService.new.call(timeline_type, Account.find(account_id)) + def perform(account_id, _ = :home) + PrecomputeFeedService.new.call(:home, Account.find(account_id)) end end -- cgit From eb023beb4975a019d6a3b3091483c91c2c837bbd Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 02:03:16 +0200 Subject: Fix #808 - smaller elephant friend PNG for frontpage --- app/assets/images/fluffy-elephant-friend.png | Bin 1101408 -> 60667 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/app/assets/images/fluffy-elephant-friend.png b/app/assets/images/fluffy-elephant-friend.png index 11787e936..f0df29927 100644 Binary files a/app/assets/images/fluffy-elephant-friend.png and b/app/assets/images/fluffy-elephant-friend.png differ -- cgit From c22388fc792c171dda9c3f80b16b5ae302cd4806 Mon Sep 17 00:00:00 2001 From: walfie Date: Tue, 4 Apr 2017 00:00:56 -0400 Subject: Fix typo in Heroku guide --- docs/Running-Mastodon/Heroku-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Running-Mastodon/Heroku-guide.md b/docs/Running-Mastodon/Heroku-guide.md index 799b8a64c..b66e56200 100644 --- a/docs/Running-Mastodon/Heroku-guide.md +++ b/docs/Running-Mastodon/Heroku-guide.md @@ -8,6 +8,6 @@ Mastodon can theoretically run indefinitely on a free [Heroku](https://heroku.co 1. Click the above button. 2. Fill in the options requested. * You can use a .herokuapp.com domain, which will be simple to set up, or you can use a custom domain. If you want a custom domain and HTTPS, you will need to upgrade to a paid plan (to use Heroku's SSL features), or set up [CloudFlare](https://cloudflare.com) who offer free "Flexible SSL" (note: CloudFlare have some undefined limits on WebSockets. So far, no one has reported hitting concurrent connection limits). - * You will want Amazon S3 for file storage. The only exception is for development purposes, where you may not care if files are not saaved. Follow a guide online for creating a free Amazon S3 bucket and Access Key, then enter the details. + * You will want Amazon S3 for file storage. The only exception is for development purposes, where you may not care if files are not saved. Follow a guide online for creating a free Amazon S3 bucket and Access Key, then enter the details. * If you want your Mastodon to be able to send emails, configure SMTP settings here (or later). Consider using [Mailgun](https://mailgun.com) or similar, who offer free plans that should suit your interests. 3. Deploy! The app should be set up, with a working web interface and database. You can change settings and manage versions from the Heroku dashboard. -- cgit From e81ba26be9fa12cc9efbba665d359b17a01054c6 Mon Sep 17 00:00:00 2001 From: Leo Wzukw Date: Tue, 4 Apr 2017 06:58:17 +0200 Subject: More consistent typography --- docs/Using-Mastodon/User-guide.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Using-Mastodon/User-guide.md b/docs/Using-Mastodon/User-guide.md index f78921c6f..f8018909a 100644 --- a/docs/Using-Mastodon/User-guide.md +++ b/docs/Using-Mastodon/User-guide.md @@ -26,17 +26,17 @@ Mastodon User's Guide ## Intro -Mastodon is a social network application based on the GNU Social protocol. It behaves a lot like other social networks, especially Twitter, with one key difference - it is open-source and anyone can start their own server (also called an "instance"), and users of any instance can interact freely with those of other instances (called "federation"). Thus, it is possible for small communities to set up their own servers to use amongst themselves while also allowing interaction with other communities. +Mastodon is a social network application based on the GNU Social protocol. It behaves a lot like other social networks, especially Twitter, with one key difference - it is open-source and anyone can start their own server (also called an "*instance*"), and users of any instance can interact freely with those of other instances (called "*federation*"). Thus, it is possible for small communities to set up their own servers to use amongst themselves while also allowing interaction with other communities. #### Decentralization and Federation -Mastodon is a system decentralized through a concept called "federation" - rather than depending on a single person or organization to run its infrastructure, anyone can download and run the software and run their own server. Federation means different Mastodon servers can interact with each other seamlessly, similar to e.g. e-mail. +Mastodon is a system decentralized through a concept called "*federation*" - rather than depending on a single person or organization to run its infrastructure, anyone can download and run the software and run their own server. Federation means different Mastodon servers can interact with each other seamlessly, similar to e.g. e-mail. As such, anyone can download Mastodon and e.g. run it for a small community of people, but any user registered on that instance can follow and send and read posts from other Mastodon instances (as well as servers running other GNU Social-compatible services). This means that not only is users' data not inherently owned by a company with an interest in selling it to advertisers, but also that if any given server shuts down its users can set up a new one or migrate to another instance, rather than the entire service being lost. Within each Mastodon instance, usernames just appear as `@username`, similar to other services such as Twitter. Users from other instances appear, and can be searched for and followed, as `@user@servername.ext` - so e.g. `@gargron` on the `mastodon.social` instance can be followed from other instances as `@gargron@mastodon.social`). -Posts from users on external instances are "federated" into the local one, i.e. if `user1@mastodon1` follows `user2@gnusocial2`, any posts `user2@gnusocial2` makes appear in both `user1@mastodon`'s Home feed and the public timeline on the `mastodon1` server. Mastodon server administrators have some control over this and can exclude users' posts from appearing on the public timeline; post privacy settings from users on Mastodon instances also affect this, see below in the [Toot Privacy](User-guide.md#toot-privacy) section. +Posts from users on external instances are "*federated*" into the local one, i.e. if `user1@mastodon1` follows `user2@gnusocial2`, any posts `user2@gnusocial2` makes appear in both `user1@mastodon`'s Home feed and the public timeline on the `mastodon1` server. Mastodon server administrators have some control over this and can exclude users' posts from appearing on the public timeline; post privacy settings from users on Mastodon instances also affect this, see below in the [Toot Privacy](User-guide.md#toot-privacy) section. ## Getting Started -- cgit From ce9df2fa8295a3bdd0da583ba5d0d90251e1d448 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 13:01:14 +0200 Subject: Optimize filter methods in FeedManager a bit, use redis pipelining on merge/unmerge feed methods, do not re-create a dynamic class on each feed push call, make sure redis-rb uses hiredis --- Gemfile | 2 +- app/lib/feed_manager.rb | 72 ++++++++++++++++++++------------------------ app/lib/inline_rabl_scope.rb | 17 +++++++++++ 3 files changed, 50 insertions(+), 41 deletions(-) create mode 100644 app/lib/inline_rabl_scope.rb diff --git a/Gemfile b/Gemfile index 46baed307..cb9824131 100644 --- a/Gemfile +++ b/Gemfile @@ -38,7 +38,7 @@ gem 'rqrcode' gem 'twitter-text' gem 'oj' gem 'hiredis' -gem 'redis', '~>3.2' +gem 'redis', '~>3.2', require: ['redis', 'redis/connection/hiredis'] gem 'fast_blank' gem 'htmlentities' gem 'simple_form' diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index cd6ca1291..2c29275c8 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -51,9 +51,11 @@ class FeedManager def merge_into_timeline(from_account, into_account) timeline_key = key(:home, into_account.id) - from_account.statuses.limit(MAX_ITEMS).each do |status| - next if status.direct_visibility? || filter?(:home, status, into_account) - redis.zadd(timeline_key, status.id, status.id) + redis.pipelined do + from_account.statuses.limit(MAX_ITEMS).each do |status| + next if status.direct_visibility? || filter?(:home, status, into_account) + redis.zadd(timeline_key, status.id, status.id) + end end trim(:home, into_account.id) @@ -62,30 +64,18 @@ class FeedManager def unmerge_from_timeline(from_account, into_account) timeline_key = key(:home, into_account.id) - from_account.statuses.select('id').find_each do |status| - redis.zrem(timeline_key, status.id) - redis.zremrangebyscore(timeline_key, status.id, status.id) + from_account.statuses.select('id').find_in_batches do |statuses| + redis.pipelined do + statuses.each do |status| + redis.zrem(timeline_key, status.id) + redis.zremrangebyscore(timeline_key, status.id, status.id) + end + end end end def inline_render(target_account, template, object) - rabl_scope = Class.new do - include RoutingHelper - - def initialize(account) - @account = account - end - - def current_user - @account.try(:user) - end - - def current_account - @account - end - end - - Rabl::Renderer.new(template, object, view_path: 'app/views', format: :json, scope: rabl_scope.new(target_account)).render + Rabl::Renderer.new(template, object, view_path: 'app/views', format: :json, scope: InlineRablScope.new(target_account)).render end private @@ -95,37 +85,39 @@ class FeedManager end def filter_from_home?(status, receiver) - return true if receiver.muting?(status.account) + return true if status.reply? && status.in_reply_to_id.nil? + + check_for_mutes = [status.account_id] + check_for_mutes.concat([status.reblog.account_id]) if status.reblog? + + return true if receiver.muting?(check_for_mutes) - should_filter = false + check_for_blocks = status.mentions.map(&:account_id) + check_for_blocks.concat([status.reblog.account_id]) if status.reblog? - if status.reply? && status.in_reply_to_id.nil? - should_filter = true - elsif status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply + return true if receiver.blocking?(check_for_blocks) + + if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply should_filter = !receiver.following?(status.in_reply_to_account) # and I'm not following the person it's a reply to should_filter &&= !(receiver.id == status.in_reply_to_account_id) # and it's not a reply to me should_filter &&= !(status.account_id == status.in_reply_to_account_id) # and it's not a self-reply + return should_filter elsif status.reblog? # Filter out a reblog - should_filter = receiver.blocking?(status.reblog.account) # if I'm blocking the reblogged person - should_filter ||= receiver.muting?(status.reblog.account) # or muting that person - should_filter ||= status.reblog.account.blocking?(receiver) # or if the author of the reblogged status is blocking me + return status.reblog.account.blocking?(receiver) # or if the author of the reblogged status is blocking me end - should_filter ||= receiver.blocking?(status.mentions.map(&:account_id)) # or if it mentions someone I blocked - - should_filter + false end def filter_from_mentions?(status, receiver) + check_for_blocks = [status.account_id] + check_for_blocks.concat(status.mentions.select('account_id').map(&:account_id)) + check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil? + should_filter = receiver.id == status.account_id # Filter if I'm mentioning myself - should_filter ||= receiver.blocking?(status.account) # or it's from someone I blocked - should_filter ||= receiver.blocking?(status.mentions.includes(:account).map(&:account)) # or if it mentions someone I blocked + should_filter ||= receiver.blocking?(check_for_blocks) # or it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked should_filter ||= (status.account.silenced? && !receiver.following?(status.account)) # of if the account is silenced and I'm not following them - if status.reply? && !status.in_reply_to_account_id.nil? # or it's a reply - should_filter ||= receiver.blocking?(status.in_reply_to_account) # to a user I blocked - end - should_filter end end diff --git a/app/lib/inline_rabl_scope.rb b/app/lib/inline_rabl_scope.rb new file mode 100644 index 000000000..26adcb03a --- /dev/null +++ b/app/lib/inline_rabl_scope.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class InlineRablScope + include RoutingHelper + + def initialize(account) + @account = account + end + + def current_user + @account.try(:user) + end + + def current_account + @account + end +end -- cgit From b21f7c28f6832817d5de616ab0c4c2d3c28d90b0 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 13:02:49 +0200 Subject: Move OStatus processing back into default queue --- app/workers/processing_worker.rb | 2 +- app/workers/salmon_worker.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb index 4a467d924..5df404bcc 100644 --- a/app/workers/processing_worker.rb +++ b/app/workers/processing_worker.rb @@ -3,7 +3,7 @@ class ProcessingWorker include Sidekiq::Worker - sidekiq_options queue: 'pull', backtrace: true + sidekiq_options backtrace: true def perform(account_id, body) ProcessFeedService.new.call(body, Account.find(account_id)) diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb index 2888b574b..fc95ce47f 100644 --- a/app/workers/salmon_worker.rb +++ b/app/workers/salmon_worker.rb @@ -3,7 +3,7 @@ class SalmonWorker include Sidekiq::Worker - sidekiq_options queue: 'pull', backtrace: true + sidekiq_options backtrace: true def perform(account_id, body) ProcessInteractionService.new.call(body, Account.find(account_id)) -- cgit From b1f3499c3806682375a0496f99b4bc908d89cd84 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 13:43:36 +0200 Subject: Optimize FeedManager#unmerge, and slightly optimize FeedManager#merge --- app/lib/feed_manager.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 2c29275c8..919bc3df9 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -50,9 +50,15 @@ class FeedManager def merge_into_timeline(from_account, into_account) timeline_key = key(:home, into_account.id) + query = from_account.statuses.limit(MAX_ITEMS) + + if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS + oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 + query = query.where('id > ?', oldest_home_score) + end redis.pipelined do - from_account.statuses.limit(MAX_ITEMS).each do |status| + query.each do |status| next if status.direct_visibility? || filter?(:home, status, into_account) redis.zadd(timeline_key, status.id, status.id) end @@ -63,8 +69,9 @@ class FeedManager def unmerge_from_timeline(from_account, into_account) timeline_key = key(:home, into_account.id) + oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 - from_account.statuses.select('id').find_in_batches do |statuses| + from_account.statuses.select('id').where('id > ?', oldest_home_score).find_in_batches do |statuses| redis.pipelined do statuses.each do |status| redis.zrem(timeline_key, status.id) -- cgit From 82aaedec467815c2947a11651d5216bb88ce4038 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 13:58:34 +0200 Subject: Reduce number of items in feeds, optimize regeneration worker slightly, make regeneration worker unique, (only schedule/execute once at a time) --- Gemfile | 2 ++ Gemfile.lock | 9 +++++++++ app/lib/feed_manager.rb | 6 +++--- app/services/precompute_feed_service.rb | 8 +++++--- app/workers/regeneration_worker.rb | 2 +- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index cb9824131..41c636904 100644 --- a/Gemfile +++ b/Gemfile @@ -46,6 +46,8 @@ gem 'will_paginate' gem 'rack-attack' gem 'rack-cors', require: 'rack/cors' gem 'sidekiq' +gem 'sidekiq-unique-jobs' +gem 'sidekiq-merger' gem 'rails-settings-cached' gem 'simple-navigation' gem 'statsd-instrument' diff --git a/Gemfile.lock b/Gemfile.lock index 6e3115249..27de1bee0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -387,6 +387,13 @@ GEM connection_pool (~> 2.2, >= 2.2.0) rack-protection (>= 1.5.0) redis (~> 3.2, >= 3.2.1) + sidekiq-merger (0.0.11) + activesupport (>= 3.2, < 6) + concurrent-ruby (~> 1.0) + sidekiq (>= 3.4, < 5) + sidekiq-unique-jobs (4.0.18) + sidekiq (>= 2.6) + thor simple-navigation (4.0.3) activesupport (>= 2.3.2) simple_form (3.2.1) @@ -510,6 +517,8 @@ DEPENDENCIES sass-rails (~> 5.0) sdoc (~> 0.4.0) sidekiq + sidekiq-merger + sidekiq-unique-jobs simple-navigation simple_form simplecov diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 919bc3df9..a2efcce10 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -5,7 +5,7 @@ require 'singleton' class FeedManager include Singleton - MAX_ITEMS = 800 + MAX_ITEMS = 400 def key(type, id) "feed:#{type}:#{id}" @@ -50,9 +50,9 @@ class FeedManager def merge_into_timeline(from_account, into_account) timeline_key = key(:home, into_account.id) - query = from_account.statuses.limit(MAX_ITEMS) + query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4) - if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS + if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4 oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 query = query.where('id > ?', oldest_home_score) end diff --git a/app/services/precompute_feed_service.rb b/app/services/precompute_feed_service.rb index e1ec56e8d..a57c401d0 100644 --- a/app/services/precompute_feed_service.rb +++ b/app/services/precompute_feed_service.rb @@ -5,9 +5,11 @@ class PrecomputeFeedService < BaseService # @param [Symbol] type :home or :mentions # @param [Account] account def call(_, account) - Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS).each do |status| - next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account) - redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id) + redis.pipelined do + Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS / 4).each do |status| + next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account) + redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id) + end end end diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb index 82665b581..da8b845f6 100644 --- a/app/workers/regeneration_worker.rb +++ b/app/workers/regeneration_worker.rb @@ -3,7 +3,7 @@ class RegenerationWorker include Sidekiq::Worker - sidekiq_options queue: 'pull', backtrace: true + sidekiq_options queue: 'pull', backtrace: true, unique: :until_executed def perform(account_id, _ = :home) PrecomputeFeedService.new.call(:home, Account.find(account_id)) -- cgit From 38b504b7a70c5b100396f36d4c6c6762542984c9 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 14:28:57 +0200 Subject: Remove sidekiq-merger --- Gemfile | 1 - Gemfile.lock | 5 ----- 2 files changed, 6 deletions(-) diff --git a/Gemfile b/Gemfile index 41c636904..4c6314763 100644 --- a/Gemfile +++ b/Gemfile @@ -47,7 +47,6 @@ gem 'rack-attack' gem 'rack-cors', require: 'rack/cors' gem 'sidekiq' gem 'sidekiq-unique-jobs' -gem 'sidekiq-merger' gem 'rails-settings-cached' gem 'simple-navigation' gem 'statsd-instrument' diff --git a/Gemfile.lock b/Gemfile.lock index 27de1bee0..26c7b9962 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -387,10 +387,6 @@ GEM connection_pool (~> 2.2, >= 2.2.0) rack-protection (>= 1.5.0) redis (~> 3.2, >= 3.2.1) - sidekiq-merger (0.0.11) - activesupport (>= 3.2, < 6) - concurrent-ruby (~> 1.0) - sidekiq (>= 3.4, < 5) sidekiq-unique-jobs (4.0.18) sidekiq (>= 2.6) thor @@ -517,7 +513,6 @@ DEPENDENCIES sass-rails (~> 5.0) sdoc (~> 0.4.0) sidekiq - sidekiq-merger sidekiq-unique-jobs simple-navigation simple_form -- cgit From be2e7e18029d1b461decb0771d2b075ddc959e48 Mon Sep 17 00:00:00 2001 From: Niclas Darville Date: Tue, 4 Apr 2017 14:46:08 +0200 Subject: Create ISSUE_TEMPLATE.md --- ISSUE_TEMPLATE.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 ISSUE_TEMPLATE.md diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..142b930a9 --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,5 @@ +[Issue text goes here]. + +* * * * + +- [ ] I searched or or browsed the repo’s other issues to ensure this is not a duplicate. -- cgit From 904f9266ef25e073e4b3d592d1d689d60a1464b8 Mon Sep 17 00:00:00 2001 From: Niclas Darville Date: Tue, 4 Apr 2017 14:49:31 +0200 Subject: Fix typo in ISSUE_TEMPLATE --- ISSUE_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 142b930a9..8394b2424 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -2,4 +2,4 @@ * * * * -- [ ] I searched or or browsed the repo’s other issues to ensure this is not a duplicate. +- [ ] I searched or browsed the repo’s other issues to ensure this is not a duplicate. -- cgit